diff --git a/.buckconfig b/.buckconfig new file mode 100644 index 0000000000..495ec1734d --- /dev/null +++ b/.buckconfig @@ -0,0 +1,35 @@ +[cells] + effect_utils = . + capabilities = .buck2/capabilities + prelude = prelude + +[cell_aliases] + config = prelude + ovr_config = prelude + fbsource = prelude + toolchains = effect_utils + +[external_cells] + prelude = bundled + +[parser] + target_platform_detector_spec = target:effect_utils//...->effect_utils//buck2/platforms:host_platform + +[build] + execution_platforms = effect_utils//buck2/platforms:host_execution_platform + +[buck2] + file_watcher = notify + default_allow_cache_upload = true + digest_algorithms = SHA256 + +[buck2_re_client] + action_cache_address = grpc://dev3:41045 + cas_address = grpc://dev3:41045 + engine_address = grpc://dev3:41045 + http_headers = authorization: Basic $BUCK2_REMOTE_CACHE_BASIC_AUTH + instance_name = effect-utils + tls = false + +[project] + ignore = **/__pycache__,**/dist,**/node_modules,**/node_modules/**,**/target,**/target/**,.buck2/capabilities.candidate.*,.devenv,.git,buck-out,node_modules,packages/.editor-view,target,tmp diff --git a/.buckroot b/.buckroot new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61e99ddbe5..f1577e71cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -440,6 +440,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -460,10 +468,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -481,7 +485,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -495,7 +499,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -508,7 +512,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -528,8 +532,8 @@ jobs: GITHUB_TOKEN: ${{ github.token }} shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run otel:test' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run otel:test' + cd "${GITHUB_WORKSPACE:-$PWD}" && __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' + bash "$__genie_ci_retry_script" 'devenv tasks run otel:test' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run otel:test' command -v script >/dev/null 2>&1 tmp_log="$(mktemp)" before="$(git status --porcelain=v1)" @@ -545,7 +549,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run buck2:check' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run buck2:check' + bash "$__genie_ci_retry_script" 'devenv tasks run buck2:check' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run buck2:check' - name: Nix diagnostics summary if: failure() shell: bash @@ -596,9 +600,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-typecheck" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -615,6 +657,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -635,10 +685,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -656,7 +702,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -670,7 +716,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -683,7 +729,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -703,7 +749,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run genie:check lint:check' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run genie:check lint:check' + bash "$__genie_ci_retry_script" 'devenv tasks run genie:check lint:check' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run genie:check lint:check' - name: Nix diagnostics summary if: failure() shell: bash @@ -754,9 +800,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-lint" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -776,6 +860,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -796,10 +888,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -817,7 +905,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -831,7 +919,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -844,7 +932,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -864,7 +952,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run test:run' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:run' + bash "$__genie_ci_retry_script" 'devenv tasks run test:run' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:run' - name: Nix diagnostics summary if: failure() shell: bash @@ -915,9 +1003,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-test-${{ strategy.job-index }}" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -934,6 +1060,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -954,10 +1088,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -975,7 +1105,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -989,7 +1119,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -1002,7 +1132,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -1022,7 +1152,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run test:pw:utils' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:pw:utils' + bash "$__genie_ci_retry_script" 'devenv tasks run test:pw:utils' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:pw:utils' - name: Nix diagnostics summary if: failure() shell: bash @@ -1073,9 +1203,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-test-playwright-utils" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -1092,6 +1260,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -1112,10 +1288,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -1133,7 +1305,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -1147,7 +1319,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -1160,7 +1332,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -1180,7 +1352,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run test:pw:tui-react' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:pw:tui-react' + bash "$__genie_ci_retry_script" 'devenv tasks run test:pw:tui-react' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:pw:tui-react' - name: Nix diagnostics summary if: failure() shell: bash @@ -1231,9 +1403,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-test-playwright-tui-react" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -1250,6 +1460,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -1270,10 +1488,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -1291,7 +1505,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -1305,7 +1519,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -1318,7 +1532,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -1338,7 +1552,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run test:megarepo-cold-gc' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:megarepo-cold-gc' + bash "$__genie_ci_retry_script" 'devenv tasks run test:megarepo-cold-gc' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:megarepo-cold-gc' - name: Nix diagnostics summary if: failure() shell: bash @@ -1389,9 +1603,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-test-megarepo-cold-gc" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -1408,6 +1660,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -1428,10 +1688,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -1449,7 +1705,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -1463,7 +1719,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -1476,7 +1732,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -1496,7 +1752,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail audit=genie/ci-scripts/native-dep-policy-audit.ts if command -v bun >/dev/null 2>&1; then bun "$audit" @@ -1506,7 +1762,7 @@ jobs: - name: Guard pnpm builder contract shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail builder='nix/workspace-tools/lib/mk-pnpm-deps.nix' policy='nix/workspace-tools/lib/pnpm-install-policy.nix' if [ ! -f "$builder" ]; then @@ -1598,9 +1854,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-pnpm-builder-contract" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -1617,6 +1911,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -1637,10 +1939,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -1658,7 +1956,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -1672,7 +1970,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -1685,7 +1983,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -1704,7 +2002,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && bash genie/ci-scripts/nix-gc-race-retry.test.sh + cd "${GITHUB_WORKSPACE:-$PWD}" && bash genie/ci-scripts/nix-gc-race-retry.test.sh bash genie/ci-scripts/ci-measurement-comparison.test.sh bash genie/ci-scripts/native-dep-policy-audit.test.sh bash nix/workspace-tools/lib/mk-pnpm-cli/tests/run.sh --skip-genie --skip-megarepo --skip-devenv-shell --skip-downstream-megarepo @@ -1758,9 +2056,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-pnpm-regression" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -1777,6 +2113,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -1797,10 +2141,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -1818,7 +2158,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -1832,7 +2172,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -1845,7 +2185,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -1865,7 +2205,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run bundle:smoke' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run bundle:smoke' + bash "$__genie_ci_retry_script" 'devenv tasks run bundle:smoke' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run bundle:smoke' - name: Nix diagnostics summary if: failure() shell: bash @@ -1916,9 +2256,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-bundle-smoke" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -1935,6 +2313,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -1955,10 +2341,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -1976,7 +2358,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -1990,7 +2372,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -2003,7 +2385,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -2023,7 +2405,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run cargo:check' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run cargo:check' + bash "$__genie_ci_retry_script" 'devenv tasks run cargo:check' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run cargo:check' - name: Nix diagnostics summary if: failure() shell: bash @@ -2074,9 +2456,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-cargo" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -2093,6 +2513,14 @@ jobs: run: shell: bash steps: + - name: Prepare CI OpenTelemetry capture + shell: bash + run: | + set -euo pipefail + spool_dir="${{ runner.temp }}/composition-state/otel-spans" + mkdir -p "$spool_dir" + printf 'OTEL_SPAN_SPOOL_DIR=%s\n' "$spool_dir" >> "$GITHUB_ENV" + printf 'OTEL_SPOOL_MULTI_WRITER=1\n' >> "$GITHUB_ENV" - uses: actions/checkout@v6 with: persist-credentials: false @@ -2113,10 +2541,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -2134,7 +2558,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -2148,7 +2572,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -2161,7 +2585,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -2178,7 +2602,7 @@ jobs: exit 1 - name: 'Fetch baseline history for weaver:diff (SC-R11)' run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -uo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -uo pipefail # weaver:diff needs the merge-base with origin/main; the default checkout is shallow. git fetch --no-tags --prune --unshallow origin 2>/dev/null \ || git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main 2>/dev/null \ @@ -2188,7 +2612,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run weaver:check weaver:diff weaver:live-check' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run weaver:check weaver:diff weaver:live-check' + bash "$__genie_ci_retry_script" 'devenv tasks run weaver:check weaver:diff weaver:live-check' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run weaver:check weaver:diff weaver:live-check' - name: Nix diagnostics summary if: failure() shell: bash @@ -2239,9 +2663,47 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' + - name: Summarize CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl) + if [ "${#span_files[@]}" -eq 0 ]; then + echo "::notice::No CI OpenTelemetry task spans were emitted" + exit 0 + fi + { + echo "## Slowest devenv tasks" + echo "" + echo "| Task | Duration | Status |" + echo "| --- | ---: | --- |" + while IFS=$'\t' read -r duration task status; do + printf '| `%s` | %.2fs | %s |\n' "$task" "$duration" "$status" + done < <( + jq -s -r ' + [.[].resourceSpans[].scopeSpans[].spans[] + | select(.name == "devenv.task.exec") + | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs + | [ + (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000), + ($attrs["task.name"] // $attrs["span.label"] // .name), + (if (.status.code // 0) == 2 then "error" else "ok" end) + ] + | @tsv + ] | .[] + ' "${span_files[@]}" | sort -nr | sed -n '1,20p' + ) + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload CI OpenTelemetry spans + if: always() && env.OTEL_SPAN_SPOOL_DIR != '' + uses: actions/upload-artifact@v4 + with: + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: ${{ env.OTEL_SPAN_SPOOL_DIR }} + if-no-files-found: warn + retention-days: 14 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-weaver" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -2290,14 +2752,10 @@ jobs: uses: cachix/cachix-action@v17 with: name: overeng-effect-utils - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -2311,7 +2769,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -2324,19 +2782,19 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Check generated sources env: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run genie:check' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run genie:check' + bash "$__genie_ci_retry_script" 'devenv tasks run genie:check' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run genie:check' - name: Run focused normalized, projection, and runner tests env: GITHUB_TOKEN: ${{ github.token }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail "${DEVENV_BIN:?DEVENV_BIN not set}" shell -- bun test \ genie/buck2/typescript-package-projection.unit.test.ts \ genie/buck2/javascript-candidates.unit.test.ts \ @@ -2346,12 +2804,12 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail "${DEVENV_BIN:?DEVENV_BIN not set}" shell -- env -u GITHUB_EVENT_NAME bash nix/workspace-tools/lib/tests/buck2-release-products.sh "$PWD" "${DEVENV_BIN:?DEVENV_BIN not set}" shell -- bash nix/workspace-tools/lib/tests/javascript-product-import.sh "$PWD" - name: Reject tracked product and editor payload bytes run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail tracked_editor=$(git ls-files -- '**/.editor-view/**' '.editor-view/**') tracked_product=$(git ls-files -- 'nix/buck2-products/**' | grep -Ev '^nix/buck2-products/(cache\.nix|cache-targets\.json|cache-targets\.json\.genie\.ts|consumer-root\.nix|default\.nix|from-source\.nix|manifest\.json|publish\.sh|source-recipes\.nix|targets\.json|targets\.json\.genie\.ts)$' || true) if [ -n "$tracked_editor$tracked_product" ]; then @@ -2362,19 +2820,16 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail "${DEVENV_BIN:?DEVENV_BIN not set}" shell -- bash -euo pipefail -c ' - cd "${EFFECT_UTILS_WORKSPACE_ROOT:?EFFECT_UTILS_WORKSPACE_ROOT not set}" - buck="$PWD/.megarepo/bin/buck2" + cd "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE not set}" + buck="${BUCK2_BIN:?BUCK2_BIN not set}" "$buck" query effect_utils//packages/@overeng/ci-tools:ci-tools-candidate "$buck" query effect_utils//:editor_view_inputs "$buck" build \ effect_utils//packages/@overeng/ci-tools:ci-tools-candidate \ effect_utils//:editor_view_inputs ' - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-pr-a-inert-buck" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -2428,11 +2883,6 @@ jobs: access-tokens = github.com=${{ github.token }} summarize: true if: steps.publication-scope.outputs.publish == 'true' - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - if: steps.publication-scope.outputs.publish == 'true' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -2454,7 +2904,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail proposal="${RUNNER_TEMP:?RUNNER_TEMP not set}/buck2-products-manifest.json" mapfile -t product_names < <(jq -r '.products[].name' nix/buck2-products/cache-targets.json) product_refs=() @@ -2484,9 +2934,6 @@ jobs: --title 'chore(buck2): update product cache manifest' \ --body 'Updates the generated v2 manifest after the trusted main-branch cache publication job.' fi - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-publish-products" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -2520,14 +2967,10 @@ jobs: extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= access-tokens = github.com=${{ github.token }} summarize: true - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -2541,7 +2984,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -2554,7 +2997,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Prove fresh-root remote action and test-cache hits env: @@ -2570,13 +3013,14 @@ jobs: echo "::error::BUCK2_REMOTE_CACHE_BASIC_AUTH is required for the trusted remote-cache proof" exit 1 fi - cd "${EFFECT_UTILS_WORKSPACE_ROOT:?EFFECT_UTILS_WORKSPACE_ROOT not set}" - buck="$PWD/.megarepo/bin/buck2" + source_root="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE not set}" + cd "$source_root" + buck="${BUCK2_BIN:?BUCK2_BIN not set}" context_b="${RUNNER_TEMP:?RUNNER_TEMP not set}/buck2-remote-cache-proof-context-b" target="effect_utils//packages/@overeng/ci-tools:ci-tools-candidate" test_target="effect_utils//packages/@overeng/content-address:test" - proof_source="${EFFECT_UTILS_MEMBER_ROOT:?EFFECT_UTILS_MEMBER_ROOT not set}/packages/@overeng/ci-tools/bin/ci-tools.ts" - test_proof_source="${EFFECT_UTILS_MEMBER_ROOT:?EFFECT_UTILS_MEMBER_ROOT not set}/packages/@overeng/content-address/src/mod.unit.test.ts" + proof_source="$source_root/packages/@overeng/ci-tools/bin/ci-tools.ts" + test_proof_source="$source_root/packages/@overeng/content-address/src/mod.unit.test.ts" printf '%s\n' '' "// trusted remote-cache proof ${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> "$proof_source" printf '%s\n' '' "// trusted test-cache proof ${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> "$test_proof_source" evidence_a="${RUNNER_TEMP:?RUNNER_TEMP not set}/buck2-remote-cache-proof-a.jsonl" @@ -2587,7 +3031,6 @@ jobs: trap 'rm -f "$evidence_a" "$test_evidence_a" "$evidence_b" "$test_evidence_b" "$test_evidence_c"; rm -rf "$context_b"' EXIT # Context A has run-unique source inputs, executes locally, and uploads to the remote cache. - # The composition wrapper fixes --isolation-dir, so freshness comes from daemon and state removal. "$buck" kill rm -rf buck-out "$buck" build --local-only "$target" @@ -2603,14 +3046,11 @@ jobs: exit 1 fi - # Context B is a second composed root with a fresh daemon and materializer over identical inputs. + # Context B is a second standalone root with a fresh daemon and materializer over identical inputs. "$buck" kill rm -rf buck-out "$context_b" - mkdir -p "$context_b/.buck2" "$context_b/.megarepo" "$context_b/repos/effect-utils" - cp -a .buckconfig .buckroot BUCK megarepo.kdl "$context_b/" - cp -a .buck2/capabilities "$context_b/.buck2/" - cp -a .megarepo/bin "$context_b/.megarepo/" - tar -C repos/effect-utils \ + mkdir -p "$context_b" + tar -C "$source_root" \ --exclude='./.devenv' \ --exclude='./.git' \ --exclude='./buck-out' \ @@ -2622,9 +3062,8 @@ jobs: --exclude='*/dist' \ --exclude='*/node_modules' \ --exclude='*/target' \ - -cf - . | tar -C "$context_b/repos/effect-utils" -xf - + -cf - . | tar -C "$context_b" -xf - cd "$context_b" - buck="$PWD/.megarepo/bin/buck2" # Buck event data must classify the independent build as a remote action-cache hit. "$buck" build --local-only "$target" @@ -2651,7 +3090,7 @@ jobs: fi # A source file outside the representative target graph must not change its test action key. - printf '%s\n' '' "// trusted irrelevant-mutation proof ${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> repos/effect-utils/README.md + printf '%s\n' '' "// trusted irrelevant-mutation proof ${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> README.md "$buck" test --target-platforms effect_utils//buck2/platforms:host_platform --local-only "$test_target" "$buck" log show --recent 1 > "$test_evidence_c" if ! jq -e 'select(.Event.data.Instant.data.TestResult.name == "effect_utils//packages/@overeng/content-address:test" and .Event.data.Instant.data.TestResult.status == 1)' "$test_evidence_c" >/dev/null; then @@ -2665,9 +3104,6 @@ jobs: echo "Fresh-root remote action and test-cache proof passed" BUCK2_REMOTE_CACHE_PROOF "${DEVENV_BIN:?DEVENV_BIN not set}" shell -- bash "$proof_script" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-trusted-buck2-remote-cache-proof" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -2700,10 +3136,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -2721,7 +3153,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -2735,7 +3167,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -2748,7 +3180,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -2768,7 +3200,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run bootstrap:cold-proof' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run bootstrap:cold-proof' + bash "$__genie_ci_retry_script" 'devenv tasks run bootstrap:cold-proof' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run bootstrap:cold-proof' - name: Nix diagnostics summary if: failure() shell: bash @@ -2819,9 +3251,6 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-bootstrap-cold-proof" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -2874,10 +3303,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -2895,7 +3320,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -2909,7 +3334,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -2922,7 +3347,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -2942,7 +3367,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail ensure_ci_measurement_tool() { tool_name="$1" @@ -3632,9 +4057,6 @@ jobs: !${{ github.workspace }}/tmp/devenv-perf-ci/baseline/** if-no-files-found: error retention-days: 7 - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' timeout-minutes: 90 concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-devenv-perf" @@ -3740,10 +4162,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -3761,7 +4179,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -3775,7 +4193,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -3788,7 +4206,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -3807,10 +4225,10 @@ jobs: shell: bash env: GITHUB_TOKEN: ${{ github.token }} - ARTIFACT_DIR: '${{ env.EFFECT_UTILS_MEMBER_ROOT }}/tmp/nix-closure-ci/current/genie_package' + ARTIFACT_DIR: '${{ github.workspace }}/tmp/nix-closure-ci/current/genie_package' RUNNER_CLASS: '${{ runner.os }}-${{ runner.arch }}' run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail mkdir -p "$ARTIFACT_DIR" installable='.#genie' @@ -3964,10 +4382,10 @@ jobs: shell: bash env: GITHUB_TOKEN: ${{ github.token }} - ARTIFACT_DIR: '${{ env.EFFECT_UTILS_MEMBER_ROOT }}/tmp/nix-closure-ci/current/megarepo_package' + ARTIFACT_DIR: '${{ github.workspace }}/tmp/nix-closure-ci/current/megarepo_package' RUNNER_CLASS: '${{ runner.os }}-${{ runner.arch }}' run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail mkdir -p "$ARTIFACT_DIR" installable='.#megarepo' @@ -4121,10 +4539,10 @@ jobs: shell: bash env: GITHUB_TOKEN: ${{ github.token }} - ARTIFACT_DIR: '${{ env.EFFECT_UTILS_MEMBER_ROOT }}/tmp/nix-closure-ci/current/oxlint_npm_package' + ARTIFACT_DIR: '${{ github.workspace }}/tmp/nix-closure-ci/current/oxlint_npm_package' RUNNER_CLASS: '${{ runner.os }}-${{ runner.arch }}' run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail mkdir -p "$ARTIFACT_DIR" installable='.#oxlint-npm' @@ -4280,8 +4698,8 @@ jobs: with: name: nix-closure-measurements path: | - ${{ env.EFFECT_UTILS_MEMBER_ROOT }}/tmp/nix-closure-ci - !${{ env.EFFECT_UTILS_MEMBER_ROOT }}/tmp/nix-closure-ci/baseline/** + ${{ github.workspace }}/tmp/nix-closure-ci + !${{ github.workspace }}/tmp/nix-closure-ci/baseline/** if-no-files-found: error retention-days: 14 - name: Nix diagnostics summary @@ -4334,9 +4752,6 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-nix-closure-sizes" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -7173,10 +7588,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -7194,7 +7605,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -7208,7 +7619,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -7221,7 +7632,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -7246,7 +7657,7 @@ jobs: NOTION_DATASOURCE_SYNC_DEMO_PAGE_ID: ${{ github.event_name == 'workflow_dispatch' && (inputs.run_datasource_sync_demo == true || inputs.run_datasource_sync_demo == 'true') && secrets.NOTION_DATASOURCE_SYNC_DEMO_PAGE_ID || '' }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run test:notion-integration' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:notion-integration' + bash "$__genie_ci_retry_script" 'devenv tasks run test:notion-integration' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:notion-integration' - name: Nix diagnostics summary if: failure() shell: bash @@ -7297,9 +7708,6 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-test-integration-notion" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} @@ -7339,10 +7747,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -7360,7 +7764,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -7374,7 +7778,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -7387,7 +7791,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -7407,7 +7811,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run test:restate-integration' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:restate-integration' + bash "$__genie_ci_retry_script" 'devenv tasks run test:restate-integration' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run test:restate-integration' - name: Nix diagnostics summary if: failure() shell: bash @@ -7458,9 +7862,6 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' env: FORCE_SETUP: '1' CI: 'true' @@ -7497,10 +7898,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -7518,7 +7915,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -7532,7 +7929,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -7545,7 +7942,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -7596,7 +7993,7 @@ jobs: NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && netlify_pkg="$(nix build --no-link --print-out-paths .#netlify-cli)" + cd "${GITHUB_WORKSPACE:-$PWD}" && netlify_pkg="$(nix build --no-link --print-out-paths .#netlify-cli)" export CI_TOOLS_LIVE_NETLIFY_BIN="$netlify_pkg/bin/netlify" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run buck2:editor:publish DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" shell --no-reload -- bun test packages/@overeng/ci-tools/src/deploy-netlify.live.e2e.test.ts @@ -7613,7 +8010,7 @@ jobs: VERCEL_SCOPE: ${{ secrets.VERCEL_SCOPE }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && vercel_pkg="$(nix build --no-link --print-out-paths .#vercel-cli)" + cd "${GITHUB_WORKSPACE:-$PWD}" && vercel_pkg="$(nix build --no-link --print-out-paths .#vercel-cli)" export CI_TOOLS_LIVE_VERCEL_BIN="$vercel_pkg/bin/vercel" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run buck2:editor:publish DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" shell --no-reload -- bun test packages/@overeng/ci-tools/src/deploy-vercel.live.e2e.test.ts @@ -7668,9 +8065,6 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' env: FORCE_SETUP: '1' CI: 'true' @@ -7706,10 +8100,6 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.measurement_baseline_ref }} - - name: Prepare effect-utils composition - env: - GITHUB_TOKEN: ${{ github.token }} - run: '$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh' - name: Provide cachix CLI from nixpkgs shell: bash env: @@ -7727,7 +8117,7 @@ jobs: - name: Prepare CI helper scripts shell: bash run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && set -euo pipefail + cd "${GITHUB_WORKSPACE:-$PWD}" && set -euo pipefail scripts_src='genie/ci-scripts' scripts_dst='${{ runner.temp }}/composition-state/ci-runtime' if [ ! -d "$scripts_src" ]; then @@ -7741,7 +8131,7 @@ jobs: chmod +x "$scripts_dst"/*.sh - name: Use pinned devenv from lock run: | - cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') + cd "${GITHUB_WORKSPACE:-$PWD}" && DEVENV_REV=$(jq -r .nodes.devenv.locked.rev 'devenv.lock') if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then printf '::error::%s missing .nodes.devenv.locked.rev\n' 'devenv.lock' exit 1 @@ -7754,7 +8144,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' + bash "$__genie_ci_retry_script" 'resolve devenv (devenv.lock)' 'cd "${GITHUB_WORKSPACE:-$PWD}" && '"'"'${{ runner.temp }}/composition-state/ci-runtime/resolve-devenv.sh'"'"' '"'"'devenv.lock'"'"'' shell: bash - name: Force diagnostics failure (debug) if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} @@ -7781,11 +8171,11 @@ jobs: if [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "refs/heads/main" ]; then deploy_ran=1 __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run netlify:deploy --show-output --input type=prod --input missingAuthPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run netlify:deploy --show-output --input type=prod --input missingAuthPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' + bash "$__genie_ci_retry_script" 'devenv tasks run netlify:deploy --show-output --input type=prod --input missingAuthPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run netlify:deploy --show-output --input type=prod --input missingAuthPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' elif [ "${{ github.event_name }}" = "pull_request" ]; then deploy_ran=1 __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run netlify:deploy --show-output --input type=pr --input pr=${{ github.event.pull_request.number }} --input missingAuthPolicy=skip --input unauthorizedPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run netlify:deploy --show-output --input type=pr --input pr=${{ github.event.pull_request.number }} --input missingAuthPolicy=skip --input unauthorizedPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' + bash "$__genie_ci_retry_script" 'devenv tasks run netlify:deploy --show-output --input type=pr --input pr=${{ github.event.pull_request.number }} --input missingAuthPolicy=skip --input unauthorizedPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run netlify:deploy --show-output --input type=pr --input pr=${{ github.event.pull_request.number }} --input missingAuthPolicy=skip --input unauthorizedPolicy=skip --input urlEnvKey=NETLIFY_DEPLOY_URL_STORYBOOK' fi if [ "$deploy_ran" = "1" ] && [ ! -s "$workflow_report_path" ]; then echo "Error: ci-tools did not emit a Netlify workflow report record." >&2 @@ -7809,7 +8199,7 @@ jobs: WORKFLOW_REPORT_ALLOW_MISSING_INPUT: '1' run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run workflow-report:collect-bundle --show-output' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run workflow-report:collect-bundle --show-output' + bash "$__genie_ci_retry_script" 'devenv tasks run workflow-report:collect-bundle --show-output' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run workflow-report:collect-bundle --show-output' - name: Render workflow report comment shell: bash env: @@ -7831,7 +8221,7 @@ jobs: WORKFLOW_REPORT_MANAGED_MARKER: '' run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run workflow-report:render-comment-body --show-output' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run workflow-report:render-comment-body --show-output' + bash "$__genie_ci_retry_script" 'devenv tasks run workflow-report:render-comment-body --show-output' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run workflow-report:render-comment-body --show-output' - name: Publish workflow report if: always() && !cancelled() shell: bash @@ -7848,7 +8238,7 @@ jobs: WORKFLOW_REPORT_MANAGED_MARKER: '' run: | __genie_ci_retry_script='${{ runner.temp }}/composition-state/ci-runtime/run-with-nix-gc-race-retry.sh' - bash "$__genie_ci_retry_script" 'devenv tasks run workflow-report:publish --show-output' 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run workflow-report:publish --show-output' + bash "$__genie_ci_retry_script" 'devenv tasks run workflow-report:publish --show-output' 'cd "${GITHUB_WORKSPACE:-$PWD}" && if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run workflow-report:publish --show-output' - name: Nix diagnostics summary if: failure() shell: bash @@ -7899,9 +8289,6 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" - - name: Cleanup effect-utils composition - if: always() - run: '$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh' concurrency: group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '' && format('measurement-baseline-{0}', inputs.measurement_baseline_ref) || (github.event_name == 'workflow_dispatch' && format('manual-run-{0}', github.run_id) || (github.event_name == 'pull_request' && (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('label-{0}', github.event.label.name) || 'code')) }}-deploy-storybooks" cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }} diff --git a/.github/workflows/ci.yml.genie.ts b/.github/workflows/ci.yml.genie.ts index db77f95593..ff0586408c 100644 --- a/.github/workflows/ci.yml.genie.ts +++ b/.github/workflows/ci.yml.genie.ts @@ -5,12 +5,13 @@ import { cachixCliBuildStep, cachixStep, checkoutStep, - cleanupEffectUtilsCompositionStep, + ciOtelSpansArtifactStep, + ciOtelSpansSummaryStep, prepareCiScriptsStep, - prepareEffectUtilsCompositionStep, notifyAlignmentJob, pnpmBuilderContractStep, preparePinnedDevenvStep, + prepareCiOtelSpoolStep, installNixStep, runDevenvTasksBefore, ciWorkflow, @@ -58,7 +59,6 @@ const baseSteps = [ checkoutStep(), installNixStep(), ciMeasurementBaselineCheckoutStep, - prepareEffectUtilsCompositionStep, cachixCliBuildStep, trustedCachixStep, prepareCiScriptsStep, @@ -531,14 +531,12 @@ const jobs: Record | ReturnType = { installNixStep(), cachixCliBuildStep, cachixStep({ name: 'overeng-effect-utils' }), - prepareEffectUtilsCompositionStep, prepareCiScriptsStep, preparePinnedDevenvStep, validateNixStoreStep, @@ -692,8 +689,8 @@ const extraJobs: Record = { [ 'set -euo pipefail', '"${DEVENV_BIN:?DEVENV_BIN not set}" shell -- bash -euo pipefail -c \'', - ' cd "${EFFECT_UTILS_WORKSPACE_ROOT:?EFFECT_UTILS_WORKSPACE_ROOT not set}"', - ' buck="$PWD/.megarepo/bin/buck2"', + ' cd "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE not set}"', + ' buck="${BUCK2_BIN:?BUCK2_BIN not set}"', ' "$buck" query effect_utils//packages/@overeng/ci-tools:ci-tools-candidate', ' "$buck" query effect_utils//:editor_view_inputs', ' "$buck" build \\', @@ -743,10 +740,6 @@ const extraJobs: Record = { ...installNixStep(), if: "steps.publication-scope.outputs.publish == 'true'", }, - { - ...prepareEffectUtilsCompositionStep, - if: "steps.publication-scope.outputs.publish == 'true'", - }, { ...cachixCliBuildStep, if: "steps.publication-scope.outputs.publish == 'true'", @@ -800,8 +793,8 @@ const extraJobs: Record = { ], }, /** - * Trusted-only proof that a freshly materialized Buck context can consume an - * action uploaded by an independent local context through the tailnet cache. + * Trusted-only proof that a second plain checkout can consume an action + * uploaded by an independent standalone root through the tailnet cache. */ 'trusted-buck2-remote-cache-proof': { if: trustedSecretCiIf, @@ -813,13 +806,11 @@ const extraJobs: Record = { defaults: bashShellDefaults, permissions: { contents: 'read' }, env: { - // Composition only suppresses remote-cache projection for the exact value `1`. BUCK2_NO_REMOTE_CACHE: '0', }, steps: [ checkoutStep(), installNixStep(), - prepareEffectUtilsCompositionStep, prepareCiScriptsStep, preparePinnedDevenvStep, validateNixStoreStep, @@ -839,13 +830,14 @@ const extraJobs: Record = { ' echo "::error::BUCK2_REMOTE_CACHE_BASIC_AUTH is required for the trusted remote-cache proof"', ' exit 1', 'fi', - 'cd "${EFFECT_UTILS_WORKSPACE_ROOT:?EFFECT_UTILS_WORKSPACE_ROOT not set}"', - 'buck="$PWD/.megarepo/bin/buck2"', + 'source_root="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE not set}"', + 'cd "$source_root"', + 'buck="${BUCK2_BIN:?BUCK2_BIN not set}"', 'context_b="${RUNNER_TEMP:?RUNNER_TEMP not set}/buck2-remote-cache-proof-context-b"', 'target="effect_utils//packages/@overeng/ci-tools:ci-tools-candidate"', 'test_target="effect_utils//packages/@overeng/content-address:test"', - 'proof_source="${EFFECT_UTILS_MEMBER_ROOT:?EFFECT_UTILS_MEMBER_ROOT not set}/packages/@overeng/ci-tools/bin/ci-tools.ts"', - 'test_proof_source="${EFFECT_UTILS_MEMBER_ROOT:?EFFECT_UTILS_MEMBER_ROOT not set}/packages/@overeng/content-address/src/mod.unit.test.ts"', + 'proof_source="$source_root/packages/@overeng/ci-tools/bin/ci-tools.ts"', + 'test_proof_source="$source_root/packages/@overeng/content-address/src/mod.unit.test.ts"', `printf '%s\\n' '' "// trusted remote-cache proof \${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-\${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> "$proof_source"`, `printf '%s\\n' '' "// trusted test-cache proof \${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-\${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> "$test_proof_source"`, 'evidence_a="${RUNNER_TEMP:?RUNNER_TEMP not set}/buck2-remote-cache-proof-a.jsonl"', @@ -856,7 +848,6 @@ const extraJobs: Record = { `trap 'rm -f "$evidence_a" "$test_evidence_a" "$evidence_b" "$test_evidence_b" "$test_evidence_c"; rm -rf "$context_b"' EXIT`, '', '# Context A has run-unique source inputs, executes locally, and uploads to the remote cache.', - '# The composition wrapper fixes --isolation-dir, so freshness comes from daemon and state removal.', '"$buck" kill', 'rm -rf buck-out', '"$buck" build --local-only "$target"', @@ -872,14 +863,11 @@ const extraJobs: Record = { ' exit 1', 'fi', '', - '# Context B is a second composed root with a fresh daemon and materializer over identical inputs.', + '# Context B is a second standalone root with a fresh daemon and materializer over identical inputs.', '"$buck" kill', 'rm -rf buck-out "$context_b"', - 'mkdir -p "$context_b/.buck2" "$context_b/.megarepo" "$context_b/repos/effect-utils"', - 'cp -a .buckconfig .buckroot BUCK megarepo.kdl "$context_b/"', - 'cp -a .buck2/capabilities "$context_b/.buck2/"', - 'cp -a .megarepo/bin "$context_b/.megarepo/"', - 'tar -C repos/effect-utils \\', + 'mkdir -p "$context_b"', + 'tar -C "$source_root" \\', ` --exclude='./.devenv' \\`, ` --exclude='./.git' \\`, ` --exclude='./buck-out' \\`, @@ -891,9 +879,8 @@ const extraJobs: Record = { ` --exclude='*/dist' \\`, ` --exclude='*/node_modules' \\`, ` --exclude='*/target' \\`, - ' -cf - . | tar -C "$context_b/repos/effect-utils" -xf -', + ' -cf - . | tar -C "$context_b" -xf -', 'cd "$context_b"', - 'buck="$PWD/.megarepo/bin/buck2"', '', '# Buck event data must classify the independent build as a remote action-cache hit.', '"$buck" build --local-only "$target"', @@ -920,7 +907,7 @@ const extraJobs: Record = { 'fi', '', '# A source file outside the representative target graph must not change its test action key.', - `printf '%s\\n' '' "// trusted irrelevant-mutation proof \${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-\${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> repos/effect-utils/README.md`, + `printf '%s\\n' '' "// trusted irrelevant-mutation proof \${GITHUB_RUN_ID:?GITHUB_RUN_ID not set}-\${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT not set}" >> README.md`, '"$buck" test --target-platforms effect_utils//buck2/platforms:host_platform --local-only "$test_target"', '"$buck" log show --recent 1 > "$test_evidence_c"', `if ! jq -e 'select(.Event.data.Instant.data.TestResult.name == "effect_utils//packages/@overeng/content-address:test" and .Event.data.Instant.data.TestResult.status == 1)' "$test_evidence_c" >/dev/null; then`, @@ -1379,22 +1366,28 @@ const deployJobs: Record = { }, } as const -const withEffectUtilsCompositionCleanup = (jobMap: Record) => +const withCiOtelCapture = (jobMap: Record) => Object.fromEntries( Object.entries(jobMap).map(([name, ciJob]) => { const steps = ciJob.steps as readonly any[] | undefined return [ name, - steps?.some((step) => step.name === prepareEffectUtilsCompositionStep.name) === true - ? { + steps === undefined + ? ciJob + : { ...ciJob, - steps: [...steps, cleanupEffectUtilsCompositionStep], - } - : ciJob, + steps: [ + prepareCiOtelSpoolStep, + ...steps, + ciOtelSpansSummaryStep, + ciOtelSpansArtifactStep, + ], + }, ] }), ) + // oxlint-disable-next-line overeng/exports-first -- generated entrypoint is assembled after its job atoms export default ciWorkflow({ trustTier: 'public', @@ -1427,7 +1420,7 @@ export default ciWorkflow({ }, }, permissions: { contents: 'read' }, - jobs: withEffectUtilsCompositionCleanup({ + jobs: { // Keep default-ref/source-policy separate from product checks: downstream // validation branches should fail one authority job, not obscure // lint/typecheck/test signal. @@ -1446,7 +1439,7 @@ export default ciWorkflow({ defaultRefs: { 'livestorejs/livestore': 'dev' }, }), }, - ...jobs, + ...withCiOtelCapture(jobs), ...extraJobs, ...deployJobs, 'notify-alignment': { @@ -1459,5 +1452,5 @@ export default ciWorkflow({ ], }), }, - }), + }, } satisfies CiWorkflowArgs) diff --git a/.gitignore b/.gitignore index 139cd700f8..5f37d0712e 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ __pycache__/ # Buck2 daemon and materialized build state buck-out/ +# Standalone trust-tier cache posture, reconciled before every Buck invocation. +/.buckconfig.local # Generated editor snapshots. Unanchored so every admitted package root — present # and future — is covered: these stores are machine-local, digest-addressed, and # hardened read-only at publication, which git cannot represent (it records only @@ -43,6 +45,7 @@ buck-out/ /.buck2/capabilities.candidate.* /.buck2/capabilities.lock /.buck2/capability-generations/ +/.buck2/capability-roots/ # Direnv repos/ diff --git a/AGENTS.md b/AGENTS.md index 275d608b87..d6f3b378d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ Use the `--no-tui` flag to see all output. If tools aren't directly in `$PATH`, We're using megarepo for repo management. We're using `pnpm` temporarily for installs (bun is still used to run scripts) and `devenv` to manage the development environment. -Buck owns checking for all 39 TypeScript projects and declaration production for every emitting project. `buck2:typescript:materialize-dist` publishes those declarations atomically for source-side consumers such as type-aware lint. Publication requires the repository's reciprocal composed megarepo worktree and fails in a detached or lookalike checkout; work on this repo through its megarepo composition. +Buck owns checking for all 39 TypeScript projects and declaration production for every emitting project. The checkout itself is the standalone Buck root; CI and devenv Buck tasks invoke the pinned `BUCK2_BIN` from that root. `buck2:typescript:materialize-dist` publishes those declarations atomically for source-side consumers such as type-aware lint. # Genie (Config File Generation) diff --git a/BUCK b/BUCK index 0ef3c9547f..af4f4260c2 100644 --- a/BUCK +++ b/BUCK @@ -1,3 +1,13 @@ +# Generated file - DO NOT EDIT +# Source: BUCK.genie.ts + +# Projection source: BUCK.genie.ts +# Projection schema version: 1 +# Projection generator: effect-utils/genie/buck2-root-aggregate-projection +# Semantic fingerprint: sha256:8862262d9f5166e14c5b9ebd8a066e40baade59d8c5659d2f64b250418b77bff +# Semantic inputs: BUCK.genie.ts, genie/buck2/mod.ts, genie/buck2/root-aggregate-projection.ts, genie/buck2/typescript-admissions.ts, context/effect/socket/BUCK.genie.ts, context/opentui/BUCK.genie.ts, packages/@overeng/agent-session-ingest/BUCK.genie.ts, packages/@overeng/buck2-tools/BUCK.genie.ts, packages/@overeng/ci-tools/BUCK.genie.ts, packages/@overeng/content-address/BUCK.genie.ts, packages/@overeng/effect-ai-claude-cli/BUCK.genie.ts, packages/@overeng/effect-distributed-lock/BUCK.genie.ts, packages/@overeng/effect-path/BUCK.genie.ts, packages/@overeng/effect-react/BUCK.genie.ts, packages/@overeng/effect-rpc-tanstack/BUCK.genie.ts, packages/@overeng/effect-rpc-tanstack/examples/basic/BUCK.genie.ts, packages/@overeng/effect-schema-form-aria/BUCK.genie.ts, packages/@overeng/effect-schema-form/BUCK.genie.ts, packages/@overeng/genie/BUCK.genie.ts, packages/@overeng/gh-ci-utils/BUCK.genie.ts, packages/@overeng/kdl-effect/BUCK.genie.ts, packages/@overeng/kdl/BUCK.genie.ts, packages/@overeng/megarepo/BUCK.genie.ts, packages/@overeng/notion-cli/BUCK.genie.ts, packages/@overeng/notion-core/BUCK.genie.ts, packages/@overeng/notion-datasource-sync/BUCK.genie.ts, packages/@overeng/notion-effect-client/BUCK.genie.ts, packages/@overeng/notion-effect-schema/BUCK.genie.ts, packages/@overeng/notion-md/BUCK.genie.ts, packages/@overeng/notion-property-write/BUCK.genie.ts, packages/@overeng/notion-react/BUCK.genie.ts, packages/@overeng/npm-release/BUCK.genie.ts, packages/@overeng/otel-contract/BUCK.genie.ts, packages/@overeng/oxc-config/BUCK.genie.ts, packages/@overeng/pty-effect/BUCK.genie.ts, packages/@overeng/react-inspector/BUCK.genie.ts, packages/@overeng/restate-effect/BUCK.genie.ts, packages/@overeng/stylex-tokens/BUCK.genie.ts, packages/@overeng/tui-core/BUCK.genie.ts, packages/@overeng/tui-react/BUCK.genie.ts, packages/@overeng/tui-stories/BUCK.genie.ts, packages/@overeng/utils-dev/BUCK.genie.ts, packages/@overeng/utils/BUCK.genie.ts +# Regenerate: devenv tasks run genie:run + load("//buck2:editor_view.bzl", "editor_view_inputs") load("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain") load("//buck2:static_checks.bzl", "STATIC_SOURCE_EXCLUDES", "STATIC_SOURCE_GLOBS", "static_source_set") @@ -108,3 +118,132 @@ export_file( src = "patches/@myobie__pty@0.10.0.patch", visibility = ["PUBLIC"], ) + +# Scoped repository checks derived from the TypeScript admission registry. +filegroup( + name = "quick", + srcs = { + "000": "//context/effect/socket:typecheck", + "001": "//context/opentui:typecheck", + "002": "//packages/@overeng/agent-session-ingest:typecheck", + "003": "//packages/@overeng/buck2-tools:typecheck", + "004": "//packages/@overeng/ci-tools:typecheck", + "005": "//packages/@overeng/content-address:typecheck", + "006": "//packages/@overeng/effect-ai-claude-cli:typecheck", + "007": "//packages/@overeng/effect-distributed-lock:typecheck", + "008": "//packages/@overeng/effect-path:typecheck", + "009": "//packages/@overeng/effect-react:typecheck", + "010": "//packages/@overeng/effect-rpc-tanstack:typecheck", + "011": "//packages/@overeng/effect-rpc-tanstack/examples/basic:typecheck", + "012": "//packages/@overeng/effect-schema-form:typecheck", + "013": "//packages/@overeng/effect-schema-form-aria:typecheck", + "014": "//packages/@overeng/genie:typecheck", + "015": "//packages/@overeng/gh-ci-utils:typecheck", + "016": "//packages/@overeng/kdl:typecheck", + "017": "//packages/@overeng/kdl-effect:typecheck", + "018": "//packages/@overeng/megarepo:typecheck", + "019": "//packages/@overeng/notion-cli:typecheck", + "020": "//packages/@overeng/notion-core:typecheck", + "021": "//packages/@overeng/notion-datasource-sync:typecheck", + "022": "//packages/@overeng/notion-effect-client:typecheck", + "023": "//packages/@overeng/notion-effect-schema:typecheck", + "024": "//packages/@overeng/notion-md:typecheck", + "025": "//packages/@overeng/notion-property-write:typecheck", + "026": "//packages/@overeng/notion-react:typecheck", + "027": "//packages/@overeng/npm-release:typecheck", + "028": "//packages/@overeng/otel-contract:typecheck", + "029": "//packages/@overeng/oxc-config:typecheck", + "030": "//packages/@overeng/pty-effect:typecheck", + "031": "//packages/@overeng/react-inspector:typecheck", + "032": "//packages/@overeng/react-inspector:strict_consumer_typecheck", + "033": "//packages/@overeng/restate-effect:typecheck", + "034": "//packages/@overeng/stylex-tokens:typecheck", + "035": "//packages/@overeng/tui-core:typecheck", + "036": "//packages/@overeng/tui-react:typecheck", + "037": "//packages/@overeng/tui-stories:typecheck", + "038": "//packages/@overeng/utils:typecheck", + "039": "//packages/@overeng/utils-dev:typecheck", + }, + visibility = ["PUBLIC"], +) + +filegroup( + name = "all", + srcs = { + "000": ":quick", + "001": "//packages/@overeng/agent-session-ingest:dist", + "002": "//packages/@overeng/buck2-tools:dist", + "003": "//packages/@overeng/ci-tools:dist", + "004": "//packages/@overeng/content-address:dist", + "005": "//packages/@overeng/effect-ai-claude-cli:dist", + "006": "//packages/@overeng/effect-distributed-lock:dist", + "007": "//packages/@overeng/effect-path:dist", + "008": "//packages/@overeng/effect-react:dist", + "009": "//packages/@overeng/effect-rpc-tanstack:dist", + "010": "//packages/@overeng/effect-schema-form:dist", + "011": "//packages/@overeng/effect-schema-form-aria:dist", + "012": "//packages/@overeng/genie:dist", + "013": "//packages/@overeng/gh-ci-utils:dist", + "014": "//packages/@overeng/kdl:dist", + "015": "//packages/@overeng/kdl-effect:dist", + "016": "//packages/@overeng/megarepo:dist", + "017": "//packages/@overeng/notion-cli:dist", + "018": "//packages/@overeng/notion-core:dist", + "019": "//packages/@overeng/notion-datasource-sync:dist", + "020": "//packages/@overeng/notion-effect-client:dist", + "021": "//packages/@overeng/notion-effect-schema:dist", + "022": "//packages/@overeng/notion-md:dist", + "023": "//packages/@overeng/notion-property-write:dist", + "024": "//packages/@overeng/notion-react:dist", + "025": "//packages/@overeng/npm-release:dist", + "026": "//packages/@overeng/otel-contract:dist", + "027": "//packages/@overeng/oxc-config:dist", + "028": "//packages/@overeng/pty-effect:dist", + "029": "//packages/@overeng/react-inspector:dist", + "030": "//packages/@overeng/restate-effect:dist", + "031": "//packages/@overeng/stylex-tokens:dist", + "032": "//packages/@overeng/tui-core:dist", + "033": "//packages/@overeng/tui-react:dist", + "034": "//packages/@overeng/tui-stories:dist", + "035": "//packages/@overeng/utils:dist", + "036": "//packages/@overeng/utils-dev:dist", + "037": "effect_utils//packages/@overeng/agent-session-ingest:test", + "038": "effect_utils//packages/@overeng/ci-tools:test", + "039": "effect_utils//packages/@overeng/content-address:test", + "040": "effect_utils//packages/@overeng/effect-ai-claude-cli:test", + "041": "effect_utils//packages/@overeng/effect-distributed-lock:test", + "042": "effect_utils//packages/@overeng/effect-path:test", + "043": "effect_utils//packages/@overeng/effect-react:test", + "044": "effect_utils//packages/@overeng/effect-rpc-tanstack:test", + "045": "effect_utils//packages/@overeng/effect-schema-form-aria:test", + "046": "effect_utils//packages/@overeng/effect-schema-form:test", + "047": "effect_utils//packages/@overeng/genie:test", + "048": "effect_utils//packages/@overeng/gh-ci-utils:test", + "049": "effect_utils//packages/@overeng/kdl-effect:test", + "050": "effect_utils//packages/@overeng/kdl:test", + "051": "effect_utils//packages/@overeng/megarepo:test", + "052": "effect_utils//packages/@overeng/notion-cli:test", + "053": "effect_utils//packages/@overeng/notion-core:test", + "054": "effect_utils//packages/@overeng/notion-datasource-sync:test", + "055": "effect_utils//packages/@overeng/notion-effect-client:test", + "056": "effect_utils//packages/@overeng/notion-effect-schema:test", + "057": "effect_utils//packages/@overeng/notion-md:test", + "058": "effect_utils//packages/@overeng/notion-property-write:test", + "059": "effect_utils//packages/@overeng/notion-react:test", + "060": "effect_utils//packages/@overeng/npm-release:test", + "061": "effect_utils//packages/@overeng/otel-contract:test", + "062": "effect_utils//packages/@overeng/oxc-config:test", + "063": "effect_utils//packages/@overeng/pty-effect:bundle_smoke", + "064": "effect_utils//packages/@overeng/pty-effect:test", + "065": "effect_utils//packages/@overeng/react-inspector:test", + "066": "effect_utils//packages/@overeng/restate-effect:test", + "067": "effect_utils//packages/@overeng/tui-core:test", + "068": "effect_utils//packages/@overeng/tui-react:test", + "069": "effect_utils//packages/@overeng/tui-stories:test", + "070": "effect_utils//packages/@overeng/utils-dev:test", + "071": "effect_utils//packages/@overeng/utils:test", + "072": "//buck2/toolchains:archive_tool", + "073": "//buck2/toolchains:product_tool", + }, + visibility = ["PUBLIC"], +) diff --git a/BUCK.genie.ts b/BUCK.genie.ts new file mode 100644 index 0000000000..5e9fad6f39 --- /dev/null +++ b/BUCK.genie.ts @@ -0,0 +1,3 @@ +import { rootBuckAggregateProjection } from './genie/buck2/root-aggregate-projection.ts' + +export default rootBuckAggregateProjection() diff --git a/CHANGELOG.md b/CHANGELOG.md index 32d19e986a..3f813fb7af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ All notable changes to this project will be documented in this file. ### Added +- **Genie CI**: Capture every devenv task graph as OpenTelemetry spans with + workflow, job, run, revision, runner, and task attributes; render the slowest + tasks in each job summary; and retain the OTLP payload as a CI artifact. - **@overeng/gh-ci-utils**: `run` accepts workflow-dispatch inputs via repeated `--field key=value` (matching `gh workflow run -f` ergonomics) and a `--inputs` JSON object for scripts. `--field` wins on key collision; @@ -32,6 +35,22 @@ All notable changes to this project will be documented in this file. - **Buck2 rules distribution**: Export deterministic rules and executor-capability projections plus a standalone consumer-root helper, so downstream repositories can build source products without mounting effect-utils as a Buck cell. +- **Buck2 checks**: Add standalone project roots, Nix-owned capability + projections shared with composed roots, registry-derived `//:quick` and + `//:all` aggregates, check-verb wiring, and a guard against untracked + Buck-versus-legacy producer overlap. +- **CI**: Run every effect-utils lane from the checked-out standalone Buck root, + removing the synthesized composition lifecycle and its member-root plumbing. +- **Buck2 editor views**: Add explicit coalesced publishers for Restate, + OpenTelemetry contract, and the shared Playwright package union so those CI + lanes publish only their required views while retaining whole-workspace + dependency authority. +- **Buck2 editor bootstrap**: Bound the pre-freshness bootstrap to the declared + Genie generator import closure (the repository root and OpenTelemetry + contract views), retain whole-workspace ownership authority, fail authoring + checks when that declaration misses a first-party runtime import, emit + per-phase timings, and avoid duplicate admitted-tree and post-publication + snapshot traversals. - **Devenv tasks**: Add a reusable 50,000-file recursive eval-cache input budget that names the offending cached attribute and gates quick and full diff --git a/buck2/toolchains/BUCK b/buck2/toolchains/BUCK index be318d3945..4ac7cf2124 100644 --- a/buck2/toolchains/BUCK +++ b/buck2/toolchains/BUCK @@ -1,4 +1,4 @@ -load("//.buck2/capabilities:defs.bzl", "CAPABILITIES", "GENERATION") +load("@capabilities//:defs.bzl", "CAPABILITIES", "GENERATION") load("//buck2/platforms:defs.bzl", "host_platform_label") load("//buck2/toolchains:defs.bzl", "bun_toolchain", "configured_rust_toolchain", "effect_tsgo_toolchain", "host_rust_target_triple", "nix_go_bootstrap_toolchain", "nix_python_bootstrap_toolchain") load("//buck2/rust:toolchains.bzl", "native_rust_toolchains") diff --git a/buck2/toolchains/configured.bzl b/buck2/toolchains/configured.bzl index 64cbdc0492..162d372107 100644 --- a/buck2/toolchains/configured.bzl +++ b/buck2/toolchains/configured.bzl @@ -2,7 +2,7 @@ load("//buck2/platforms:defs.bzl", "host_execution_constraints") load("//buck2/toolchains:defs.bzl", "host_capability_platform") -load("//.buck2/capabilities:defs.bzl", "CAPABILITIES") +load("@capabilities//:defs.bzl", "CAPABILITIES") BuckSupportToolInfo = provider(fields = { "content_digest": str, @@ -56,7 +56,7 @@ _support_tool = rule( def support_tool(name, protocol, tool_id, **kwargs): platform = host_capability_platform() metadata = CAPABILITIES[platform][tool_id] - capability = "//.buck2/capabilities/generations/{}/{}/{}".format(metadata["generation"], platform, tool_id) + capability = "capabilities//generations/{}/{}/{}".format(metadata["generation"], platform, tool_id) _support_tool( name = name, content_digest = metadata["contentDigest"], diff --git a/context/buck2/.decisions/0020-one-writable-mount-workspaces.md b/context/buck2/.decisions/0020-one-writable-mount-workspaces.md index 3651e9c714..d4e46c289a 100644 --- a/context/buck2/.decisions/0020-one-writable-mount-workspaces.md +++ b/context/buck2/.decisions/0020-one-writable-mount-workspaces.md @@ -147,3 +147,12 @@ receipt or explicit runtime admission; fixture proof does not discharge tierA or ## Amendment 3 (2026-09-15) The one-writable-mount workspace is no longer the default development context; it is retained on `main` as the paused composed shape until the last consumer edge leaves it, then deleted with its ledger rows. See [decision 0034](./0034-artifact-default-composition-no-registry.md). + +## Amendment 4 (2026-09-22) + +The one-writable-mount workspace applies only to explicitly requested +cross-repository compositions during the paused retirement window. It does not +govern standalone development or single-repository CI. Effect-utils CI runs +from its tracked standalone checkout and does not synthesize or clean a +composed workspace. The composed shape retains its fail-closed mount, +ownership, and publication guards until its deletion at L3 cut 2. diff --git a/context/buck2/.decisions/0027-composed-default-worktrees.md b/context/buck2/.decisions/0027-composed-default-worktrees.md index 6fb03966f8..2a3d62536c 100644 --- a/context/buck2/.decisions/0027-composed-default-worktrees.md +++ b/context/buck2/.decisions/0027-composed-default-worktrees.md @@ -56,6 +56,11 @@ evidence rather than silently rolled back. - Per-root disk cost multiplies by live worktrees until decision 0025's CoW economics are deliverable on the host filesystem. -## Amendment 1 (2026-09-15) +## Amendment 1 (2026-09-15; clarified 2026-09-22) -Composed-by-default is reverted: standalone worktrees are the default (MR-R11 as amended); composed workspaces are a declared exception. See [decision 0034](./0034-artifact-default-composition-no-registry.md). +Composed-by-default is reverted: tracked standalone worktrees are the default +for ordinary development, agents, and single-repository CI (MR-R11 as +amended). CI runs directly from its checkout and does not synthesize a +composition root. Composed workspaces are an explicitly requested, +cross-repository exception during the paused retirement window. See +[decision 0034](./0034-artifact-default-composition-no-registry.md). diff --git a/context/buck2/.decisions/0028-hermetic-python-bootstrap-for-consumer-cells.md b/context/buck2/.decisions/0028-hermetic-python-bootstrap-for-consumer-cells.md index eb0ba81431..0f896ca23d 100644 --- a/context/buck2/.decisions/0028-hermetic-python-bootstrap-for-consumer-cells.md +++ b/context/buck2/.decisions/0028-hermetic-python-bootstrap-for-consumer-cells.md @@ -157,3 +157,31 @@ schema. by its own `contentDigest` instead of by whole-projection `GENERATION`, and whether the remaining top-level `rust-*`/`archive-tool`/`product` capabilities should also move under named authorities. + +## Amendment 1 — Nix Owns The Capability Projection + +Accepted 2026-09-17 for the standalone Buck root and aggregate cutover. + +The flake is now the sole producer of capability projection bytes. +`packages..buck2-capabilities` reads the tracked `buck2-member.json`, +realizes the manifest's `flakePackage` outputs, obtains each transitive runtime +closure through `pkgs.closureInfo`, and invokes the shared TypeScript projection +renderer in a pure derivation. The derivation does not invoke Nix recursively. + +The resolver remains the verification boundary, not a second producer. With +`MR_CAPABILITY_PROJECTION`, it checks the Nix output's platform, exact root and +tool BUCK bytes, manifests, executable digests and store paths, closure +membership, and generation identity. `mr apply` then atomically links that +verified output at the composition root's `.buck2/capabilities`. A standalone +devenv shell links the same output at the standalone root. Both roots declare +that directory as the `capabilities//` cell, and hub toolchains use only +cross-cell labels into it. + +This amendment supersedes the “mr's resolver is the sole producer” row and +paragraph above. The invariant remains one producer, one tool set, and one +generation digest; ownership moves from mr's imperative resolver path to the +flake's pure package output. The resolver and atomic installer remain +load-bearing consumers and verifiers. The legacy resolver projection path stays +available only through the absence of `MR_CAPABILITY_PROJECTION` until +composition adoption reaches L3; it is not used by the standalone or composed +effect-utils paths. diff --git a/context/buck2/.experiments/2026-09-18-standalone-buck-root.md b/context/buck2/.experiments/2026-09-18-standalone-buck-root.md new file mode 100644 index 0000000000..f885854963 --- /dev/null +++ b/context/buck2/.experiments/2026-09-18-standalone-buck-root.md @@ -0,0 +1,114 @@ +# Standalone Buck root + +Date: 2026-09-18 +Host: dev3 (x86_64-linux) + +## Question + +Does a plain effect-utils worktree build the admitted Buck graph without megarepo composition, and do the warm no-op, warm-cache/fresh-output, `check:quick`, and shell-entry measurements remain within the accepted budgets? + +## Method + +Run every Buck, Nix, and devenv command through `/srv/bulk/coding-agents/_briefs/buck2-heavy.sh`. Before each accepted sample, record `MemAvailable`, `user-1000.slice/memory.current`, and memory PSI `some avg60`. Measure at least three samples for each regime: + +1. `buck2 build //:quick` after an unchanged successful build. +2. `buck2 build //:quick` with a fresh Buck output directory and a warm remote cache. +3. `check:quick` before and after the standalone-root change. +4. `devenv shell -- true` after one untimed warm-up. + +Use a fresh `git worktree add` checkout with no megarepo state for the standalone samples. Use an `mr store worktree new` root only for the composed compatibility proof. Verify the deliberately broken admitted-package control and the unchanged rerun's local action count separately from the timing samples. + +## Result + +Revision `e2073b0882` passed the original standalone Buck proofs. Revision +`5b249696f4` then changed only the standalone root to Buck's `notify` watcher. +The branch was rebased onto current PR #1283 and reconciled at revision +`f8528ed38e`, preserving the current parent composition behavior. + +- A fresh detached Git worktree with no megarepo state built + `//packages/@overeng/tui-core:typecheck`. +- `mr store worktree new --commit e2073b0882` produced the compatibility-control + worktree, and the same target built there. +- The full `//:quick` aggregate completed from the plain worktree. +- After adding a deliberate `number`-to-`string` error to the admitted + `tui-core` source, `//:quick` failed at + `//packages/@overeng/tui-core:typecheck` with TS2322. Restoring the source + returned the worktree to a clean state. +- With Watchman, three unchanged reruns completed in 16-18 ms with no network + traffic and no scheduled command summary, which is Buck's zero-command + result. +- With `notify`, three unchanged reruns completed in 16-50 ms with the same + zero-command and zero-network result. + +All accepted samples ran through the shared heavy-command gate. The readings +below were captured immediately before each timed sample. + +| Watcher and regime | Elapsed samples | Resource readings before samples | Result | +| --- | --- | --- | --- | +| Watchman, unchanged `//:quick` | 17 ms, 18 ms, 16 ms; median 17 ms | `MemAvailable` 25,741,880 / 25,705,348 / 25,677,884 KiB; slice memory 84,167,573,504 / 84,169,175,040 / 84,181,024,768 bytes; PSI `some avg60` 0.78 / 0.78 / 0.78 | PASS: all samples are below the 5 s BUCK-R07 warm no-op budget | +| `notify`, unchanged `//:quick` | 17 ms, 50 ms, 16 ms; median 17 ms | `MemAvailable` 22,256,252 / 22,243,932 / 22,220,676 KiB; slice memory 82,014,629,888 / 82,013,585,408 / 82,014,101,504 bytes; PSI `some avg60` 0.72 / 0.72 / 0.72 | PASS: all samples are below the 5 s budget; no warm no-op regression | +| Watchman, fresh `buck-out`, warm shared cache | 18.077 s, 8.443 s, 6.412 s; median 8.443 s | `MemAvailable` 25,086,560 / 27,771,072 / 27,709,148 KiB; slice memory 81,304,829,952 / 82,396,282,880 / 82,268,454,912 bytes; PSI `some avg60` 1.38 / 1.08 / 0.95 | PASS: all samples are below the 3 min BUCK-R07 budget; each reported 1,208 cached commands, 100% cache hits, and zero local commands | +| `notify`, fresh `buck-out` | 18.726 s | `MemAvailable` 27,053,620 KiB; slice memory 79,179,030,528 bytes; PSI `some avg60` 1.37 | PASS: below 3 min; 1,208 commands, 668 cache hits (55%), and 540 local commands | +| sandboxed second context, fresh `HOME` / `TMPDIR` / hostname / uid / `buck-out` | 15 min 36.3 s | Heavy-command gate admitted the run; Buck reported 648 MiB peak process memory | FAIL BUCK-R16: 556 cached actions, 633 local actions, 561 other actions, and zero remote actions. Local classes were `package_tree`, `pnpm_store_entry`, `pnpm_store_scc`, `pnpm_store_view`, `tsgo_emit`, and `tsgo_typecheck`. The build later reached the same pre-existing `preferSchemaOverJson` warning and exited non-zero | +| dev4 aarch64 (informational), fresh `buck-out` | 64.2 s | dev4: 31 GiB total memory; 666 MiB peak process memory | INFORMATIONAL: 1,185 cached actions, 8 local actions, 561 other actions, and zero remote actions; local action classes included `package_tree`, `tsgo_typecheck`, and `tsgo_emit`. The build reached the pre-existing `preferSchemaOverJson` warning in `composition-root-publisher.integration.test.ts` and exited non-zero because tsgo treats the warning as exit 2 | +| Warm `devenv shell -- true` | 505 ms, 424 ms, 446 ms; median 446 ms | `MemAvailable` 25,855,352 / 25,799,616 / 25,786,556 KiB; slice memory 81,212,649,472 / 81,204,805,632 / 81,209,909,248 bytes; PSI `some avg60` 1.18 / 1.18 / 1.14 | PASS: all samples are below the 20 s shell-entry budget | + +The checked-in standalone root now uses `file_watcher = notify`. The composition +root generator continues to own and emit `file_watcher = watchman`. A regression +guard checks both choices. `notify` retains the 17 ms warm median, stays far +inside the 5 s budget, and avoids the shared Watchman daemon that previously +timed out during root synchronization. The single requested fresh `notify` +sample was 0.649 s slower than the slowest accepted Watchman fresh sample and +had a lower remote-cache hit rate, but it remained far inside the 3 min budget. + +The `check:quick` before/after control is the named **S14 mr-row deletion +dependency** under decision 0034. `check:quick` still runs `mr:*` gates that +cannot pass in a standalone worktree by construction. Amendment 3 defers their +deletion and the composed control to S14. The standalone `//:quick` fresh-output +and unchanged-rerun proofs above satisfy S8's aggregate acceptance; S8 makes no +`check:quick` wall-clock claim. + +Decision q40 / BUCK-R16 adds a same-platform second-context proof and an +informational cross-architecture observation. The sandbox used a fresh +`HOME`, `TMPDIR`, hostname (`other-host`), uid/gid (4242), and `buck-out` while +sharing only the checked-out revision, `/nix/store`, the Nix database, system +certificates, and the network. It produced 633 local actions, so the required +zero-local-action acceptance does not hold. `buck2 log what-ran` identified +`package_tree`, `pnpm_store_entry`, `pnpm_store_scc`, `pnpm_store_view`, +`tsgo_emit`, and `tsgo_typecheck` as the local action classes. Per q40, the +sandbox was not tuned further to hide this BUCK-R06 key-stability regression. + +The dev4 row is the cross-architecture evidence cited by the portable-product +question in `context/buck2/04-reuse/open-questions.md` on PR #1309. The +aarch64 keys were also not fully reusable: eight actions ran locally, across +`package_tree`, `tsgo_typecheck`, and `tsgo_emit`. This is informational and +does not change S8 acceptance. + +Strict VRS validation remains blocked by the pre-existing decision-shape errors +in decisions 0035 and 0036 (`Status:`, `Context`, `Evidence and Argument`, and +`Options` are absent according to the current strict schema). + +## Conclusion + +The standalone Buck root and aggregate meet the original BUCK-R07 warm and +fresh-context budgets. Shell entry also remains within its accepted budget. The +composed store-worktree target control and the broken-package control both +behave as required. `notify` does not regress warm no-op performance and +removes the standalone root's dependency on the flaky shared Watchman daemon. + +The new BUCK-R16 second-context proof does not achieve zero local actions: +633 actions ran locally. Under q40, this BUCK-R06 key-stability regression is +the finding and deliverable; no sandbox tuning was used to conceal it. +The `check:quick` composed control is deferred to the named S14 mr-row deletion +dependency by Amendment 3 and does not block S8. + +## VRS Impact + +This experiment closes the original S8 standalone-root, aggregate, watcher, +and shell-entry measurement gaps. Its same-platform sample initially reported +633 local actions. The +[controlled follow-up](./2026-09-19-second-context-key-stability.md) resolves +that result as a test-ordering artifact: after warming the sampled revision, +the sandbox reused every successful action. S14 owns deletion of the remaining +composition-dependent `check:quick` residual gates and its composed +before/after control. diff --git a/context/buck2/.experiments/2026-09-19-second-context-key-stability.md b/context/buck2/.experiments/2026-09-19-second-context-key-stability.md new file mode 100644 index 0000000000..040b27790a --- /dev/null +++ b/context/buck2/.experiments/2026-09-19-second-context-key-stability.md @@ -0,0 +1,98 @@ +# Second-context key stability + +Date: 2026-09-20 +Host: dev3 (x86_64-linux) + +## Question + +Did the S8 second-context sample expose context-dependent action keys, or did it +run before the shared cache was warm at the sampled revision? + +## Method + +At pinned revision `948d397a2e`, run exactly two `//:quick` builds through the +shared heavy-command gate, in this order: + +1. Run the normal worktree context to populate the shared action cache. +2. Remove the proof worktree's `buck-out` and run the same revision with + Bubblewrap using `--unshare-all --share-net`, hostname `other-host`, uid/gid + 4242, a fresh `HOME` and `TMPDIR`, read-only Nix store/database and `/etc`, + and the proof worktree mounted at `/work`. + +For each build, capture `buck2 log what-ran --format json +--emit-cache-queries`. Compare the remaining sandbox-local action's action +digest, command, and execution environment against the normal-context action. +The pinned Buck2 does not provide `buck2 audit action-keys`, so the remote action +digest from `what-ran` and `log show` is the action-key evidence. + +## Result + +Both builds reached the pre-existing `preferSchemaOverJson` warning in +`composition-root-publisher.integration.test.ts`. Tsgo treats the warning as +exit 2, so both builds ended at the same `megarepo:typecheck` action. + +| Context | Wall time | Cache queries | Cached | Local | Result | +| --- | ---: | ---: | ---: | ---: | --- | +| Normal worktree | 83.1 s | 1,195 | 648 | 547 | Populated the missing cache entries, then stopped at the warning | +| Sandboxed second context | 68.5 s | 1,195 | 1,192 | 1 | Every successful action reused the warmed cache; only the failing typecheck ran locally | + +The original six sandbox-local classes reduced to one action in one class: + +```text +effect_utils//packages/@overeng/megarepo:typecheck (effect_utils//buck2/platforms:linux_x86_64#a312ca1b0cfd7c35) (tsgo_typecheck typecheck) +``` + +The normal and sandbox cache queries used the identical action digest: + +```text +76ecc7d96ba19cabf193f3e0fc6f48509e214e0b4b1560f75c5dca70c2cf5297:142 +``` + +The command arrays were byte-for-byte equal. Their NUL-delimited SHA-256 was: + +```text +4f0a142b51a27f02fb71ddce96ada8870928d8e027d8844570e274180ca75fab +``` + +No declared action input differed: the equal remote action digest covers the +command and declared input root. `log show` exposed only per-execution values +that Buck does not include in that digest: + +```text +normal TMPDIR=/buck-out/v2/tmp/effect_utils/40bdb25f91291859/tsgo_typecheck/typecheck +sandbox TMPDIR=/work/buck-out/v2/tmp/effect_utils/40bdb25f91291859/tsgo_typecheck/typecheck +normal BUCK2_DAEMON_UUID=305fb556-547b-484d-ac8c-6e1ec2b48dd4 +sandbox BUCK2_DAEMON_UUID=3e084141-8c9b-4607-ab1e-ee6d044d1973 +normal BUCK_BUILD_ID=bfb22410-589d-4928-83b2-f4512a96626c +sandbox BUCK_BUILD_ID=5de2112a-3511-4839-85bd-03ceee1c60ec +``` + +`BUCK_SCRATCH_PATH` was identical and worktree-relative in both runs: + +```text +buck-out/v2/tmp/effect_utils/40bdb25f91291859/tsgo_typecheck/typecheck +``` + +## Conclusion + +The 633 local actions in the S8 sandbox sample were a test-ordering artifact. +That sandbox was the first build at its rebased revision. After a normal-context +build warmed the same revision, the sandbox reused every successful action, +including the five previously missed classes other than `tsgo_typecheck`. +There is no context-dependent action-key input in the remaining action: its +digest and command are identical across contexts. + +The one remaining local action does not violate key stability. It fails +identically in both contexts, and failed actions do not produce a reusable +successful cache entry. The separate finding is that the +`packages/@overeng/megarepo` `preferSchemaOverJson` warning makes `//:quick` +red on the sampled branch. The landing seat for PRs #1283 and #1301 owns that +finding; this experiment does not fix it. + +## VRS Impact + +This experiment resolves DELTA-001 as a test-ordering artifact: S8's sandbox +run was the first build at its rebased revision; at one revision with the +normal context warmed first, the sandbox reused 1,192 of 1,195 queried actions, +and the remaining action failed identically in both contexts. DELTA-001 is +removed. BUCK-R06 and REUSE-R02 remain unchanged. diff --git a/context/buck2/03-materialization/.delta/DELTA-002-editor-bootstrap-warm-integrity.md b/context/buck2/03-materialization/.delta/DELTA-002-editor-bootstrap-warm-integrity.md new file mode 100644 index 0000000000..265cceceb0 --- /dev/null +++ b/context/buck2/03-materialization/.delta/DELTA-002-editor-bootstrap-warm-integrity.md @@ -0,0 +1,41 @@ +# DELTA-002: Editor bootstrap warm integrity exceeds BUCK-R07 + +Status: open + +## Divergence + +BUCK-R07 requires a warm no-op at no more than 5 s. The bounded generator +bootstrap now publishes the repository-root view and the OpenTelemetry contract +view required by the Weaver import closure, but local-disk warm samples measured +8.61 s, 7.18 s, and 7.76 s. The fresh sample remains within budget at 62.6 s. + +## VRS + +- [BUCK-R07](../../requirements.md) carries the 5 s warm and 3 min fresh + budgets. +- [REUSE-R03](../../04-reuse/requirements.md) applies those budgets to the + admitted reuse surface. +- The implementation and timing evidence are recorded in the private T4 worker + report `effect-utils.editor-bootstrap-r07`. + +## Implementation + +Each published view retains the editor-view integrity contract: a warm +publication re-hashes every byte-owned immutable snapshot payload before reuse. +The second required view adds another payload proof. Backing and `node_modules` +fingerprints within a snapshot and publications using disjoint editor-root +locks now overlap, but shared-disk contention leaves the two-view warm path +2.18–3.61 s above the budget. + +## Direction + +update implementation + +## Resolution Signal + +Decide whether the editor-view integrity record may persist a recursive metadata +proof alongside the byte digest. If approved, warm reuse validates that metadata +proof and falls back to the byte proof on any mismatch; tamper regressions must +still fail closed. Close this delta when three controlled warm bootstrap samples +are each at or below 5 s while the two-view closure checker, direct Weaver load, +and snapshot-integrity tests remain green. diff --git a/context/buck2/03-materialization/spec.md b/context/buck2/03-materialization/spec.md index 360cfdf8a6..cc46284e41 100644 --- a/context/buck2/03-materialization/spec.md +++ b/context/buck2/03-materialization/spec.md @@ -114,18 +114,21 @@ An existing lock fails immediately and prints the explicit token-gated recovery operation. There is no age heuristic, timeout, or automatic lock theft. Under the lock, the publisher: -1. fingerprints the selected dependency view and finite declared roots; +1. fingerprints each distinct selected dependency view and finite declared root + once for the admitted state; 2. recursively copies the selected view and disjoint backing roots into a same-filesystem candidate with dereferenced, byte-owned regular files; 3. relocates internal links into `.backing/`, rejects links outside the declared roots, and proves no snapshot file shares an inode with a disposable source; 4. verifies the complete payload digest and writes `editor-view.json`; -5. hardens the candidate read-only and renames it to the deterministic snapshot; +5. hardens a new candidate read-only and renames it to the deterministic + snapshot, or verifies an existing immutable snapshot before reuse; 6. atomically renames the current pointer, installs or validates the package first hop, and emits the package-manifest settle signal required by live language servers; -7. checks the published view, updates its retention record, and garbage-collects - snapshots outside the configured finite retention set. +7. updates the retention record and garbage-collects snapshots outside the + configured finite retention set. Pointer helpers validate their exact writes; + a separate `buck2:editor:check` performs the full admitted-state traversal. If a legacy root install occupies the first hop, immutable GNU `mv --exchange --no-copy` installs the symlink without an absent-path window and @@ -135,10 +138,16 @@ leaves the prior current view intact. Snapshot payloads never retain links into ## Staleness Gate -`buck2:editor:bootstrap` first derives a dependency-only consumer set from the -committed generated root manifest. It may publish those committed-graph views -only to make `genie:check` runnable; it reports no governed evidence. After -freshness and workspace reconciliation, `buck2:editor:publish` and +`buck2:editor:bootstrap` regenerates whole-workspace ownership authority but +builds and publishes only the declared source-generator import closure: the +repository-root dependency view backed by Genie's package tree and the +OpenTelemetry contract view needed by Genie's Weaver runtime. The shared Genie +runtime-closure walker checks every generator before `genie:check` and names any +first-party package imported outside that declaration, so a new edge cannot +silently rely on a stale whole-workspace publication. Bootstrap therefore stays +bounded by source-generator dependencies instead of every workspace package. It +exists only to make `genie:check` runnable and reports no governed evidence. +After freshness and workspace reconciliation, `buck2:editor:publish` and `buck2:editor:check` derive the complete root-plus-package set from the canonical source registry, regenerate whole-workspace ownership authority, build every `:editor_view_inputs` manifest in one Buck invocation, then publish or validate diff --git a/context/buck2/05-composition/open-questions.md b/context/buck2/05-composition/open-questions.md index 44e732e9fe..1b76a9b213 100644 --- a/context/buck2/05-composition/open-questions.md +++ b/context/buck2/05-composition/open-questions.md @@ -1,5 +1,20 @@ # Composition Open Questions +## Resolved 2026-09-22: effect-utils CI uses the standalone checkout root + +Decision q58 made the tracked standalone repository root normative for +ordinary development and single-repository CI. Effect-utils CI runs every lane +from the actions checkout; the trusted remote-cache proof compares a second +plain checkout at the same revision. CI no longer prepares or cleans a +composition root. + +COMP-T01 and COMP-R01/R02/R06/R07 now scope canonical `repos/` mounts, +the one-writable-mount contract, and the `megarepo` isolation directory to an +explicitly requested cross-repository composition during the paused retirement +window. Decisions 0020 Amendment 4 and 0027 Amendment 1 record the same +boundary. The public trust-tier deployment gates the live cache proof, not the +root-shape contract. + ## Resolved 2026-09-15: accept artifact-default composition? — decision 0034; composition machinery is on the deletion path (q47, 2026-09-19) Composed cells exist for vision criterion 6 as originally ratified (a consumer @@ -18,27 +33,19 @@ hybrid, and Nix outputs on the same edge, and the no-registry publication proof (PR #1289: `@overeng/utils` published as a release asset, dotfiles notion-scan consuming by URL, typecheck + 28 tests green) met the proposal's gate. -**Resolved 2026-09-15 by -[decision 0034](../.decisions/0034-artifact-default-composition-no-registry.md)** -(q22/q23/q29): artifact-default cross-repository composition with no registry; -composed-by-default reverted; the composed shape stays on `main`, paused, as -the fallback until the last consumer edge leaves it. Remaining follow-ups, -carried as requirements in 0034: a strict second-install no-op per consumer -(pnpm injected-workspace pruning), peer-contract alignment per consumer, and -the L3 retirement ledger rows. - -## Open 2026-09-12: root-owned capability cell (superseded for consumers by 0037 - a standalone root takes capabilities as a Nix output; remains only for the composed development root until L3 cut 2) - -The hub loads the per-host capability projection from inside its own cell -(`buck2/toolchains/BUCK:1`, `configured.bzl:5,59`: -`//.buck2/capabilities/…`), so mr must write the projection into every mount -and no fetched or read-only hub can carry it -([2026-09-12-hub-as-external-cell](./.experiments/2026-09-12-hub-as-external-cell.md)). -Moving it to a root-provided `capabilities//` cell (declared by the root -generator, referenced by cross-cell labels) is the right ownership boundary in -every option on the table and is a precondition for rules-only external-cell -distribution of the hub. Blocked on: deciding the cell's contract (visibility, -generation identity checks) and the mr change that declares it. +## Resolved 2026-09-17: root-owned capability cell (superseded for consumers by 0037 - a standalone root takes capabilities as a Nix output; remains only for the composed development root until L3 cut 2) + +The composition root declares `capabilities = .buck2/capabilities`, and hub +toolchains load `capabilities//:defs.bzl` plus generation-keyed labels from that +cell. Nix is the sole producer: `packages..buck2-capabilities` derives +the projection from the tracked member manifest and the same flake package +outputs that the resolver consumes. The devenv shell links that store output +for a standalone root. `mr apply` verifies the same output and atomically links +it into the composition root. The shared TypeScript renderer defines the +projection bytes and generation identity for both paths. This removes the +per-mount write requirement while retaining strict manifest, platform, +executable, closure, and generation checks. Decision 0028 Amendment 1 records +the ownership change. ## Resolved 2026-08-30: consumers share the hub's toolchain pins diff --git a/context/buck2/05-composition/requirements.md b/context/buck2/05-composition/requirements.md index 7ede4d748d..080ac97db6 100644 --- a/context/buck2/05-composition/requirements.md +++ b/context/buck2/05-composition/requirements.md @@ -17,28 +17,25 @@ BUCK-R05 and BUCK-R14. Architecture: ## Acceptable Tradeoffs -- **COMP-T01 External namespaces:** An external consumer building a public - member standalone uses the same synthesized root shape but inhabits its own - cache namespace; no attempt is made to share keys outside the fleet. +- **COMP-T01 Trust-tier namespaces:** External consumers and single-repository + CI build public members from their tracked standalone repository roots. A + trust tier may select a separate cache namespace; sharing action keys across + trust tiers is not required. ## Requirements -- **COMP-R01 Synthesized root for the composed shape:** While a repository - uses the paused composed shape - ([decision 0034](../.decisions/0034-artifact-default-composition-no-registry.md)), - every build — composed, single-repo - CI, and standalone — runs from a synthesized composition root. A bare - checkout as its own project root is a cache island and is not a supported - build shape. The workspace root is located at the store worktree path and is - not itself a git repository; the owned member is - ([decision 0027](../.decisions/0027-composed-default-worktrees.md)). +- **COMP-R01 Standalone root by default:** The tracked repository checkout is + the normative Buck project root for ordinary development and + single-repository CI. Only an explicitly requested cross-repository composed + build synthesizes a workspace root while the paused composed shape exists + ([decision 0034](../.decisions/0034-artifact-default-composition-no-registry.md)). Git external cells are not a composition mechanism ([decision 0030](../.decisions/0030-external-cells-are-not-a-composition-mechanism.md)). -- **COMP-R02 Canonical mounts within the composed shape:** Within the paused - composed shape, every repository — including - the one under development — has one canonical mount path (`repos/`), - identical in every composition and at every nesting level. No repo builds - from its own root as a cell: one cell identity per repo, one cache namespace +- **COMP-R02 Canonical mounts within the composed shape:** Within an explicitly + requested paused composed build, every repository — including the one under + development — has one canonical mount path (`repos/`), identical in + every composition and at every nesting level. A standalone repository maps + the same canonical cell name to `.` ([decision 0020](../.decisions/0020-one-writable-mount-workspaces.md)). - **COMP-R03 Canonical cell names:** Each member has one canonical cell name, identical everywhere; a member's checked-in `[cell_aliases]` must agree with @@ -52,12 +49,15 @@ BUCK-R05 and BUCK-R14. Architecture: - **COMP-R05 Shared platform labels:** Platform targets live in one canonical hub cell present in every composition; the same labels resolve everywhere (the label, not its content, enters the configuration hash). -- **COMP-R06 No member `.buckroot`:** Members ship no `.buckroot`; the - composition root owns it. A cwd inside a member must not silently become its - own project root with a second `buck-out`. -- **COMP-R07 Fixed isolation dir:** One isolation dir across all shapes; it is - part of output paths and therefore of action identity. Per-invocation - isolation dirs are forbidden. +- **COMP-R06 Project-root markers:** A standalone repository ships `.buckroot` + at its normative project root. A composed generator treats a nested member + root marker as member content without discovering a second Buck project; the + outer composition root remains the project authority. +- **COMP-R07 Fixed isolation per supported shape:** Each supported root shape + has one fixed isolation dir, which is part of output paths and action + identity. Standalone and paused composed shapes may use different isolation + dirs; cross-shape action-key parity is not promised. Per-invocation isolation + dirs are forbidden. - **COMP-R08 Content-reachable mounts and admissible links:** Member bytes must be reachable at the mount path without traversing an absolute symlink, and any relative symlink must normalize to a path inside the project root. diff --git a/context/buck2/05-composition/spec.md b/context/buck2/05-composition/spec.md index b0d9db44a8..f729e29e4b 100644 --- a/context/buck2/05-composition/spec.md +++ b/context/buck2/05-composition/spec.md @@ -1,8 +1,9 @@ # Composition Spec -This document specifies the composition root and its generation. It builds on -[requirements.md](./requirements.md). The shape below is validated against the -real repositories +This document specifies the normative standalone repository root and the +explicitly requested composed root retained during its paused retirement +window. It builds on [requirements.md](./requirements.md). The composed shape +is validated against the real repositories ([.experiments/2026-08-26-composition-root-real-repos.md](./.experiments/2026-08-26-composition-root-real-repos.md)). ## Status @@ -17,24 +18,22 @@ variant. **Does not define:** member semantics (01), platforms (02), or cache wiring (04). -## Composition Root Shape +## Paused Composition Root Shape -The mr-generated root `.buckconfig` (validated on real content): +An explicitly requested cross-repository composition uses the mr-generated +root `.buckconfig` below: ```ini [cells] - workspace = . # the synthesized shell; declares no targets + workspace = . # the synthesized shell prelude = prelude - toolchains = toolchains - none = none + capabilities = .buck2/capabilities # root-owned, Nix-produced projection = repos/ # one line per member incl. the owned repo [cell_aliases] config = prelude ovr_config = prelude - fbcode = none - fbsource = none - fbcode_macros = none - buck = none + fbsource = prelude + toolchains = [external_cells] prelude = bundled [parser] @@ -106,29 +105,32 @@ member-owned executable capability that attempts to override the required toolchain _or any tool id an authority provides_. This is an explicit shared-pin contract, not silent inheritance. -`--isolation-dir` is CLI-only and cannot be pinned by buckconfig, so mr also -owns the invocation wrapper that fixes it (COMP-R07); an unwrapped `buck2` call -relies on the default and is consistent by accident only. - -Member repositories ship no `.buckconfig` project root of their own: deleting -effect-utils' `.buckconfig` is part of landing the generator, so the -unsupported bare-checkout shape fails loudly instead of silently building a -cache island. (A member's `.buckconfig` is inert under composition — only its -`[cell_aliases]` are honored — so nothing else is lost.) The gitignored -`.buck2/capabilities` cell is per-host projected state with exactly one -producer, mr's composition capability resolver: the mount pipeline projects it -per read-only mount, and `mr apply` installs it into the owned member. A member -ships no projector of its own, and a member-shipped script under `scripts/` is -inert data the resolver never reads or executes. Buck analysis of the hub's -`buck2/toolchains` package reads that projection, so every task that invokes -Buck is ordered after `mr apply`. - -## Workspace Anatomy - -Per [decision 0020](../.decisions/0020-one-writable-mount-workspaces.md), the -workspace root sits at the store worktree path (policy-compatible with the -fleet worktree-placement and search-depth guards, and the layout under which -store GC and hygiene rules keep working): +`--isolation-dir` is CLI-only and cannot be pinned by buckconfig, so mr owns +the invocation wrapper that fixes the paused composed shape to its isolation +dir (COMP-R07). Standalone invocations use their own fixed isolation identity; +action-key parity between the two shapes is not promised. + +Ordinary development and single-repository CI use standalone Buck project +roots. Their tracked `.buckconfig` declares the member cell at `.`, the bundled +Prelude, and the root-owned `capabilities//` cell. The devenv shell links the +pure `packages..buck2-capabilities` output at +`.buck2/capabilities`; no projector runs during shell entry. + +The same Nix output is the only capability projection used by composition. +`mr apply` treats the member manifest as data, verifies the projection's exact +BUCK and `defs.bzl` bytes, platform, tool and executable identities, closure +paths, and generation, then atomically links the output at the composition +root's `.buck2/capabilities`. A member-shipped projector remains inert data. +Hub toolchains address the projection through `capabilities//`, so read-only +member mounts do not carry or mutate a projection. + +## Paused Composed Workspace Anatomy + +For an explicitly requested cross-repository composition, the workspace root +sits at the store worktree path per +[decision 0020](../.decisions/0020-one-writable-mount-workspaces.md) +(policy-compatible with the fleet worktree-placement and search-depth guards, +and the layout under which store GC and hygiene rules keep working): ```text ~/.megarepo/github.com///refs/heads// # workspace root @@ -154,7 +156,10 @@ defaults to `--kind cell` (the member) — scripts wanting the workspace pass `--kind project`. Teardown is an mr operation (protected mounts need a dirs-only unprotect before removal), never a bare `rm -rf`. -## Agent Workflow Contract — Revision 3 +## Composed-Exception Agent Workflow Contract — Revision 3 + +This contract applies only when a cross-repository composition is explicitly +requested during the paused retirement window: ```text branchy/mr owns / @@ -164,11 +169,11 @@ branchy/mr owns / +-- repos// reference only; outside Buck ``` -The workspace root is orchestration state, not an authoring checkout. Agents -follow these rules: +The workspace root is orchestration state, not an authoring checkout. In this +exceptional shape, agents follow these rules: -1. Start and resume work through the store-backed workspace; do not create an - independent checkout outside the store. +1. Start and resume the composition through its store-backed workspace; do not + create an independent composed checkout outside the store. 2. Use `repos/` as the default cwd and the only source tree mutated by the session. 3. Run git, devenv, Genie, pnpm, and package-local commands from the owned @@ -187,9 +192,9 @@ follow these rules: the current cell root. 8. Use mr for apply, advance, recovery, status, and teardown. Never replace protected-mount teardown with `rm -rf` or an in-place copy. -9. CI creates a job-owned store branch with an explicit worktree mode, - synthesizes composition before credentials, runs source-dependent commands - from the owned member, and always invokes guarded teardown. +9. Effect-utils CI runs from the actions checkout as its tracked standalone + Buck root. The paused composed shape is a development-only exception until + L3 cut 2 and is never synthesized as CI setup. 10. A dirty non-owned mount, a foreign real path, a missing ownership manifest, or an R6 mismatch is a hard stop. Do not repair around the guard. 11. Handoffs name both the workspace root and owned-member cwd, plus any @@ -199,14 +204,19 @@ follow these rules: ## Standalone Variant -A single-member build is simply a workspace with no other members mounted: -the owned repo still lives at `repos/` under its canonical cell name, -and the platform labels are byte-identical. Proven at the action-digest -level: digests are identical across single-member, two-member, renamed-root, -and real-dotfiles-root shapes, and between a writable branch worktree and a -read-only mount at the same commit — the root cell's name, the root's -absolute path, and the mount's write bit are all irrelevant to member -identity. +A standalone repository is the normative Buck project root for ordinary +development and single-repository CI. Its tracked `.buckconfig` maps the +canonical member cell name to `.`, declares the same platform labels and +Nix-produced capability cell, and its tracked `.buckroot` prevents accidental +discovery of an outer project. Effect-utils CI and devenv Buck tasks use this +shape directly; a second standalone checkout at the same revision is the +BUCK-R06 cache-reuse comparison context. + +The paused composed shape remains distinct until L3 cut 2: the member is +mounted at `repos/` under the same canonical cell name. Action-digest +parity between these two shapes must be measured rather than assumed; the +standalone CI cutover does not weaken the same-shape, cross-checkout BUCK-R06 +zero-reexecution requirement. ## Invariants Worth Restating diff --git a/context/buck2/intuition.md b/context/buck2/intuition.md index 4aad193280..fa9e51871a 100644 --- a/context/buck2/intuition.md +++ b/context/buck2/intuition.md @@ -18,12 +18,12 @@ that changes a live system. Contracts pass data in one direction; none of the systems becomes a hidden second producer for another. Reuse is the point, and identity is what makes it work. Action keys follow -result-affecting inputs and nothing else, so one shared cache serves every -worktree, machine, and composed repository — provided the composition shape is -canonical. That is why every build runs from a synthesized composition root: -mount paths, cell names, platform labels, and the isolation dir all enter -action identity, and megarepo/genie hold them constant so identical work keys -identically everywhere. +result-affecting inputs and nothing else, so one shared cache serves +same-platform standalone checkouts within a trust tier. The tracked repository +root is the normative development and single-repository CI shape. Mount paths, +cell names, platform labels, and the isolation dir enter action identity, so +the paused composed exception is a distinct migration shape rather than a +promised source of cross-shape action-key parity. The reusable part is deliberately smaller than a repository: shared rules and schemas carry no private facts, so a second consumer (dotfiles first) can diff --git a/context/buck2/requirements.md b/context/buck2/requirements.md index d7d5aaa410..926ca31a58 100644 --- a/context/buck2/requirements.md +++ b/context/buck2/requirements.md @@ -76,10 +76,14 @@ invariants named in its own document: the product digest a consumer pins, not a shared action key ([decision 0034](./.decisions/0034-artifact-default-composition-no-registry.md)). - **BUCK-R06 Shared reuse:** Admitted actions read and write the remote action - cache of their trust tier (decision 0033). A second same-platform context of - the same repository at an identical revision re-executes zero actions for - unchanged admitted targets; a violation is a - key-stability regression ([04-reuse](./04-reuse/requirements.md)). + cache of their trust tier (decision 0033). A second same-platform standalone + checkout of the same repository at an identical revision re-executes zero + actions for unchanged admitted targets; a violation is a key-stability + regression ([04-reuse](./04-reuse/requirements.md), + [second-context key-stability evidence](./.experiments/2026-09-19-second-context-key-stability.md)). + Cross-shape comparison + between standalone and paused composed roots is a migration diagnostic, not + an enduring action-key portability contract. - **BUCK-R07 Wall-clock budgets:** The admitted surface holds a warm no-op check at ≤ 5 s and a fresh-context green with warm shared cache at ≤ 3 min. Admission widening that breaks a budget is a regression to fix before diff --git a/devenv.nix b/devenv.nix index 3bbc65d788..7785165e70 100644 --- a/devenv.nix +++ b/devenv.nix @@ -29,6 +29,7 @@ let ''; builtins.getFlake "git+file://${toString ./.}"; currentSystem = pkgs.stdenv.hostPlatform.system; + buck2Capabilities = repoFlake.packages.${currentSystem}.buck2-capabilities; flakePkgs = import repoFlake.inputs.nixpkgs { system = currentSystem; }; trackedBuck2Products = import ./nix/buck2-products { pkgs = flakePkgs; }; # `restate` ships under BSL-1.1; scope allowUnfree to just that package so the @@ -373,6 +374,66 @@ let after = lane.unboundedAfter; }) (builtins.filter (lane: lane.unboundedFiles != [ ]) buck2TestLanes); sourceTestPackages = sourceOnlyTestPackages ++ unboundedTestPackages; + typescriptPublicationRootPredicate = '' + typescript_publication_workspace_root() { + local member_root workspace_root branch_ref repo_root bare_repo common_dir admin_dir + local backlink backlink_dir repository_root + + member_root="$(${pkgs.coreutils}/bin/realpath "$1")" || return 1 + + # The tracked Buck root is the ordinary publication shape. Its root marker and + # Git top-level identity prevent a directory that merely resembles repos/effect-utils + # from inheriting write authority. + if [ -f "$member_root/.buckroot" ]; then + repository_root="$(${pkgs.git}/bin/git -C "$member_root" rev-parse \ + --path-format=absolute --show-toplevel)" || return 1 + repository_root="$(${pkgs.coreutils}/bin/realpath "$repository_root")" || return 1 + [ "$repository_root" = "$member_root" ] || return 1 + printf "%s\n" "$member_root" + return 0 + fi + + # The composed shape remains an explicit downstream compatibility boundary. + workspace_root="$(${pkgs.coreutils}/bin/realpath "$member_root/../..")" || return 1 + [ "$member_root" = "$workspace_root/repos/effect-utils" ] || return 1 + [ -f "$member_root/.git" ] || return 1 + + branch_ref="$(${pkgs.git}/bin/git -C "$member_root" symbolic-ref --quiet HEAD)" || return 2 + case "$branch_ref" in + refs/heads/*) ;; + *) return 1 ;; + esac + + common_dir="$(${pkgs.git}/bin/git -C "$member_root" rev-parse \ + --path-format=absolute --git-common-dir)" || return 2 + common_dir="$(${pkgs.coreutils}/bin/realpath "$common_dir")" || return 2 + bare_repo="$common_dir" + [ "$(${pkgs.coreutils}/bin/basename "$bare_repo")" = ".bare" ] || return 2 + repo_root="$(${pkgs.coreutils}/bin/dirname "$bare_repo")" + [ "$workspace_root" = "$repo_root/$branch_ref" ] || return 1 + + admin_dir="$(${pkgs.git}/bin/git -C "$member_root" rev-parse \ + --path-format=absolute --git-dir)" || return 2 + admin_dir="$(${pkgs.coreutils}/bin/realpath "$admin_dir")" || return 2 + [ "$(${pkgs.coreutils}/bin/dirname "$admin_dir")" = "$bare_repo/worktrees" ] || + return 2 + [ -f "$admin_dir/gitdir" ] || return 2 + backlink="$(<"$admin_dir/gitdir")" + case "$backlink" in + /*) ;; + *) backlink="$admin_dir/$backlink" ;; + esac + backlink_dir="$(${pkgs.coreutils}/bin/realpath \ + "$(${pkgs.coreutils}/bin/dirname "$backlink")")" || return 2 + backlink="$backlink_dir/$(${pkgs.coreutils}/bin/basename "$backlink")" + [ "$backlink" = "$member_root/.git" ] || return 2 + + printf "%s\n" "$workspace_root" + } + ''; + standaloneBuckCachePosture = '' + ${pkgs.bun}/bin/bun "$root/scripts/buck2-cache-posture.ts" "$root" + ''; buck2BuildExec = { name, targets }: @@ -385,15 +446,15 @@ let pkgs.watchman ] } - workspace_root="$(${pkgs.coreutils}/bin/realpath "$root/../..")" - buck="$workspace_root/.megarepo/bin/buck2" - exec "$buck" build \ + ${standaloneBuckCachePosture} + cd "$root" + exec "$BUCK2_BIN" build \ --target-platforms effect_utils//buck2/platforms:host_platform \ ${lib.concatStringsSep " \\\n " targets} ''; - # Buck-invoking tasks discover the same pinned composed binary as `buck2:check`, so a lane - # cannot run against a different Buck than the one the check gate proved. + # Every Buck-invoking task uses the checkout's pinned binary and standalone + # project root, so CI lanes cannot silently fall back to a composed workspace. buck2UnitTestExec = { name, targets }: trace.exec name '' @@ -405,9 +466,9 @@ let pkgs.watchman ] } - workspace_root="$(${pkgs.coreutils}/bin/realpath "$root/../..")" - buck="$workspace_root/.megarepo/bin/buck2" - exec "$buck" test \ + ${standaloneBuckCachePosture} + cd "$root" + exec "$BUCK2_BIN" test \ --target-platforms effect_utils//buck2/platforms:host_platform \ --local-only \ ${lib.concatStringsSep " \\\n " targets} @@ -419,7 +480,7 @@ let lane: lib.nameValuePair lane.taskName { description = "Execute the bounded ${lane.packageName} unit-test lane under Buck"; - after = [ "mr:apply" ] ++ lib.optional (lane ? unboundedTaskName) lane.unboundedTaskName; + after = [ "genie:check" ] ++ lib.optional (lane ? unboundedTaskName) lane.unboundedTaskName; # trace-audit-allow: buck2UnitTestExec returns a trace.exec-wrapped command. exec = buck2UnitTestExec { name = lane.taskName; @@ -529,103 +590,67 @@ let "pnpm-lock.yaml" "pnpm-workspace.yaml" ]; - composedWorkspaceRootPredicate = '' - composed_workspace_root() { - local member_root workspace_root branch_ref repo_root bare_repo common_dir admin_dir - local backlink backlink_dir current_worktree current_branch registered_worktree registered_branch - local matching_path_registrations matching_branch_registrations - - member_root="$(${pkgs.coreutils}/bin/realpath "$1")" || return 1 - workspace_root="$(${pkgs.coreutils}/bin/realpath "$member_root/../..")" || return 1 - [ "$member_root" = "$workspace_root/repos/effect-utils" ] || return 1 - [ -f "$member_root/.git" ] || return 1 - - branch_ref="$(${pkgs.git}/bin/git -C "$member_root" symbolic-ref --quiet HEAD)" || return 2 - case "$branch_ref" in - refs/heads/*) ;; - *) return 1 ;; - esac - case "$workspace_root" in - */"$branch_ref") repo_root="''${workspace_root%/"$branch_ref"}" ;; - *) return 1 ;; - esac - bare_repo="$repo_root/.bare" - [ -d "$bare_repo" ] || return 2 - common_dir="$(${pkgs.git}/bin/git -C "$member_root" rev-parse \ - --path-format=absolute --git-common-dir)" || return 2 - [ "$common_dir" = "$bare_repo" ] || return 2 - - admin_dir="$(${pkgs.git}/bin/git -C "$member_root" rev-parse \ - --path-format=absolute --git-dir)" || return 2 - admin_dir="$(${pkgs.coreutils}/bin/realpath "$admin_dir")" || return 2 - [ "$(${pkgs.coreutils}/bin/dirname "$admin_dir")" = "$bare_repo/worktrees" ] || - return 2 - [ -f "$admin_dir/gitdir" ] || return 2 - backlink="$(<"$admin_dir/gitdir")" - case "$backlink" in - /*) ;; - *) backlink="$admin_dir/$backlink" ;; - esac - backlink_dir="$(${pkgs.coreutils}/bin/realpath \ - "$(${pkgs.coreutils}/bin/dirname "$backlink")")" || return 2 - backlink="$backlink_dir/$(${pkgs.coreutils}/bin/basename "$backlink")" - [ "$backlink" = "$member_root/.git" ] || return 2 - - current_worktree= - current_branch= - registered_worktree= - registered_branch= - matching_path_registrations=0 - matching_branch_registrations=0 - while IFS= read -r -d "" field; do - case "$field" in - worktree\ *) current_worktree="''${field#worktree }" ;; - branch\ *) current_branch="''${field#branch }" ;; - "") - if [ "$current_worktree" = "$member_root" ]; then - registered_branch="$current_branch" - matching_path_registrations=$((matching_path_registrations + 1)) - fi - if [ "$current_branch" = "$branch_ref" ]; then - registered_worktree="$current_worktree" - matching_branch_registrations=$((matching_branch_registrations + 1)) - fi - current_worktree= - current_branch= - ;; - esac - done < <(${pkgs.git}/bin/git --git-dir="$bare_repo" worktree list --porcelain -z) - [ "$matching_path_registrations" -eq 1 ] || return 2 - [ "$matching_branch_registrations" -eq 1 ] || return 2 - [ "$registered_branch" = "$branch_ref" ] || return 2 - [ "$registered_worktree" = "$member_root" ] || return 2 - - printf "%s\n" "$workspace_root" - } - ''; + # Single declaration for the source-generator import closure published before Genie can load. + # genie:editor-view-closure:check walks every generator with the shared bootstrap closure checker + # and fails when this list omits a first-party runtime package boundary. + editorBootstrapRootPackagePath = "packages/@overeng/genie"; + editorBootstrapPackagePaths = [ + "." + "packages/@overeng/otel-contract" + ]; + buck2AggregateExec = + taskName: target: + trace.exec taskName '' + set -euo pipefail + root="''${DEVENV_ROOT:-$PWD}" + export PATH=${lib.makeBinPath [ pkgs.watchman ]} + cd "$root" + ${standaloneBuckCachePosture} + exec "$BUCK2_BIN" build ${lib.escapeShellArg target} + ''; editorViewExec = - mode: - trace.exec "buck2:editor:${mode}" '' + { + mode, + packagePaths ? null, + traceScope ? null, + }: + let + traceName = + "buck2:editor:${mode}${lib.optionalString (traceScope != null) ":${traceScope}"}"; + packageArgument = lib.optionalString (packagePaths != null) + " --packages ${lib.escapeShellArg (builtins.toJSON packagePaths)}"; + in + trace.exec traceName '' set -euo pipefail - ${composedWorkspaceRootPredicate} root="''${DEVENV_ROOT:-$PWD}" - workspace_root="$(composed_workspace_root "$root")" || { - identity_status=$? - echo "buck2:editor:${mode} requires a composed megarepo workspace" >&2 - exit "$identity_status" - } + ${standaloneBuckCachePosture} exec ${pkgs.bun}/bin/bun "$root/scripts/editor-view-authority.ts" ${mode} \ --repo-root "$root" \ - --workspace-root "$workspace_root" \ + --workspace-root "$root" \ --cell effect_utils \ - --buck2 "$workspace_root/.megarepo/bin/buck2" \ + --buck2 "$BUCK2_BIN" \ --git ${pkgs.git}/bin/git \ --output "$root/.devenv/editor-workspace-authority.json" \ --publisher "$root/packages/@overeng/buck2-tools/src/editor-view.ts" \ --cp ${pkgs.coreutils}/bin/cp \ --mv ${pkgs.coreutils}/bin/mv \ - --snapshot-retention 3 + --snapshot-retention 3${packageArgument} ''; + scopedEditorViewPublisher = + { + description, + packagePaths, + traceScope, + }: + { + inherit description; + after = [ "genie:check" ]; + # trace-audit-allow: editorViewExec returns a trace.exec-wrapped command. + exec = editorViewExec { + mode = "publish"; + inherit packagePaths traceScope; + }; + }; in { imports = [ @@ -717,7 +742,7 @@ in # unlike the deterministic check/diff runs, this is a subprocess e2e (spawns otelite, binds an # ephemeral port, depends on export-flush timing), so it lives in CI rather than gating every # local `check:all` on capture reliability. - (taskModules.weaver-live-check { installTask = "buck2:editor:publish"; }) + (taskModules.weaver-live-check { installTask = "buck2:editor:publish:otel-contract"; }) # Version-pin consistency smoke (SC-DQ4): catches weaver/semconv pin drift the content # gate (weaver:check) silently degrades past (a bumped version with a stale FOD hash). (taskModules.weaver-version-smoke { }) @@ -735,7 +760,7 @@ in # exists exactly where the baseline gate still needs a source report. (taskModules.test-playwright { playwrightPkg = inputs.playwright.packages.${currentSystem}.playwright; - installTask = "buck2:editor:publish"; + installTask = "buck2:editor:publish:playwright"; # Launch the CLI through @playwright/test so the runner and test imports # share one module instance inside the Buck editor dependency view. playwrightBin = "node_modules/@playwright/test/cli.js"; @@ -821,7 +846,7 @@ in # fingerprint so a warm shell cannot bypass projection invalidation. extraFingerprintGlobs = genieExtraInputGlobs; # Run the one ordered mutating entrypoint. Its internal task sequence - # preserves generator/freshness/composition/publication happens-before. + # preserves generator/freshness/publication happens-before. optionalTasks = [ "buck2:editor:materialize" ]; completionsCliNames = [ "genie" @@ -842,13 +867,14 @@ in # The packaged Genie CLI is self-contained; generator sources resolve their # external imports through the committed-graph bootstrap editor views. This - # stage-zero publication cannot report governed Buck evidence: genie:check - # must first prove the graph fresh, then mr:apply and the authoritative - # publisher replay it. - tasks."genie:run".after = [ "buck2:editor:bootstrap" ]; - tasks."genie:check".after = [ "buck2:editor:bootstrap" ]; - tasks."lint:check:genie".after = [ "buck2:editor:bootstrap" ]; - tasks."genie:watch".after = [ "buck2:editor:bootstrap" ]; + # stage-zero publication cannot report governed Buck evidence: the closure + # checker first proves that the declared publication set covers every + # first-party runtime boundary, then genie:check proves the tracked standalone + # graph fresh and the authoritative publisher replays it. + tasks."genie:run".after = [ "genie:editor-view-closure:check" ]; + tasks."genie:check".after = lib.mkForce [ "genie:prepare" "genie:editor-view-closure:check" ]; + tasks."lint:check:genie".after = [ "genie:editor-view-closure:check" ]; + tasks."genie:watch".after = [ "genie:editor-view-closure:check" ]; tasks."lint:check:lockfile".description = lib.mkForce "Verify lockfile and package specifiers through source-side Genie freshness"; tasks."lint:check:lockfile".after = lib.mkForce [ "genie:check" ]; @@ -857,7 +883,7 @@ in ); tasks."lint:fix:oxlint".after = [ "buck2:editor:publish" ]; tasks."devenv-modules:test".after = lib.mkForce [ "buck2:editor:publish" ]; - tasks."test:restate-integration".after = lib.mkForce [ "buck2:editor:publish" ]; + tasks."test:restate-integration".after = lib.mkForce [ "buck2:editor:publish:restate-effect" ]; tasks."test:notion-integration:notion-effect-client".after = lib.mkForce [ "buck2:editor:publish" ]; tasks."test:notion-integration:notion-cli".after = lib.mkForce [ "buck2:editor:publish" ]; tasks."test:notion-integration:notion-datasource-sync".after = lib.mkForce [ @@ -865,7 +891,7 @@ in ]; tasks."test:notion-integration:notion-md".after = lib.mkForce [ "buck2:editor:publish" ]; tasks."test:notion-integration:notion-react".after = lib.mkForce [ "buck2:editor:publish" ]; - tasks."weaver:live-check".after = lib.mkForce [ "buck2:editor:publish" ]; + tasks."weaver:live-check".after = lib.mkForce [ "buck2:editor:publish:otel-contract" ]; tasks."test:pty-effect:unbounded".env = { NODE_PTY_NATIVE_PACKAGE = "${nodePtyNative}/node_modules/node-pty"; NODE_OPTIONS = "--import=${./. + "/packages/@overeng/pty-effect/test/node-pty-native-hook.ts"}"; @@ -873,18 +899,18 @@ in # Read-only formatting and linting are Buck actions over the exact generated # source manifest. Mutation remains source-side under lint:fix. - tasks."lint:check:format".after = lib.mkForce [ "mr:apply" ]; + tasks."lint:check:format".after = lib.mkForce [ "genie:check" ]; tasks."lint:check:format".exec = lib.mkForce (buck2BuildExec { name = "lint:check:format"; targets = [ "effect_utils//buck2/static:check_format" ]; }); - tasks."lint:check:oxlint".after = lib.mkForce [ "mr:apply" ]; + tasks."lint:check:oxlint".after = lib.mkForce [ "genie:check" ]; tasks."lint:check:oxlint".exec = lib.mkForce (buck2BuildExec { name = "lint:check:oxlint"; targets = [ "effect_utils//buck2/static:check_lint" ]; }); tasks."lint:check:asset-import-needs-type-reference" = { - after = [ "mr:apply" ]; + after = [ "genie:check" ]; description = "Require travelling type references for compiled asset imports through Buck"; # trace-audit-allow: buck2BuildExec returns a trace.exec-wrapped command. exec = buck2BuildExec { @@ -893,13 +919,13 @@ in }; }; tasks."lint:check".after = lib.mkAfter [ "lint:check:asset-import-needs-type-reference" ]; - tasks."lint:check:genie:coverage".after = lib.mkForce [ "mr:apply" ]; + tasks."lint:check:genie:coverage".after = lib.mkForce [ "genie:check" ]; tasks."lint:check:genie:coverage".exec = lib.mkForce (buck2BuildExec { name = "lint:check:genie:coverage"; targets = [ "effect_utils//buck2/static:check_policy" ]; }); tasks."workspace:check" = { - after = [ "mr:apply" ]; + after = [ "genie:check" ]; description = "Validate generated workspace package inventory through Buck"; # trace-audit-allow: buck2BuildExec returns a trace.exec-wrapped command. exec = buck2BuildExec { @@ -959,16 +985,15 @@ in env.MR_COMPOSITION_GIT_BIN = "${pkgs.git}/bin/git"; env.MR_COMPOSITION_WATCHMAN_BIN = "${pkgs.watchman}/bin/watchman"; env.MR_CAPABILITY_NIX_BIN = "${pkgs.nix}/bin/nix"; + env.MR_CAPABILITY_PROJECTION = "${buck2Capabilities}"; env.MR_CAPABILITY_MV_BIN = "${pkgs.coreutils}/bin/mv"; # restate-server binary path for restate-effect integration tests (test/test-utils.ts # reads RESTATE_SERVER_BIN to locate the native server, else falls back to $PATH). env.RESTATE_SERVER_BIN = "${restate}/bin/restate-server"; - # Genie and mr run from packaged products, but the generated projection still defines - # the graph mr composes. Generation freshness is therefore a source-side stage-zero - # prerequisite: a stale graph must fail before reconciliation can publish it to Buck. - # The composed-root mutators also remain serialized behind mr:setup. + # Composed development workspaces remain available until L3 cut 2, but no CI + # or repository Buck task depends on their mutators. tasks."mr:setup".after = [ "mr:bootstrap" ]; tasks."mr:apply".after = [ "genie:check" @@ -1026,7 +1051,7 @@ in tasks."test:megarepo-cold-gc" = { after = [ "buck2:editor:publish" ]; - description = "Run isolated megarepo cold-GC integration tests"; + description = "Run fixture-isolated megarepo cold-GC integration tests from the standalone checkout"; cwd = "packages/@overeng/megarepo"; exec = trace.exec "test:megarepo-cold-gc" '' set -euo pipefail @@ -1041,7 +1066,7 @@ in }; tasks."bundle:smoke" = { - after = [ "mr:apply" ]; + after = [ "genie:check" ]; description = "Bundle representative public entries through Buck with Vite/Rollup"; # trace-audit-allow: buck2UnitTestExec returns a trace.exec-wrapped command. exec = buck2UnitTestExec { @@ -1172,7 +1197,7 @@ in tasks."buck2:nix-bridge:check" = { description = "Check build-product, cache-publication, and fail-closed artifact-import contracts"; - after = [ "mr:apply" ]; + after = lib.mkForce [ "genie:check" ]; exec = trace.exec "buck2:nix-bridge:check" '' set -euo pipefail ${pkgs.bash}/bin/bash nix/workspace-tools/lib/tests/buck2-build-product-contract.sh "$PWD" @@ -1182,49 +1207,82 @@ in }; tasks."buck2:editor:bootstrap" = { - description = "Bootstrap source-generator dependencies from the committed Buck graph"; - after = [ "mr:setup" ]; + description = "Bootstrap source-generator dependencies from the committed standalone Buck graph"; # trace-audit-allow: editorViewExec returns a trace.exec-wrapped command. - exec = editorViewExec "bootstrap"; + exec = editorViewExec { + mode = "bootstrap"; + packagePaths = editorBootstrapPackagePaths; + }; + }; + + tasks."genie:editor-view-closure:check" = { + description = "Prove the bootstrap editor views cover every generator runtime package boundary"; + after = [ "buck2:editor:bootstrap" ]; + exec = trace.exec "genie:editor-view-closure:check" '' + set -euo pipefail + root="''${DEVENV_ROOT:-$PWD}" + exec ${pkgs.bun}/bin/bun "$root/genie/ci-scripts/bootstrap-closure-check.ts" \ + --root "$root" \ + --editor-view-root-package-path ${lib.escapeShellArg editorBootstrapRootPackagePath} \ + --editor-view-package-paths ${lib.escapeShellArg (builtins.toJSON editorBootstrapPackagePaths)} + ''; }; # Authoring and declaration publication need generated projections to be # updated before freshness is checked, but standalone genie:check must remain - # mutation-free. Keep that mutating sequence in one explicit entrypoint - # rather than adding global edges between genie:run and genie:check. + # mutation-free. Keep that mutating sequence in one explicit entrypoint. tasks."buck2:editor:materialize" = { - description = "Regenerate, freshness-check, recompose, and publish every editor dependency view in order"; + description = "Regenerate, freshness-check, and publish every editor dependency view in order"; exec = trace.exec "buck2:editor:materialize" '' set -euo pipefail export DEVENV_TUI=false - devenv tasks run mr:setup devenv tasks run buck2:editor:bootstrap --mode single devenv tasks run genie:run --mode single devenv tasks run genie:check --mode single - devenv tasks run mr:apply --mode single devenv tasks run buck2:editor:publish --mode single ''; }; tasks."buck2:editor:authority" = { description = "Prove complete Buck ownership of every workspace editor dependency view"; - after = [ "mr:apply" ]; + after = [ "genie:check" ]; # trace-audit-allow: editorViewExec returns a trace.exec-wrapped command. - exec = editorViewExec "authority"; + exec = editorViewExec { mode = "authority"; }; }; tasks."buck2:editor:publish" = { description = "Atomically publish every Buck-owned workspace editor dependency view"; - after = [ "mr:apply" ]; + after = [ "genie:check" ]; # trace-audit-allow: editorViewExec returns a trace.exec-wrapped command. - exec = editorViewExec "publish"; + exec = editorViewExec { mode = "publish"; }; + }; + + tasks."buck2:editor:publish:restate-effect" = scopedEditorViewPublisher { + description = "Atomically publish the Restate integration editor dependency view"; + packagePaths = [ "packages/@overeng/restate-effect" ]; + traceScope = "restate-effect"; + }; + + tasks."buck2:editor:publish:otel-contract" = scopedEditorViewPublisher { + description = "Atomically publish the Weaver live-check editor dependency view"; + packagePaths = [ "packages/@overeng/otel-contract" ]; + traceScope = "otel-contract"; + }; + + tasks."buck2:editor:publish:playwright" = scopedEditorViewPublisher { + description = "Atomically publish the shared Playwright editor dependency views"; + packagePaths = [ + "packages/@overeng/tui-react" + "packages/@overeng/utils" + ]; + traceScope = "playwright"; }; tasks."buck2:editor:check" = { description = "Fail when any published workspace editor dependency view is stale"; - after = [ "mr:apply" ]; + after = [ "genie:check" ]; # trace-audit-allow: editorViewExec returns a trace.exec-wrapped command. - exec = editorViewExec "check"; + exec = editorViewExec { mode = "check"; }; }; tasks."buck2:editor:recover-lock" = { @@ -1246,7 +1304,7 @@ in after = [ "buck2:editor:materialize" ]; exec = trace.exec "buck2:typescript:materialize-dist" '' set -euo pipefail - ${composedWorkspaceRootPredicate} + ${typescriptPublicationRootPredicate} root="''${DEVENV_ROOT:-$PWD}" export PATH=${ lib.makeBinPath [ @@ -1254,13 +1312,17 @@ in pkgs.watchman ] } - workspace_root="$(composed_workspace_root "$root")" || { + export WORKSPACE_ROOT="$root" + workspace_root="$(typescript_publication_workspace_root "$root")" || { identity_status=$? - echo "buck2:typescript:materialize-dist requires a composed megarepo workspace" >&2 + echo "buck2:typescript:materialize-dist requires a composed" \ + "megarepo workspace or a standalone Buck root" >&2 exit "$identity_status" } - export WORKSPACE_ROOT="$workspace_root" - export BUCK2_BIN="$WORKSPACE_ROOT/.megarepo/bin/buck2" + if [ "$workspace_root" != "$root" ]; then + export WORKSPACE_ROOT="$workspace_root" + export BUCK2_BIN="$workspace_root/.megarepo/bin/"buck2 + fi exec ${pkgs.bun}/bin/bun "$root/genie/buck2/typescript-authority-runtime.ts" \ materialize-dist "$root" ${pkgs.bash}/bin/bash ''; @@ -1278,10 +1340,22 @@ in ''; }; + tasks."check:buck2-producer-overlap" = { + description = "Reject duplicate Buck and legacy TypeScript producers"; + after = [ "genie:check" ]; + exec = trace.exec "check:buck2-producer-overlap" '' + set -euo pipefail + root="''${DEVENV_ROOT:-$PWD}" + exec ${pkgs.bun}/bin/bun "$root/genie/buck2/producer-overlap.ts" check \ + "$root/.devenv/gc/task-config-devenv-config-task-config" + ''; + }; + + # The provider audit remains separate because it validates the + # capability/toolchain boundary rather than producing an admitted artifact. tasks."buck2:check" = { description = "Build every admitted TypeScript check, declared test lane, and the archive/product Buck2 surface"; after = [ - "mr:apply" "buck2:nix-bridge:check" "buck2:task-guards:check" "buck2:rust-deps:check" @@ -1289,37 +1363,49 @@ in exec = trace.exec "buck2:check" '' set -euo pipefail root="''${DEVENV_ROOT:-$PWD}" - export PATH=${ - lib.makeBinPath [ - pkgs.coreutils - pkgs.watchman - ] - } - workspace_root="$(${pkgs.coreutils}/bin/realpath "$root/../..")" - buck="$workspace_root/.megarepo/bin/buck2" - "$buck" audit providers \ - --target-platforms effect_utils//buck2/platforms:host_platform \ - effect_utils//buck2/toolchains:cross_cell_provider_identity \ - effect_utils//buck2/toolchains:cross_cell_product_identity - exec ${pkgs.bun}/bin/bun "$root/genie/buck2/typescript-authority-runtime.ts" \ - build "$buck" + export PATH=${lib.makeBinPath [ pkgs.watchman ]} + cd "$root" + exec "$BUCK2_BIN" audit providers \ + --target-platforms //buck2/platforms:host_platform \ + //buck2/toolchains:cross_cell_provider_identity \ + //buck2/toolchains:cross_cell_product_identity ''; }; + tasks."buck2:quick" = { + description = "Build the admitted quick Buck aggregate"; + after = [ "buck2:check" ]; + # trace-audit-allow: buck2AggregateExec returns a trace.exec-wrapped command. + exec = buck2AggregateExec "buck2:quick" "//:quick"; + }; + + tasks."buck2:all" = { + description = "Build the complete admitted Buck aggregate"; + after = [ "buck2:check" ]; + # trace-audit-allow: buck2AggregateExec returns a trace.exec-wrapped command. + exec = buck2AggregateExec "buck2:all" "//:all"; + }; + + tasks."check:quick".after = [ + "buck2:quick" + "check:buck2-producer-overlap" + ]; + # One Buck invocation executes every admitted bounded lane. This is what `test:run` waits on; # the per-lane `test:` tasks (imported above) exist for standalone use and are not # part of that graph, so no suite is scheduled twice. tasks."test:buck2:unit" = { description = "Execute every admitted bounded unit-test lane under Buck"; - after = [ "mr:apply" ]; + after = [ "genie:check" ]; # trace-audit-allow: buck2UnitTestExec returns a trace.exec-wrapped command. exec = buck2UnitTestExec { name = "test:buck2:unit"; targets = map (lane: lane.target) buck2TestLanes; }; }; - tasks."check:all".after = [ + "buck2:all" + "check:buck2-producer-overlap" "cargo:check" "dependency-materialization:evidence:check" ]; @@ -1338,11 +1424,10 @@ in pkgs.watchman ] } - workspace_root="$(${pkgs.coreutils}/bin/realpath "$root/../..")" exec ${pkgs.bun}/bin/bun "$root/packages/@overeng/utils-dev/src/check-baseline-test-collection.ts" \ --root "$root" \ - --buck2 "$workspace_root/.megarepo/bin/buck2" \ - --buck2-cwd "$workspace_root" + --buck2 "$BUCK2_BIN" \ + --buck2-cwd "$root" '' ); @@ -1358,6 +1443,16 @@ in enterShell = '' export WORKSPACE_ROOT="$PWD" export PATH="$WORKSPACE_ROOT/node_modules/.bin:$PATH" + # Buck2 expands the cache header in the daemon; keep the optional credential + # defined so unauthenticated cache reads work when SecretSpec is not active. + export BUCK2_REMOTE_CACHE_BASIC_AUTH="''${BUCK2_REMOTE_CACHE_BASIC_AUTH:-}" + capability_parent="$WORKSPACE_ROOT/.buck2" + capability_link="$capability_parent/capabilities" + ${pkgs.coreutils}/bin/mkdir -p "$capability_parent" + if [ -e "$capability_link" ] && [ ! -L "$capability_link" ]; then + ${pkgs.coreutils}/bin/rm -rf -- "$capability_link" + fi + ${pkgs.coreutils}/bin/ln -sfnT ${buck2Capabilities} "$capability_link" ${cliBuildStamp.shellHook} ''; diff --git a/flake.nix b/flake.nix index bc1ae72cfb..3549fb28b1 100644 --- a/flake.nix +++ b/flake.nix @@ -141,6 +141,7 @@ ; products = trackedBuck2Products.products; typeProofCompilerBin = "${tsgo.packages.${system}.tsgo}/bin/tsgo"; + capabilityProjection = buck2Capabilities; }; ghCiUtils = import (rootPath + "/packages/@overeng/gh-ci-utils/nix/build.nix") { inherit @@ -163,6 +164,7 @@ }; }); }; + in { buckProducts = trackedBuck2Products; @@ -185,10 +187,6 @@ }) buckProductsFromSource ); oxlint-npm = oxlintNpm; - # oxlint-npm wrapped with automatic @overeng/oxc-config plugin injection - oxlint-with-plugins = import ./nix/oxlint-with-plugins.nix { - inherit pkgs oxlintNpm; - }; node-pty-native = nodePtyNative; } // pkgs.lib.optionalAttrs (system == "x86_64-linux") { } diff --git a/genie/buck2/producer-overlap.ts b/genie/buck2/producer-overlap.ts new file mode 100644 index 0000000000..8198419c82 --- /dev/null +++ b/genie/buck2/producer-overlap.ts @@ -0,0 +1,126 @@ +import { readFileSync } from 'node:fs' +import process from 'node:process' + +import { + authoritativeBuck2TypeScriptDeclarations, + authoritativeBuck2TypeScriptProjects, + type AuthoritativeBuck2TypeScriptDeclaration, + type AuthoritativeBuck2TypeScriptProject, +} from './typescript-admissions.ts' + +export type ProducerOperation = 'dist' | 'typecheck' + +export type ProducerOverlapAllowance = { + readonly ledgerRow: `effect-utils/${ProducerOperation}/${string}` + readonly operation: ProducerOperation + readonly packagePath: string +} + +export type ProducerOverlap = { + readonly operation: ProducerOperation + readonly packagePath: string + readonly producers: readonly [buck: string, legacy: string] +} + +/** Lower stack members retired every prior allowance with the root TypeScript solution. */ +export const producerOverlapAllowlist: readonly ProducerOverlapAllowance[] = [] + +const overlapKey = ({ + operation, + packagePath, +}: Pick): string => `${operation}:${packagePath}` + +export const findProducerOverlaps = ({ + projects, + declarations, + allowances, + devenvTaskNames, +}: { + readonly projects: readonly AuthoritativeBuck2TypeScriptProject[] + readonly declarations: readonly AuthoritativeBuck2TypeScriptDeclaration[] + readonly allowances: readonly ProducerOverlapAllowance[] + readonly devenvTaskNames: readonly string[] +}): readonly ProducerOverlap[] => { + const taskNames = new Set(devenvTaskNames) + const overlaps: ProducerOverlap[] = [] + + if (taskNames.has('ts:check')) { + overlaps.push( + ...projects.map( + (project): ProducerOverlap => ({ + operation: 'typecheck', + packagePath: project.packagePath, + producers: [`Buck ${project.typecheckTarget}`, 'devenv ts:check'], + }), + ), + ) + } + if (taskNames.has('ts:emit')) { + overlaps.push( + ...declarations.map( + (declaration): ProducerOverlap => ({ + operation: 'dist', + packagePath: declaration.packagePath, + producers: [`Buck ${declaration.distTarget}`, 'devenv ts:emit'], + }), + ), + ) + } + + const overlapKeys = new Set(overlaps.map(overlapKey)) + const staleAllowances = allowances.filter( + (allowance) => overlapKeys.has(overlapKey(allowance)) === false, + ) + if (staleAllowances.length > 0) { + throw new Error( + `stale Buck producer overlap allowances: ${staleAllowances.map(({ ledgerRow }) => ledgerRow).join(', ')}`, + ) + } + + const allowedKeys = new Set(allowances.map(overlapKey)) + return overlaps.filter((overlap) => allowedKeys.has(overlapKey(overlap)) === false) +} + +const taskNamesFromDocument = (document: unknown): readonly string[] => { + if (Array.isArray(document)) { + return document.flatMap((task) => + task !== null && typeof task === 'object' && 'name' in task && typeof task.name === 'string' + ? [task.name] + : [], + ) + } + if (document === null || typeof document !== 'object') return [] + const tasks = 'tasks' in document ? document.tasks : document + if (Array.isArray(tasks)) return taskNamesFromDocument(tasks) + if (tasks === null || typeof tasks !== 'object') return [] + return Object.entries(tasks).flatMap(([key, task]) => + task !== null && typeof task === 'object' && 'name' in task && typeof task.name === 'string' + ? [task.name] + : [key], + ) +} + +const main = (): number => { + const [operation, taskDocumentPath, ...unexpected] = process.argv.slice(2) + if (operation !== 'check' || taskDocumentPath === undefined || unexpected.length > 0) { + console.error('usage: producer-overlap.ts check ') + return 2 + } + + const taskDocument: unknown = JSON.parse(readFileSync(taskDocumentPath, 'utf8')) + const overlaps = findProducerOverlaps({ + projects: authoritativeBuck2TypeScriptProjects, + declarations: authoritativeBuck2TypeScriptDeclarations, + allowances: producerOverlapAllowlist, + devenvTaskNames: taskNamesFromDocument(taskDocument), + }) + if (overlaps.length === 0) return 0 + for (const overlap of overlaps) { + console.error( + `${overlap.packagePath} ${overlap.operation} has overlapping producers: ${overlap.producers.join(', ')}`, + ) + } + return 1 +} + +if (import.meta.main === true) process.exit(main()) diff --git a/genie/buck2/producer-overlap.unit.test.ts b/genie/buck2/producer-overlap.unit.test.ts new file mode 100644 index 0000000000..55b8248960 --- /dev/null +++ b/genie/buck2/producer-overlap.unit.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest' + +import { findProducerOverlaps, type ProducerOverlapAllowance } from './producer-overlap.ts' +import type { + AuthoritativeBuck2TypeScriptDeclaration, + AuthoritativeBuck2TypeScriptProject, +} from './typescript-admissions.ts' + +const project = { + packagePath: 'packages/@example/widget', + projectPath: 'packages/@example/widget', + projectFile: 'tsconfig.json', + typecheckTarget: '//packages/@example/widget:typecheck', +} as const satisfies AuthoritativeBuck2TypeScriptProject +const declaration = { + ...project, + declarationEntrypoint: 'src/mod.d.ts', + distTarget: '//packages/@example/widget:dist', + testTargets: [], +} as const satisfies AuthoritativeBuck2TypeScriptDeclaration + +const allAllowances = [ + { + ledgerRow: 'effect-utils/typecheck/widget', + operation: 'typecheck', + packagePath: project.packagePath, + }, + { + ledgerRow: 'effect-utils/dist/widget', + operation: 'dist', + packagePath: project.packagePath, + }, +] as const satisfies readonly ProducerOverlapAllowance[] + +describe('Buck producer overlap guard', () => { + it('fails an overlap outside the transitional allowlist', () => { + expect( + findProducerOverlaps({ + projects: [project], + declarations: [declaration], + allowances: [], + devenvTaskNames: ['ts:check', 'ts:emit'], + }), + ).toEqual([ + { + operation: 'typecheck', + packagePath: project.packagePath, + producers: ['Buck //packages/@example/widget:typecheck', 'devenv ts:check'], + }, + { + operation: 'dist', + packagePath: project.packagePath, + producers: ['Buck //packages/@example/widget:dist', 'devenv ts:emit'], + }, + ]) + }) + + it('passes known overlaps and rejects stale ledger allowances', () => { + expect( + findProducerOverlaps({ + projects: [project], + declarations: [declaration], + allowances: allAllowances, + devenvTaskNames: ['ts:check', 'ts:emit'], + }), + ).toEqual([]) + + expect(() => + findProducerOverlaps({ + projects: [project], + declarations: [declaration], + allowances: allAllowances, + devenvTaskNames: [], + }), + ).toThrow('stale Buck producer overlap allowances') + }) +}) diff --git a/genie/buck2/root-aggregate-projection.ts b/genie/buck2/root-aggregate-projection.ts new file mode 100644 index 0000000000..c38f01b0f1 --- /dev/null +++ b/genie/buck2/root-aggregate-projection.ts @@ -0,0 +1,94 @@ +import { + createGenieOutput, + type GenieOutput, +} from '../../packages/@overeng/genie/src/runtime/core.ts' +import { buck2SemanticFingerprint } from './mod.ts' +import { + authoritativeBuck2TypeScriptDeclarations, + authoritativeBuck2TypeScriptProjects, + buck2TypeScriptTestTargets, +} from './typescript-admissions.ts' + +const regenerationCommand = 'devenv tasks run genie:run' as const + +export type RootBuckAggregatePlan = { + readonly quick: readonly string[] + readonly all: readonly string[] +} +const fullAuthorityTargets = [ + '//buck2/toolchains:archive_tool', + '//buck2/toolchains:product_tool', +] as const + +export const planRootBuckAggregates = ({ + typecheckTargets = authoritativeBuck2TypeScriptProjects.map((project) => project.typecheckTarget), + distTargets = authoritativeBuck2TypeScriptDeclarations.map( + (declaration) => declaration.distTarget, + ), + testTargets = buck2TypeScriptTestTargets, +}: { + readonly typecheckTargets?: readonly string[] + readonly distTargets?: readonly string[] + readonly testTargets?: readonly string[] +} = {}): RootBuckAggregatePlan => ({ + quick: typecheckTargets, + all: [':quick', ...distTargets, ...testTargets, ...fullAuthorityTargets], +}) + +const renderFilegroup = ({ name, srcs }: { name: string; srcs: readonly string[] }): string => + [ + 'filegroup(', + ` name = ${JSON.stringify(name)},`, + ' srcs = {', + ...srcs.map( + (target, index) => ` ${JSON.stringify(index.toString().padStart(3, '0'))}: ${JSON.stringify(target)},`, + ), + ' },', + ' visibility = ["PUBLIC"],', + ')', + ].join('\n') + +const rootBuckBase = + 'load("//buck2:editor_view.bzl", "editor_view_inputs")\nload("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain")\nload("//buck2:static_checks.bzl", "STATIC_SOURCE_EXCLUDES", "STATIC_SOURCE_GLOBS", "static_source_set")\n\n# Conventional prelude toolchain targets, owned by the platform hub.\n#\n# The composition root sets `[cell_aliases] toolchains = `\n# (`composition/root/composition-root.ts`), so prelude\'s conventional\n# `toolchains//:` spelling resolves into *this* package for every member cell in the\n# composed workspace. Prelude rules used by any member therefore find exactly one instance\n# of each conventional toolchain, and it is the hub\'s capability-backed one. Keeping them\n# here preserves `05-composition/spec.md:51-56` ("the root carries no synthetic toolchains\n# or `none` cell").\ntoolchain_alias(\n name = "rust",\n actual = "//buck2/toolchains:rust",\n visibility = ["PUBLIC"],\n)\n\ntoolchain_alias(\n name = "cxx",\n actual = "//buck2/toolchains:cxx",\n visibility = ["PUBLIC"],\n)\n\ntoolchain_alias(\n name = "go_bootstrap",\n actual = "//buck2/toolchains:go_bootstrap",\n visibility = ["PUBLIC"],\n)\n\ntoolchain_alias(\n name = "python_bootstrap",\n actual = "//buck2/toolchains:python_bootstrap",\n visibility = ["PUBLIC"],\n)\n\n# Prelude\'s genrule toolchain carries no executable at all (`zip_scrubber = None`,\n# `@prelude//:genrule_toolchain.bzl`), so there is nothing to pin and nothing to project:\n# the upstream instance is already hermetic.\nsystem_genrule_toolchain(\n name = "genrule",\n visibility = ["PUBLIC"],\n)\n\nexport_file(\n name = "package.json",\n src = "package.json",\n visibility = ["PUBLIC"],\n)\n\nalias(\n name = "node_modules",\n actual = "//packages/@overeng/genie:node_modules",\n visibility = ["PUBLIC"],\n)\n\nalias(\n name = "editor_inputs",\n actual = ":node_modules",\n visibility = ["PUBLIC"],\n)\n\nalias(\n name = "root_editor_package_tree",\n actual = "//packages/@overeng/genie:package_tree",\n visibility = ["PUBLIC"],\n)\n\neditor_view_inputs(\n name = "editor_view_inputs",\n editor_inputs = ":editor_inputs",\n package_tree = ":root_editor_package_tree",\n visibility = ["PUBLIC"],\n)\nstatic_source_set(\n name = "static_sources",\n prefix = "",\n srcs = glob(\n [\n root + "/" + pattern\n for root in ["context", "packages", "scripts"]\n for pattern in STATIC_SOURCE_GLOBS\n ],\n exclude = [\n root + "/" + pattern\n for root in ["context", "packages", "scripts"]\n for pattern in STATIC_SOURCE_EXCLUDES\n ],\n ) + [\n ".oxfmtrc.json",\n ".oxlintrc.json",\n "devenv.lock",\n "devenv.yaml",\n "flake.lock",\n "flake.nix",\n "megarepo.kdl",\n "megarepo.lock",\n "tsconfig.lint.json",\n ],\n visibility = ["PUBLIC"],\n)\n\n\n# Workspace patches are declared inputs to the generated pnpm extraction actions.\nexport_file(\n name = "patches/@myobie__pty@0.10.0.patch",\n src = "patches/@myobie__pty@0.10.0.patch",\n visibility = ["PUBLIC"],\n)' + +export const rootBuckAggregateProjection = (): GenieOutput => { + const data = planRootBuckAggregates() + const packageInputs = [ + ...new Set( + authoritativeBuck2TypeScriptProjects.map((project) => `${project.packagePath}/BUCK.genie.ts`), + ), + ].toSorted() + const semanticInputs = [ + 'BUCK.genie.ts', + 'genie/buck2/mod.ts', + 'genie/buck2/root-aggregate-projection.ts', + 'genie/buck2/typescript-admissions.ts', + ...packageInputs, + ] + const fingerprint = buck2SemanticFingerprint({ + generator: 'effect-utils/genie/buck2-root-aggregate-projection', + schemaVersion: 1, + semanticData: data, + }) + + return createGenieOutput({ + data, + stringify: () => + [ + '# Projection source: BUCK.genie.ts', + '# Projection schema version: 1', + '# Projection generator: effect-utils/genie/buck2-root-aggregate-projection', + `# Semantic fingerprint: ${fingerprint}`, + `# Semantic inputs: ${semanticInputs.join(', ')}`, + `# Regenerate: ${regenerationCommand}`, + '', + rootBuckBase, + '', + '# Scoped repository checks derived from the TypeScript admission registry.', + renderFilegroup({ name: 'quick', srcs: data.quick }), + '', + renderFilegroup({ name: 'all', srcs: data.all }), + '', + ].join('\n'), + }) +} diff --git a/genie/buck2/root-aggregate-projection.unit.test.ts b/genie/buck2/root-aggregate-projection.unit.test.ts new file mode 100644 index 0000000000..63dc06077c --- /dev/null +++ b/genie/buck2/root-aggregate-projection.unit.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' + +import { planRootBuckAggregates, rootBuckAggregateProjection } from './root-aggregate-projection.ts' +import { authoritativeBuck2TypeScriptProjects } from './typescript-admissions.ts' + +describe('root Buck aggregate projection', () => { + it('derives quick and all from the authoritative target sets', () => { + expect( + planRootBuckAggregates({ + typecheckTargets: ['//packages/@example/alpha:typecheck'], + distTargets: ['//packages/@example/alpha:dist'], + testTargets: ['//packages/@example/alpha:test'], + }), + ).toEqual({ + quick: ['//packages/@example/alpha:typecheck'], + all: [ + ':quick', + '//packages/@example/alpha:dist', + '//packages/@example/alpha:test', + '//buck2/toolchains:archive_tool', + '//buck2/toolchains:product_tool', + ], + }) + }) + + it('renders the stack-head root targets and registry aggregates together', () => { + const output = rootBuckAggregateProjection().stringify({ cwd: '/repo', location: '' }) + + expect(output).toContain('name = "editor_view_inputs"') + expect(output).toContain('name = "static_sources"') + expect(output).toContain('name = "quick"') + expect(output).toContain('name = "all"') + expect(output).toContain('"000": "//context/effect/socket:typecheck"') + expect(output).not.toContain('srcs = [') + }) + + it('keeps the production quick target set equal to the admission registry', () => { + expect(planRootBuckAggregates().quick).toEqual( + authoritativeBuck2TypeScriptProjects.map((project) => project.typecheckTarget), + ) + }) +}) diff --git a/genie/buck2/typescript-package-projection.unit.test.ts b/genie/buck2/typescript-package-projection.unit.test.ts index fcf4b26feb..e467f4266e 100644 --- a/genie/buck2/typescript-package-projection.unit.test.ts +++ b/genie/buck2/typescript-package-projection.unit.test.ts @@ -174,13 +174,8 @@ describe('declared-closure package projection', () => { } }) - it('keeps admitted subpackage inputs out of the root Buck package', () => { + it('keeps package-tree runtime ownership out of the root Buck package', () => { const rootBuck = readFileSync('BUCK', 'utf8') - for (const { packagePath } of admittedPackages) { - expect(rootBuck, `root BUCK still owns files below //${packagePath}`).not.toContain( - `${packagePath}/`, - ) - } expect(rootBuck).not.toContain('name = "package_tree_runtime",') expect(buck2ToolsBuck).toContain('name = "package_tree_runtime",') for (const admitted of admittedPackages) { diff --git a/genie/ci-scripts/cleanup-effect-utils-composition.sh b/genie/ci-scripts/cleanup-effect-utils-composition.sh deleted file mode 100755 index de7b093ede..0000000000 --- a/genie/ci-scripts/cleanup-effect-utils-composition.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -store_root="${MEGAREPO_STORE:-${RUNNER_TEMP:?}/megarepo-store/${GITHUB_RUN_ID:-local}/${GITHUB_RUN_ATTEMPT:-0}/${GITHUB_JOB:-job}}" -branch_seed="ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}" -branch_name="$(printf '%s' "$branch_seed" | tr -c 'A-Za-z0-9_-' '_')" -branch_ref="refs/heads/$branch_name" -repo_root="$store_root/github.com/overengineeringstudio/effect-utils" -bare_repo="$repo_root/.bare" -workspace_root="$repo_root/$branch_ref" -member_root="$workspace_root/repos/effect-utils" - -if [ ! -e "$workspace_root" ] && [ ! -L "$workspace_root" ]; then - exit 0 -fi - -# Everything below compares these paths against Git output, and Git always -# answers with resolved paths. RUNNER_TEMP and MEGAREPO_STORE arrive as the -# runner set them, which on macOS is under the /var -> /private/var symlink, so -# an unresolved store root would never equal the worktree Git reports and the -# guards below would abort a legitimate cleanup. Resolve once, on both sides, -# and re-derive; the existence test above deliberately stays on the raw path. -store_root="$(cd "$store_root" && pwd -P)" -runner_temp="$(cd "${RUNNER_TEMP:?RUNNER_TEMP not set}" && pwd -P)" -repo_root="$store_root/github.com/overengineeringstudio/effect-utils" -bare_repo="$repo_root/.bare" -workspace_root="$repo_root/$branch_ref" -member_root="$workspace_root/repos/effect-utils" - -test -d "$bare_repo" -[ "$(git --git-dir="$bare_repo" rev-parse --is-bare-repository)" = true ] -case "$store_root" in - "$runner_temp"/megarepo-store/*) ;; - *) echo "::error::refusing cleanup outside job-local runner store: $store_root" >&2; exit 1 ;; -esac - -owned_worktree= - -current_worktree= -current_branch= -registered_branch= -matching_allowed_path_registrations=0 -matching_branch_registrations=0 -while IFS= read -r -d '' field; do - case "$field" in - worktree\ *) current_worktree="${field#worktree }" ;; - branch\ *) current_branch="${field#branch }" ;; - '') - if [ "$current_worktree" = "$workspace_root" ] || [ "$current_worktree" = "$member_root" ]; then - registered_branch="$current_branch" - owned_worktree="$current_worktree" - matching_allowed_path_registrations=$((matching_allowed_path_registrations + 1)) - fi - if [ "$current_branch" = "$branch_ref" ]; then - case "$current_worktree" in - "$workspace_root"|"$member_root") ;; - *) echo "::error::job-owned branch is registered at a foreign path: $current_worktree" >&2; exit 1 ;; - esac - matching_branch_registrations=$((matching_branch_registrations + 1)) - fi - current_worktree= - current_branch= - ;; - esac -done < <(git --git-dir="$bare_repo" worktree list --porcelain -z) - -[ "$matching_allowed_path_registrations" -eq 1 ] -[ "$matching_branch_registrations" -eq 1 ] -[ "$registered_branch" = "$branch_ref" ] -[ "$(git -C "$owned_worktree" rev-parse --path-format=absolute --show-toplevel)" = "$owned_worktree" ] -[ "$(git -C "$owned_worktree" rev-parse --path-format=absolute --git-common-dir)" = "$bare_repo" ] -[ "$(git -C "$owned_worktree" symbolic-ref --quiet HEAD)" = "$branch_ref" ] - -admin_dir="$(git -C "$owned_worktree" rev-parse --path-format=absolute --git-dir)" -admin_dir="$(cd "$admin_dir" && pwd -P)" -[ "$(dirname "$admin_dir")" = "$bare_repo/worktrees" ] -[ -f "$admin_dir/gitdir" ] -backlink="$(<"$admin_dir/gitdir")" -case "$backlink" in - /*) ;; - *) backlink="$admin_dir/$backlink" ;; -esac -backlink_dir="$(cd "$(dirname "$backlink")" && pwd -P)" -backlink="$backlink_dir/$(basename "$backlink")" -[ "$backlink" = "$owned_worktree/.git" ] - -# Composed capabilities can contain Nix-derived directories without owner write -# permission. Restore removal permission only after the ownership checks above. -chmod -R u+w "$workspace_root" - -git --git-dir="$bare_repo" worktree remove --force "$owned_worktree" -rm -rf -- "$workspace_root" -git --git-dir="$bare_repo" update-ref -d "$branch_ref" -rm -rf -- "$store_root" diff --git a/genie/ci-scripts/prepare-effect-utils-composition.sh b/genie/ci-scripts/prepare-effect-utils-composition.sh deleted file mode 100755 index b2f524db27..0000000000 --- a/genie/ci-scripts/prepare-effect-utils-composition.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -checkout_root="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE not set}" -export XDG_CACHE_HOME="${RUNNER_TEMP:?RUNNER_TEMP not set}/composition-state/nix-cache" -mkdir -p "$XDG_CACHE_HOME" -printf 'XDG_CACHE_HOME=%s\n' "$XDG_CACHE_HOME" >> "$GITHUB_ENV" -source_sha="$(git -C "$checkout_root" rev-parse --verify HEAD)" -case "$source_sha" in - ''|*[!0-9a-f]*) echo "::error::checkout HEAD is not an exact lowercase Git object id: $source_sha" >&2; exit 1 ;; -esac -checkout_status_before="$(git -C "$checkout_root" status --porcelain=v1 --untracked-files=all)" - -mr_out="$(cd "$checkout_root" && nix build --no-link --print-out-paths .#megarepo)" -mr_bin="$mr_out/bin/mr" -if [ ! -x "$mr_bin" ]; then - echo "::error::exact-checkout megarepo build did not produce an executable: $mr_bin" >&2 - exit 1 -fi - -branch_seed="ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}" -branch_name="$(printf '%s' "$branch_seed" | tr -c 'A-Za-z0-9_-' '_')" -branch_ref="refs/heads/$branch_name" -store_root="${MEGAREPO_STORE:-${RUNNER_TEMP:?}/megarepo-store/${GITHUB_RUN_ID:-local}/${GITHUB_RUN_ATTEMPT:-0}/${GITHUB_JOB:-job}}" -repo_root="$store_root/github.com/overengineeringstudio/effect-utils" -bare_repo="$repo_root/.bare" -workspace_root="$repo_root/$branch_ref" -member_root="$workspace_root/repos/effect-utils" -mkdir -p "$repo_root" -if [ ! -d "$bare_repo" ]; then - git init --bare "$bare_repo" -elif [ "$(git --git-dir="$bare_repo" rev-parse --is-bare-repository)" != true ]; then - echo "::error::canonical megarepo store path is not a bare Git repository: $bare_repo" >&2 - exit 1 -fi -origin_url="${EFFECT_UTILS_CI_ORIGIN_URL:-https://github.com/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY not set}.git}" -if git --git-dir="$bare_repo" remote get-url origin >/dev/null 2>&1; then - git --git-dir="$bare_repo" remote set-url origin "$origin_url" -else - git --git-dir="$bare_repo" remote add origin "$origin_url" -fi - -# Preserve the full main history required by merge-base consumers and fetch the -# exact checked-out commit from the public origin, never from a shallow checkout. -git --git-dir="$bare_repo" fetch --no-tags --prune origin \ - '+refs/heads/main:refs/remotes/origin/main' \ - "$source_sha" -workspace_parent="$(dirname "$workspace_root")" -if git --git-dir="$bare_repo" show-ref --verify --quiet "$branch_ref"; then - existing_sha="$(git --git-dir="$bare_repo" rev-parse "$branch_ref^{commit}")" - if [ "$existing_sha" != "$source_sha" ]; then - echo "::error::job-owned branch already names another commit: $branch_ref ($existing_sha != $source_sha)" >&2 - exit 1 - fi - if [ ! -d "$workspace_root" ]; then - echo "::error::job-owned branch exists without its canonical workspace: $workspace_root" >&2 - exit 1 - fi -else - if [ -e "$workspace_root" ] || [ -L "$workspace_root" ]; then - echo "::error::refusing foreign canonical workspace path: $workspace_root" >&2 - exit 1 - fi - mkdir -p "$workspace_parent" - created_member_root="$( - cd "$workspace_parent" - env -i \ - HOME="$HOME" \ - TMPDIR="${TMPDIR:-/tmp}" \ - XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}" \ - PATH="$PATH" \ - NIX_CONFIG='accept-flake-config = true' \ - MEGAREPO_STORE="$store_root" \ - CI=true \ - BUCK2_NO_REMOTE_CACHE="${BUCK2_NO_REMOTE_CACHE:-}" \ - "$mr_bin" store worktree new overengineeringstudio/effect-utils \ - --ref "$branch_name" \ - --base "$source_sha" \ - --porcelain - )" - case "$created_member_root" in - "$workspace_root"|"$member_root") ;; - *) - echo "::error::worktree creation returned '$created_member_root', expected '$workspace_root' or '$member_root'" >&2 - exit 1 - ;; - esac -fi -( - cd "$workspace_parent" - env -i \ - HOME="$HOME" \ - TMPDIR="${TMPDIR:-/tmp}" \ - XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}" \ - PATH="$PATH" \ - NIX_CONFIG='accept-flake-config = true' \ - MEGAREPO_STORE="$store_root" \ - CI=true \ - BUCK2_NO_REMOTE_CACHE="${BUCK2_NO_REMOTE_CACHE:-}" \ - "$mr_bin" --cwd "$workspace_root" apply --worktree-mode tracking --lock-sync off --output ci -) - -if git -C "$workspace_root" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - echo "::error::synthesized workspace root must not be a Git worktree: $workspace_root" >&2 - exit 1 -fi -test -f "$workspace_root/.megarepo/composition-generation.json" -test -f "$workspace_root/.buckconfig" -test -x "$workspace_root/.megarepo/bin/buck2" -test -L "$workspace_root/repos/effect" -member_sha="$(git -C "$member_root" rev-parse --verify HEAD)" -member_ref="$(git -C "$member_root" symbolic-ref --quiet HEAD)" -if [ "$member_sha" != "$source_sha" ] || [ "$member_ref" != "$branch_ref" ]; then - echo "::error::owned member identity mismatch: $member_ref@$member_sha, expected $branch_ref@$source_sha" >&2 - exit 1 -fi -if [ "$(git -C "$checkout_root" rev-parse --verify HEAD)" != "$source_sha" ] || [ "$(git -C "$checkout_root" status --porcelain=v1 --untracked-files=all)" != "$checkout_status_before" ]; then - echo "::error::composition preparation modified the actions checkout" >&2 - exit 1 -fi - -export EFFECT_UTILS_WORKSPACE_ROOT="$workspace_root" -export EFFECT_UTILS_MEMBER_ROOT="$member_root" -export MEGAREPO_STORE="$store_root" -printf 'EFFECT_UTILS_WORKSPACE_ROOT=%s\nEFFECT_UTILS_MEMBER_ROOT=%s\nMEGAREPO_STORE=%s\n' "$workspace_root" "$member_root" "$store_root" >> "$GITHUB_ENV" diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 7d8685563d..a71035ea35 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -23,7 +23,6 @@ * ``` */ - import type { GitHubWorkflowArgs } from '../packages/@overeng/genie/src/runtime/mod.ts' import { defaultRefPolicyCheckStep, @@ -182,8 +181,9 @@ export { appendGitHubAccessTokenToNixConfigStep, cachixCliBuildStep, cachixStep, + ciOtelSpansArtifactStep, + ciOtelSpansSummaryStep, checkoutStep, - cleanupEffectUtilsCompositionStep, ciDiagnosticsArtifactStep, ciDiagnosticsSetupStep, captureRunnerPressureStep, @@ -198,11 +198,11 @@ export { pnpmBuilderContractStep, defaultPnpmStateKeyPrefix, pnpmInstallWithDiagnosticsStep, + prepareCiOtelSpoolStep, pnpmStateCacheVersion, pnpmStatePublisherPostSteps, pnpmStateSetupStep, prepareCiScriptsStep, - prepareEffectUtilsCompositionStep, preparePinnedDevenvStep, preparePinnedDevenvStepFor, restoreNixCacheStep, diff --git a/genie/ci-workflow/setup.ts b/genie/ci-workflow/setup.ts index 0427911198..a797663139 100644 --- a/genie/ci-workflow/setup.ts +++ b/genie/ci-workflow/setup.ts @@ -19,6 +19,7 @@ import { ciPnpmHome, ciPnpmStatePaths, ciPnpmStore, + ciCompositionStateRoot, withCiSourceRoot, type NixBinaryCache, } from './shared.ts' @@ -26,8 +27,7 @@ import { type WorkflowJob = GitHubWorkflowArgs['jobs'][string] type WorkflowStep = WorkflowJob['steps'][number] -const evictPnpmDepsCachedOutputsScript = - `${preparedCiRuntimeScriptsDir}/evict-pnpm-deps-cached-outputs.sh` +const evictPnpmDepsCachedOutputsScript = `${preparedCiRuntimeScriptsDir}/evict-pnpm-deps-cached-outputs.sh` /** Evict cached pnpm-deps fixed-output outputs so CI re-derives them fresh. */ export const evictCachedPnpmDepsStep = ({ @@ -68,25 +68,6 @@ export const checkoutStep = (opts?: { repository?: string; ref?: string; path?: with: { 'persist-credentials': false, ...opts }, }) -/** - * Synthesize the disposable decision-0020 workspace used by effect-utils CI. - * - * The actions checkout remains untouched for action cleanup and artifact paths. - * Every source-dependent command after this step runs from the branch-attached - * owned member at `repos/effect-utils`. - */ -export const prepareEffectUtilsCompositionStep = { - name: 'Prepare effect-utils composition', - env: githubTokenEnv(), - run: '"$GITHUB_WORKSPACE/genie/ci-scripts/prepare-effect-utils-composition.sh"', -} as const - -/** Always remove the per-job synthesized workspace, worktree registration, and store. */ -export const cleanupEffectUtilsCompositionStep = { - name: 'Cleanup effect-utils composition', - if: 'always()', - run: '"$GITHUB_WORKSPACE/genie/ci-scripts/cleanup-effect-utils-composition.sh"', -} as const export const prepareCiScriptsStep = { name: 'Prepare CI helper scripts', @@ -734,6 +715,70 @@ export const standardSelfHostedPnpmCiPostSteps = (opts?: { ...(opts?.includeDiagnosticsArtifact === false ? [] : [ciDiagnosticsArtifactStep()]), ] as const +/** Enable file-backed OTLP delivery for every traced devenv task in this CI job. */ +export const prepareCiOtelSpoolStep = { + name: 'Prepare CI OpenTelemetry capture', + shell: 'bash', + run: [ + 'set -euo pipefail', + `spool_dir="${ciCompositionStateRoot}/otel-spans"`, + 'mkdir -p "$spool_dir"', + 'printf \'OTEL_SPAN_SPOOL_DIR=%s\\n\' "$spool_dir" >> "$GITHUB_ENV"', + 'printf \'OTEL_SPOOL_MULTI_WRITER=1\\n\' >> "$GITHUB_ENV"', + ].join('\n'), +} as const + +/** Render the slowest traced tasks directly in the GitHub job summary. */ +export const ciOtelSpansSummaryStep = { + name: 'Summarize CI OpenTelemetry spans', + if: "always() && env.OTEL_SPAN_SPOOL_DIR != ''", + shell: 'bash', + run: [ + 'set -euo pipefail', + 'shopt -s nullglob', + 'span_files=("$OTEL_SPAN_SPOOL_DIR"/*.jsonl)', + 'if [ "${#span_files[@]}" -eq 0 ]; then', + ' echo "::notice::No CI OpenTelemetry task spans were emitted"', + ' exit 0', + 'fi', + '{', + ' echo "## Slowest devenv tasks"', + ' echo ""', + ' echo "| Task | Duration | Status |"', + ' echo "| --- | ---: | --- |"', + " while IFS=$'\\t' read -r duration task status; do", + ' printf \'| `%s` | %.2fs | %s |\\n\' "$task" "$duration" "$status"', + ' done < <(', + " jq -s -r '", + ' [.[].resourceSpans[].scopeSpans[].spans[]', + ' | select(.name == "devenv.task.exec")', + ' | (.attributes | map({ key, value: (.value.stringValue // .value.intValue // .value.boolValue) }) | from_entries) as $attrs', + ' | [', + ' (((.endTimeUnixNano | tonumber) - (.startTimeUnixNano | tonumber)) / 1000000000),', + ' ($attrs["task.name"] // $attrs["span.label"] // .name),', + ' (if (.status.code // 0) == 2 then "error" else "ok" end)', + ' ]', + ' | @tsv', + ' ] | .[]', + " ' \"${span_files[@]}\" | sort -nr | sed -n '1,20p'", + ' )', + '} >> "$GITHUB_STEP_SUMMARY"', + ].join('\n'), +} as const + +/** Preserve each job's OTLP payload so traces remain inspectable without network credentials. */ +export const ciOtelSpansArtifactStep = { + name: 'Upload CI OpenTelemetry spans', + if: "always() && env.OTEL_SPAN_SPOOL_DIR != ''", + uses: 'actions/upload-artifact@v4', + with: { + name: 'ci-otel-${{ github.job }}-${{ runner.os }}-${{ runner.arch }}-run-${{ github.run_id }}-attempt-${{ github.run_attempt }}', + path: '${{ env.OTEL_SPAN_SPOOL_DIR }}', + 'if-no-files-found': 'warn', + 'retention-days': 14, + }, +} as const + export const devenvTaskStep = (name: string, ...args: [string, ...string[]]) => ({ name, env: githubTokenEnv(), diff --git a/genie/ci-workflow/shared.ts b/genie/ci-workflow/shared.ts index 554a131007..0ec5394ef9 100644 --- a/genie/ci-workflow/shared.ts +++ b/genie/ci-workflow/shared.ts @@ -381,11 +381,8 @@ export const ciNixCacheRoot = `${ciCompositionStateRoot}/nix-cache` /** Default Nix cache path restored/saved by the shared CI cache helpers. */ export const ciNixCachePath = `${ciNixCacheRoot}/nix` -/** - * Enter the source checkout selected for this shell step. Effect-utils CI exports - * the synthesized owned member; downstream workflows fall back to checkout. - */ -export const ciSourceRoot = 'cd "${EFFECT_UTILS_MEMBER_ROOT:-${GITHUB_WORKSPACE:-$PWD}}"' +/** Enter the actions checkout selected for this shell step. */ +export const ciSourceRoot = 'cd "${GITHUB_WORKSPACE:-$PWD}"' /** Run one source-dependent shell command from the canonical CI source root. */ export const withCiSourceRoot = (command: string) => `${ciSourceRoot} && ${command}` diff --git a/nix/devenv-modules/otel/otel-run.nix b/nix/devenv-modules/otel/otel-run.nix index 9e32816bbe..9094e1140a 100644 --- a/nix/devenv-modules/otel/otel-run.nix +++ b/nix/devenv-modules/otel/otel-run.nix @@ -26,7 +26,7 @@ # in separate Nix files, so there is no shared shell fragment to reuse yet. # # Usage: -# otel-run [--label ] [--join] [--] [args...] +# otel-run [--label ] [--attr ]... [--join] [--] [args...] # # Usage in a devenv module: # packages = [ (import ./otel/otel-run.nix { inherit pkgs; }) ]; @@ -36,21 +36,33 @@ pkgs.writeShellScriptBin "otel-run" '' _label="" _join=0 + _attrs=() # Parse otel-run's own flags; stop at `--` or the first non-flag (the command). while [ $# -gt 0 ]; do case "$1" in --label) _label="''${2:-}"; shift 2 ;; --label=*) _label="''${1#--label=}"; shift ;; + --attr) + [ -n "''${2:-}" ] || { echo "otel-run: --attr requires key=value" >&2; exit 2; } + _attrs+=(--attr "$2") + shift 2 + ;; + --attr=*) + [ -n "''${1#--attr=}" ] || { echo "otel-run: --attr requires key=value" >&2; exit 2; } + _attrs+=(--attr "''${1#--attr=}") + shift + ;; --join) _join=1; shift ;; --help | -h) cat >&2 <<'USAGE' - Usage: otel-run [--label ] [--join] [--] [args...] + Usage: otel-run [--label ] [--attr ]... [--join] [--] [args...] Mint a fresh root trace around and print its Grafana link. - --label Override the label derived from argv. - --join Join the ambient trace instead of minting a fresh root. + --label Override the label derived from argv. + --attr Add an attribute to the root span. Repeat as needed. + --join Join the ambient trace instead of minting a fresh root. USAGE exit 0 ;; @@ -106,22 +118,25 @@ pkgs.writeShellScriptBin "otel-run" '' ${pkgs.coreutils}/bin/timeout 1 ${pkgs.bash}/bin/bash -c "exec 3<>/dev/tcp/$_host/$_port" 2>/dev/null } - # Pick the first reachable endpoint: the configured one, else the standard - # local OTLP/HTTP ingress. Export it so both otel-span and the (possibly - # nested) child inherit the reachable endpoint instead of a dead collector. + # A spool directory is durable delivery and takes precedence over HTTP in + # otel-span. Only probe HTTP when no spool was configured. _endpoint="" - _endpoint_ok=0 - for _cand in "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" "http://127.0.0.1:4318"; do - [ -n "$_cand" ] || continue - [ -n "$_endpoint" ] || _endpoint="$_cand" - if _probe_endpoint "$_cand"; then - _endpoint="$_cand" - _endpoint_ok=1 - break + _delivery_ok=0 + if [ -n "''${OTEL_SPAN_SPOOL_DIR:-}" ] && [ -d "$OTEL_SPAN_SPOOL_DIR" ]; then + _delivery_ok=1 + else + for _cand in "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" "http://127.0.0.1:4318"; do + [ -n "$_cand" ] || continue + [ -n "$_endpoint" ] || _endpoint="$_cand" + if _probe_endpoint "$_cand"; then + _endpoint="$_cand" + _delivery_ok=1 + break + fi + done + if [ "$_delivery_ok" -eq 1 ]; then + export OTEL_EXPORTER_OTLP_ENDPOINT="$_endpoint" fi - done - if [ "$_endpoint_ok" -eq 1 ]; then - export OTEL_EXPORTER_OTLP_ENDPOINT="$_endpoint" fi # Run the command inside an otel-span root span, forwarding its exit code. @@ -129,12 +144,12 @@ pkgs.writeShellScriptBin "otel-run" '' otel-span run "effect-utils-devenv" "$_label" \ ''${_span_args[@]+"''${_span_args[@]}"} \ --attr "span.label=$_label" \ + ''${_attrs[@]+"''${_attrs[@]}"} \ -- "$@" || _exit=$? - # Warn (before the link) if no OTLP endpoint was reachable, so a printed URL - # that will dead-link is clearly caveated rather than silently misleading. - if [ "$_endpoint_ok" -ne 1 ]; then - printf '[otel] WARN: no reachable OTLP endpoint (%s); spans may not have landed. Start the stack with: devenv up\n' "''${_endpoint:-none}" >&2 + # Warn if neither a spool nor a reachable OTLP endpoint can receive spans. + if [ "$_delivery_ok" -ne 1 ]; then + printf '[otel] WARN: no span spool or reachable OTLP endpoint (%s); spans may not have landed. Start the stack with: devenv up\n' "''${_endpoint:-none}" >&2 fi # Print the Grafana explore URL for the trace (mirrors otel-span's diff --git a/nix/devenv-modules/tasks/lib/trace.nix b/nix/devenv-modules/tasks/lib/trace.nix index 9ea6fced4c..64fb19d87b 100644 --- a/nix/devenv-modules/tasks/lib/trace.nix +++ b/nix/devenv-modules/tasks/lib/trace.nix @@ -41,16 +41,33 @@ { lib }: let otelCanEmitShell = ''command -v "''${OTEL_SPAN_BIN:-otel-span}" >/dev/null 2>&1 && { [ -n "''${OTELITE_HTTP_ENDPOINT:-''${OTEL_EXPORTER_OTLP_ENDPOINT:-}}" ] || { [ -n "''${OTEL_SPAN_SPOOL_DIR:-}" ] && [ -d "''${OTEL_SPAN_SPOOL_DIR:-}" ]; }; }''; + ciAttrsShell = '' + _otel_ci_attrs=() + if [ "''${GITHUB_ACTIONS:-}" = "true" ]; then + _otel_ci_attrs=( + --attr "ci.workflow=''${GITHUB_WORKFLOW:-unknown}" + --attr "ci.job=''${GITHUB_JOB:-unknown}" + --attr "ci.run.id=''${GITHUB_RUN_ID:-unknown}" + --attr "ci.run.attempt=''${GITHUB_RUN_ATTEMPT:-unknown}" + --attr "ci.event=''${GITHUB_EVENT_NAME:-unknown}" + --attr "ci.repository=''${GITHUB_REPOSITORY:-unknown}" + --attr "ci.ref=''${GITHUB_REF:-unknown}" + --attr "ci.sha=''${GITHUB_SHA:-unknown}" + --attr "runner.os=''${RUNNER_OS:-unknown}" + --attr "runner.arch=''${RUNNER_ARCH:-unknown}" + ) + fi + ''; # Shell condition: an OTEL task trace context is actually ACTIVE — OTEL delivery # is available (otelCanEmitShell) AND a well-formed W3C traceparent is present # (OTEL_TASK_TRACEPARENT preferred, falling back to TRACEPARENT). This is the # single gate that decides whether COMMAND-level instrumentation should engage. # tsc (ts.nix) and trace.instr (oxlint/vitest) both gate on THIS exact string so - # they engage/disengage together: in a non-interactive `devenv tasks run` with no - # span parent and no OTLP endpoint (e.g. CI), every instrumented command runs - # bare; under `otel-span run -- devenv tasks run …` the task span exports - # OTEL_TASK_TRACEPARENT into the task body, so this is true and commands wrap. + # they engage/disengage together: without configured delivery and a parent + # context, every instrumented command runs bare. Under `otel-span run -- + # devenv tasks run …` the task span exports OTEL_TASK_TRACEPARENT into the + # task body, so this is true and commands wrap. otelTraceContextActive = ''${otelCanEmitShell} && [[ "''${OTEL_TASK_TRACEPARENT:-''${TRACEPARENT:-}}" =~ ^00-[0-9a-fA-F]{32}-[0-9a-fA-F]{16}-[0-9a-fA-F]{2}$ ]]''; taskFileStem = taskName: @@ -167,6 +184,7 @@ let # adapter records where the structured-source contract is met). traceExec = taskName: execBody: '' if ${otelCanEmitShell}; then + ${ciAttrsShell} _otel_project_attr=() if [ -n "''${OTEL_DEVENV_PROJECT:-}" ]; then _otel_project_attr=(--attr "devenv.project.name=$OTEL_DEVENV_PROJECT") @@ -178,6 +196,7 @@ let OTEL_EXPORTER_OTLP_ENDPOINT="''${OTELITE_HTTP_ENDPOINT:-''${OTEL_EXPORTER_OTLP_ENDPOINT:-}}" \ "''${OTEL_SPAN_BIN:-otel-span}" run "effect-utils-devenv" "devenv.task.exec" \ "''${_otel_project_attr[@]}" \ + "''${_otel_ci_attrs[@]}" \ --attr "tool.name=devenv" \ --attr "task.name=${taskName}" \ --attr "task.phase=exec" \ @@ -197,6 +216,7 @@ let # and forces span status to OK (status checks aren't errors). traceStatus = taskName: method: statusBody: '' if ${otelCanEmitShell}; then + ${ciAttrsShell} _status_exit=0 _otel_project_attr=() if [ -n "''${OTEL_DEVENV_PROJECT:-}" ]; then @@ -205,6 +225,7 @@ let OTEL_EXPORTER_OTLP_ENDPOINT="''${OTELITE_HTTP_ENDPOINT:-''${OTEL_EXPORTER_OTLP_ENDPOINT:-}}" \ "''${OTEL_SPAN_BIN:-otel-span}" run "effect-utils-devenv" "devenv.task.status" \ "''${_otel_project_attr[@]}" \ + "''${_otel_ci_attrs[@]}" \ --attr "tool.name=devenv" \ --attr "task.name=${taskName}" \ --attr "task.phase=status" \ diff --git a/nix/devenv-modules/tasks/local/devenv-module-tests.nix b/nix/devenv-modules/tasks/local/devenv-module-tests.nix index b6bed92ca3..2e1c4fe76c 100644 --- a/nix/devenv-modules/tasks/local/devenv-module-tests.nix +++ b/nix/devenv-modules/tasks/local/devenv-module-tests.nix @@ -16,7 +16,7 @@ let [ -f "$testFile" ] || continue found=true echo "Running $testFile" - NIX_FLAKE_REF="git+file://$PWD" BASH_BIN=${pkgs.bashNonInteractive}/bin/bash ${pkgs.bashNonInteractive}/bin/bash "$testFile" + NIX_FLAKE_REF="git+file://$PWD?shallow=1" BASH_BIN=${pkgs.bashNonInteractive}/bin/bash ${pkgs.bashNonInteractive}/bin/bash "$testFile" done if [ "$found" != true ]; then diff --git a/nix/devenv-modules/tasks/shared/tests/megarepo-status.test.sh b/nix/devenv-modules/tasks/shared/tests/megarepo-status.test.sh index 7bc7f12142..1c160eaf46 100755 --- a/nix/devenv-modules/tasks/shared/tests/megarepo-status.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/megarepo-status.test.sh @@ -224,6 +224,7 @@ for runtime_name in \ MR_COMPOSITION_GIT_BIN \ MR_COMPOSITION_WATCHMAN_BIN \ MR_CAPABILITY_NIX_BIN \ + MR_CAPABILITY_PROJECTION \ MR_CAPABILITY_MV_BIN do if ! grep -F "env.$runtime_name =" "$devenv_file" >/dev/null; then diff --git a/nix/devenv-modules/tasks/shared/tests/otel-instr-gating.test.sh b/nix/devenv-modules/tasks/shared/tests/otel-instr-gating.test.sh index 516c266307..afb8f4cf06 100755 --- a/nix/devenv-modules/tasks/shared/tests/otel-instr-gating.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/otel-instr-gating.test.sh @@ -88,12 +88,29 @@ env -i \ OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 \ OTELITE_HTTP_ENDPOINT=http://127.0.0.1:9876 \ OTEL_SPAN_BIN="$custom_otel_span" \ + GITHUB_ACTIONS=true \ + GITHUB_WORKFLOW=CI \ + GITHUB_JOB=test \ + GITHUB_RUN_ID=123 \ + GITHUB_RUN_ATTEMPT=2 \ + GITHUB_REPOSITORY=overengineeringstudio/effect-utils \ + GITHUB_SHA=0123456789abcdef \ + RUNNER_OS=Linux \ + RUNNER_ARCH=X64 \ OTEL_TEST_MARKER="$marker" \ "$BASH" "$task_exec" grep -q '^endpoint=http://127.0.0.1:9876$' "$marker" \ || fail "task/capture-endpoint: trace.exec should prefer the invocation-scoped otelite endpoint" grep -q '^args=run effect-utils-devenv devenv.task.exec ' "$marker" \ || fail "task/pinned-bridge: trace.exec should invoke OTEL_SPAN_BIN outside PATH" +grep -q -- '--attr ci.workflow=CI' "$marker" \ + || fail "task/ci-attributes: trace.exec should carry the workflow identity" +grep -q -- '--attr ci.run.id=123' "$marker" \ + || fail "task/ci-attributes: trace.exec should carry the run identity" +grep -q -- '--attr ci.repository=overengineeringstudio/effect-utils' "$marker" \ + || fail "task/ci-attributes: trace.exec should carry the repository identity" +grep -q -- '--attr runner.os=Linux' "$marker" \ + || fail "task/ci-attributes: trace.exec should carry the runner platform" # Common env for the "binaries present + delivery available" baseline. Each case # below overrides exactly one trigger. diff --git a/nix/devenv-modules/tasks/shared/tests/otel-run.test.sh b/nix/devenv-modules/tasks/shared/tests/otel-run.test.sh index a984a1823a..87393b682e 100644 --- a/nix/devenv-modules/tasks/shared/tests/otel-run.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/otel-run.test.sh @@ -67,11 +67,13 @@ cat > "$stubbin/otel-span" <<'STUB' shift # drop the `run` subcommand trace_id="" positional=() +attrs=() cmd=() while [ $# -gt 0 ]; do case "$1" in --trace-id) trace_id="$2"; shift 2 ;; - --attr | --span-id | --parent-span-id | --start-time-ns | --end-time-ns) shift 2 ;; + --attr) attrs+=("$2"); shift 2 ;; + --span-id | --parent-span-id | --start-time-ns | --end-time-ns) shift 2 ;; --) shift; cmd=("$@"); break ;; *) positional+=("$1"); shift ;; esac @@ -80,6 +82,7 @@ done echo "TRACE_ID=$trace_id" echo "SERVICE=${positional[0]:-}" echo "SPAN_NAME=${positional[1]:-}" + printf 'ATTR=%s\n' "${attrs[@]}" } >> "$OTEL_RUN_TEST_CAPTURE" exec "${cmd[@]}" STUB @@ -156,6 +159,25 @@ env -i PATH="$stubbin:$PATH" HOME="$tmpdir" OTEL_RUN_TEST_CAPTURE="$capD" \ [ "$(cap_val "$capD" SPAN_NAME)" = "custom-label" ] \ && ok || fail "label: --label should override, got '$(cap_val "$capD" SPAN_NAME)'" +# --- (4) ATTRIBUTES + SPOOL: forward metadata and treat a spool as durable delivery --- +capAttr="$tmpdir/capAttr" +errAttr="$tmpdir/errAttr" +spool="$tmpdir/spool" +mkdir -p "$spool" +env -i PATH="$stubbin:$PATH" HOME="$tmpdir" \ + OTEL_RUN_TEST_CAPTURE="$capAttr" \ + OTEL_SPAN_SPOOL_DIR="$spool" \ + "$otel_run" --attr ci.workflow=CI --attr ci.job=test -- true 2> "$errAttr" \ + || fail "attribute/spool run exited nonzero" +grep -q '^ATTR=ci.workflow=CI$' "$capAttr" \ + && grep -q '^ATTR=ci.job=test$' "$capAttr" \ + && ok || fail "attributes: expected repeated --attr values to reach otel-span" +if grep -q 'WARN: no reachable OTLP endpoint' "$errAttr"; then + fail "spool delivery must not warn about a missing OTLP endpoint" +else + ok +fi + # --- (5) EXIT CODE forwarding --- capE="$tmpdir/capE" set +e diff --git a/nix/devenv-modules/tasks/shared/tests/ts-otelite-e2e.test.sh b/nix/devenv-modules/tasks/shared/tests/ts-otelite-e2e.test.sh index 28e0a29357..abed0bb5ca 100755 --- a/nix/devenv-modules/tasks/shared/tests/ts-otelite-e2e.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/ts-otelite-e2e.test.sh @@ -110,7 +110,9 @@ ln -s "$otel_span_bin" "$tmpdir/bin/otel-span" ln -s "$otel_scrape_bin" "$tmpdir/bin/otel-scrape" cap="$tmpdir/capture" +# The outer CI job owns a durable spool; this nested capture must route to otelite's HTTP receiver. env -u TRACEPARENT -u OTEL_TASK_TRACEPARENT -u OTEL_SHELL_ENTRY_NS \ + -u OTEL_SPAN_SPOOL_DIR -u OTEL_SPOOL_MULTI_WRITER \ PATH="$tmpdir/bin:$PATH" DEVENV_ROOT="$tmpdir/workspace" DEVENV_TUI=false \ "$otelite_bin" run --out "$cap" --protocol http/json -- devenv tasks run ts:check \ > "$tmpdir/summary.json" 2> "$tmpdir/run.stderr" diff --git a/nix/workspace-tools/lib/buck2-product-candidates.nix b/nix/workspace-tools/lib/buck2-product-candidates.nix index fb6b33a41b..1f3647f611 100644 --- a/nix/workspace-tools/lib/buck2-product-candidates.nix +++ b/nix/workspace-tools/lib/buck2-product-candidates.nix @@ -10,6 +10,7 @@ products, nativeProducts ? (import ../../buck2-native-products { inherit pkgs; }).products, typeProofCompilerBin, + capabilityProjection ? null, oxfmtPkg ? pkgs.oxfmt, gitRev ? "unknown", commitTs ? 0, @@ -105,6 +106,9 @@ let MR_COMPOSITION_PLATFORM = if pkgs.stdenv.hostPlatform.isDarwin then "darwin" else "linux"; MR_COMPOSITION_SYSTEM = pkgs.stdenv.hostPlatform.system; MR_COMPOSITION_WATCHMAN_BIN = "${pkgs.watchman}/bin/watchman"; + } + // pkgs.lib.optionalAttrs (capabilityProjection != null) { + MR_CAPABILITY_PROJECTION = "${capabilityProjection}"; }; expectedExternalCapabilities = [ "buck2" diff --git a/packages/@overeng/buck2-tools/src/buck-watch.ts b/packages/@overeng/buck2-tools/src/buck-watch.ts index c0704b890e..cb6e91957f 100755 --- a/packages/@overeng/buck2-tools/src/buck-watch.ts +++ b/packages/@overeng/buck2-tools/src/buck-watch.ts @@ -483,6 +483,13 @@ const readEditorInputs = async ({ } } +/** Wall-clock phase emitted by one reconciliation for CI/bootstrap diagnosis. */ +export type BuckReconcileTiming = { + readonly phase: 'buck-build' | 'editor-view' + readonly durationMs: number + readonly packagePath?: string +} + /** Static configuration shared by every reconciliation pass of one watch plan. */ export type BuckReconcilerOptions = { readonly plan: BuckWatchPlan @@ -497,6 +504,9 @@ export type BuckReconcilerOptions = { readonly snapshotRetention: number readonly run?: RunCommand readonly signal?: AbortSignal + readonly onTiming?: (timing: BuckReconcileTiming) => void + /** Publish disjoint editor-root groups concurrently while preserving ordering within each lock. */ + readonly parallelEditorRoots?: boolean } /** Build the affected product set, then publish each affected editor view from provider roots. */ @@ -508,23 +518,26 @@ export const reconcileBuckViews = async ({ readonly options: BuckReconcilerOptions }): Promise => { const execute = options.run ?? runCommand + const buildStartedAt = performance.now() const built = await execute({ command: options.buck2, args: ['build', ...request.buildTargets, '--show-full-output'], cwd: options.workspaceRoot, ...(options.signal === undefined ? {} : { signal: options.signal }), }) + options.onTiming?.({ + phase: 'buck-build', + durationMs: performance.now() - buildStartedAt, + }) const outputs = parseBuildOutputs(built.stdout) const selected = new Set(request.packagePaths) const absoluteArtifact = (path: string): string => isAbsolute(path) === true ? path : resolve(options.workspaceRoot, path) - // Publication stays strictly ordered and never overlaps: the editor-view program takes a - // publication lock, and each manifest is read immediately before its own publication, so a - // failure leaves every earlier view published and every later view untouched. The promise - // chain expresses that sequence without a lexical await loop. - await options.plan.packages.reduce(async (previous, entry) => { - await previous - if (selected.has(entry.packagePath) === false || entry.editor === undefined) return + type EditorEntry = BuckWatchPackage & { + readonly editor: NonNullable + } + const publish = async (entry: EditorEntry): Promise => { + const publicationStartedAt = performance.now() const manifestOutput = outputForTarget({ outputs, target: entry.editor.inputsManifestTarget, @@ -567,7 +580,40 @@ export const reconcileBuckViews = async ({ detached: true, cwd: options.repoRoot, }) - }, Promise.resolve()) + options.onTiming?.({ + phase: 'editor-view', + packagePath: entry.packagePath, + durationMs: performance.now() - publicationStartedAt, + }) + } + const entries = options.plan.packages.filter( + (entry): entry is EditorEntry => + selected.has(entry.packagePath) === true && entry.editor !== undefined, + ) + const publishOrdered = async (group: readonly (typeof entries)[number][]): Promise => { + for (const entry of group) { + // Each group shares one publication lock, so preserve deterministic package order. + // eslint-disable-next-line no-await-in-loop + await publish(entry) + } + } + if (options.parallelEditorRoots !== true) { + await publishOrdered(entries) + return + } + + const groups = new Map() + for (const entry of entries) { + const editorRoot = + entry.packagePath === '.' + ? resolve(options.repoRoot, '.editor-view') + : resolve(options.repoRoot, entry.packagePath, '..', '..', '.editor-view') + const group = groups.get(editorRoot) ?? [] + group.push(entry) + groups.set(editorRoot, group) + } + const settled = await Promise.allSettled([...groups.values()].map(publishOrdered)) + for (const result of settled) if (result.status === 'rejected') throw result.reason } /** Atomically replace a machine-readable status file. */ diff --git a/packages/@overeng/buck2-tools/src/editor-view.ts b/packages/@overeng/buck2-tools/src/editor-view.ts index 6f5f3c4d22..c65d518423 100644 --- a/packages/@overeng/buck2-tools/src/editor-view.ts +++ b/packages/@overeng/buck2-tools/src/editor-view.ts @@ -860,16 +860,23 @@ const declaredSnapshotRoots = ({ ] } -const fingerprintDeclaredRoots = async ( - roots: readonly DeclaredSnapshotRoot[], -): Promise => { +const fingerprintDeclaredRoots = async ({ + roots, + knownRoot, +}: { + readonly roots: readonly DeclaredSnapshotRoot[] + readonly knownRoot?: { readonly source: string; readonly digest: string } +}): Promise => { // Declared roots are proven disjoint read-only trees, so their fingerprints are // computed concurrently. Settling first and rethrowing in declared order keeps both // the reported failure and the hashed sequence deterministic. const settled = await Promise.allSettled( roots.map(async (root) => ({ identity: root.identity, - digest: await canonicalTreeFingerprint({ tree: root.source }), + digest: + root.source === knownRoot?.source + ? knownRoot.digest + : await canonicalTreeFingerprint({ tree: root.source }), })), ) const entries = settled.map((result) => { @@ -891,11 +898,15 @@ const fingerprintSnapshotPayload = async (snapshotDir: string): Promise const backing = join(snapshotDir, '.backing') const nodeModules = join(snapshotDir, 'node_modules') if (pathExists(backing) === false) return canonicalTreeFingerprint({ tree: nodeModules }) - // Both payload roots are fixed and ordered, so the two digests are awaited in place: - // `.backing` is still fingerprinted strictly before `node_modules`. + // These disjoint immutable roots can be fingerprinted concurrently. Preserve their fixed order + // when framing the resulting payload digest so scheduling cannot affect the record identity. + const [backingDigest, nodeModulesDigest] = await Promise.all([ + canonicalTreeFingerprint({ tree: backing }), + canonicalTreeFingerprint({ tree: nodeModules }), + ]) const entries = [ - ['.backing', await canonicalTreeFingerprint({ tree: backing })], - ['node_modules', await canonicalTreeFingerprint({ tree: nodeModules })], + ['.backing', backingDigest], + ['node_modules', nodeModulesDigest], ] as const const hash = createHash('sha256') hash.update('effect-utils/editor-view-snapshot-payload/v1') @@ -1385,17 +1396,13 @@ export const recoverEditorViewLock = ({ rmSync(recoveredPath, { recursive: true }) } -const validateSnapshot = async ({ - snapshotDir, - expected, -}: { - snapshotDir: string - expected: EditorViewRecord -}): Promise => { +type SnapshotValidation = + | { readonly record: EditorViewRecord; readonly error?: never } + | { readonly record?: never; readonly error: unknown } + +const validateSnapshotContents = async (snapshotDir: string): Promise => { requireDirectory({ path: snapshotDir, field: 'snapshot' }) const record = readRecord(join(snapshotDir, 'editor-view.json')) - if (recordsEqual({ left: record, right: expected }) === false) - fail(`existing snapshot record mismatch: ${snapshotDir}`) const snapshotNodeModules = join(snapshotDir, 'node_modules') requireDirectory({ path: snapshotNodeModules, field: 'snapshot node_modules' }) requireReadOnlySnapshot(snapshotDir) @@ -1404,6 +1411,7 @@ const validateSnapshot = async ({ fail( `existing snapshot byte digest mismatch: recorded=${record.byteSnapshotDigest} actual=${digest}`, ) + return record } const publishCurrentPointer = ({ @@ -1536,8 +1544,48 @@ export const publishEditorView = async (options: EditorViewOptions): Promise { + if ( + pathExists(paths.current) === false || + lstatSync(paths.current).isSymbolicLink() === false + ) + return undefined + const pointer = readlinkSync(paths.current) + const snapshotDir = resolve(paths.editorRoot, pointer) + if (isWithin({ root: paths.storeDir, candidate: snapshotDir }) === false) return undefined + let record: EditorViewRecord + try { + record = readRecord(join(snapshotDir, 'editor-view.json')) + } catch { + return undefined + } + if ( + record.package !== options.package || + record.cell !== options.cell || + record.target !== options.target || + record.snapshot !== pointer || + record.editorInputsFingerprint !== fingerprint || + record.selectedViewDigest !== selectedViewDigest + ) + return undefined + return { + snapshotDir, + validation: validateSnapshotContents(snapshotDir).then( + (validated): SnapshotValidation => ({ record: validated }), + (error): SnapshotValidation => ({ error }), + ), + } + })() const finite = (options.backingRoots?.length ?? 0) > 0 const roots = finite === true @@ -1548,8 +1596,14 @@ export const publishEditorView = async (options: EditorViewOptions): Promise root.source), snapshot: candidate, }) - const after = await fingerprintDeclaredRoots(roots) + const after = await fingerprintDeclaredRoots({ roots }) if (after !== normalizedStoreDigest) fail( `declared backing roots changed while materializing: before=${normalizedStoreDigest} after=${after}`, @@ -1612,9 +1677,12 @@ export const publishEditorView = async (options: EditorViewOptions): Promise 0 - ? await fingerprintDeclaredRoots( - declaredSnapshotRoots({ + ? await fingerprintDeclaredRoots({ + roots: declaredSnapshotRoots({ nodeModules: options.nodeModules, backingRoots: options.backingRoots ?? [], }), - ) + }) : await canonicalTreeFingerprint({ tree: options.nodeModules, dereference: true }) if (normalizedStoreDigest !== record.normalizedStoreDigest) return failCheck({ diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index 1226fd261b..e9e5ce85ca 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -6,13 +6,12 @@ import { mkdtempSync, readFileSync, readlinkSync, - realpathSync, rmSync, symlinkSync, writeFileSync, } from 'node:fs' import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' +import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { describe, expect, it } from 'vitest' @@ -95,20 +94,6 @@ const nixGcRaceRetryScriptUrl = new URL( ) const nixGcRaceRetryScriptPath = fileURLToPath(nixGcRaceRetryScriptUrl) const nixGcRaceRetryScriptSource = readFileSync(nixGcRaceRetryScriptUrl, 'utf8') -const prepareEffectUtilsCompositionScriptSource = readFileSync( - new URL( - ['../../../../../../genie/ci-scripts', 'prepare-effect-utils-composition.sh'].join('/'), - import.meta.url, - ), - 'utf8', -) -const cleanupEffectUtilsCompositionScriptSource = readFileSync( - new URL( - ['../../../../../../genie/ci-scripts', 'cleanup-effect-utils-composition.sh'].join('/'), - import.meta.url, - ), - 'utf8', -) const netlifyTaskModuleSource = readFileSync( new URL( ['../../../../../../nix/devenv-modules/tasks/shared', 'netlify.nix'].join('/'), @@ -306,6 +291,23 @@ describe('ci workflow retry helpers', () => { expect(ciWorkflowSource).not.toContain('if [ ! -x "$__genie_ci_retry_script" ]') }) + it('captures ordinary CI task graphs as OpenTelemetry artifacts', () => { + expect(ciWorkflowSource).toContain('prepareCiOtelSpoolStep') + expect(ciWorkflowSource).toContain('ciOtelSpansArtifactStep') + expect(generatedCiWorkflowYamlSource).toContain('name: Prepare CI OpenTelemetry capture') + expect(generatedCiWorkflowYamlSource).toContain('OTEL_SPAN_SPOOL_DIR') + expect(generatedCiWorkflowYamlSource).toContain('name: Summarize CI OpenTelemetry spans') + expect(generatedCiWorkflowYamlSource).toContain('name: Upload CI OpenTelemetry spans') + const captureCount = generatedCiWorkflowYamlSource.match( + /name: Prepare CI OpenTelemetry capture/g, + )?.length + const uploadCount = generatedCiWorkflowYamlSource.match( + /name: Upload CI OpenTelemetry spans/g, + )?.length + expect(captureCount).toBeGreaterThan(0) + expect(uploadCount).toBe(captureCount) + }) + it('routes the devenv resolution step through the shared retry wrapper', () => { expect(validateNixStoreStepSource).toContain('withGcRaceRetry({') expect(validateNixStoreStepSource).toContain('label: `resolve devenv (${lockFile})`') @@ -1020,7 +1022,6 @@ describe('ci workflow standard job helpers', () => { downloadPreviousGitHubArtifactStep, githubTokenEnv, netlifyDeployStep, - prepareEffectUtilsCompositionStep, prSnapshotPackJob, standardCIEnv, vercelDeployJobs, @@ -1103,7 +1104,6 @@ describe('ci workflow standard job helpers', () => { .filter(({ step }) => step.env?.GITHUB_TOKEN !== expectedGitHubToken) .map(({ jobId, step }) => jobId + ': ' + step.name) const scriptBackedNixStepNames = [ - 'Prepare effect-utils composition', 'Resolve devenv', 'Bootstrap cold-proof (R32)', 'pnpm regression suite', @@ -1140,7 +1140,6 @@ describe('ci workflow standard job helpers', () => { standardEnv: standardCIEnv({ trustTier }), githubTokenEnv: githubTokenEnv(), nixStepEnv: cachixCliBuildStep.env, - compositionStepEnv: prepareEffectUtilsCompositionStep.env, devenvStepEnv: devenvTaskStep('Check', 'check:quick').env, ghStepEnv: downloadPreviousGitHubArtifactStep({ artifactName: 'baseline', @@ -1202,7 +1201,6 @@ describe('ci workflow standard job helpers', () => { standardEnv: expectedEnv, githubTokenEnv: expectedTokenEnv, nixStepEnv: expectedTokenEnv, - compositionStepEnv: expectedTokenEnv, devenvStepEnv: expectedTokenEnv, ghStepEnv: '${{ github.token }}', pnpmRegressionStepEnv: expectedTokenEnv, @@ -1477,356 +1475,36 @@ describe('ci workflow devenv perf helpers', () => { }) }) -describe('effect-utils CI composition workspace', () => { - const git = (cwd: string, ...args: string[]) => { - const result = spawnSync('git', args, { cwd, encoding: 'utf8' }) - if (result.status !== 0) { - throw new Error(`git ${args.join(' ')} failed: ${result.stderr}`) - } - return result.stdout.trim() - } - - const makeFixture = (platform: 'Linux' | 'macOS') => { - const root = mkdtempSync(join(tmpdir(), 'effect-utils-ci-composition-')) - const checkout = join(root, 'checkout with spaces') - const runnerTemp = join(root, `runner ${platform}`) - const fakeBin = join(root, 'fake-bin') - const mrOut = join(root, 'mr-out') - const envFile = join(root, 'github-env') - const nixLog = join(root, 'nix.log') - const mrLog = join(root, 'mr.log') - mkdirSync(checkout) - mkdirSync(runnerTemp) - mkdirSync(fakeBin) - mkdirSync(join(mrOut, 'bin'), { recursive: true }) - writeFileSync(envFile, '') - git(checkout, 'init', '--initial-branch=main') - const configuredIdentity = (field: 'user.email' | 'user.name', fallback: string) => - spawnSync('git', ['config', field], { - cwd: checkout, - encoding: 'utf8', - }).stdout.trim() || fallback - git( - checkout, - 'config', - 'user.email', - configuredIdentity('user.email', 'ci-fixture@example.invalid'), - ) - git(checkout, 'config', 'user.name', configuredIdentity('user.name', 'CI Fixture')) - writeFileSync(join(checkout, 'README'), 'fixture\n') - mkdirSync(join(checkout, 'genie/ci-scripts'), { recursive: true }) - writeFileSync( - join(checkout, 'genie/ci-scripts/prepare-effect-utils-composition.sh'), - prepareEffectUtilsCompositionScriptSource, - ) - writeFileSync( - join(checkout, 'genie/ci-scripts/cleanup-effect-utils-composition.sh'), - cleanupEffectUtilsCompositionScriptSource, - ) - chmodSync(join(checkout, 'genie/ci-scripts/prepare-effect-utils-composition.sh'), 0o755) - chmodSync(join(checkout, 'genie/ci-scripts/cleanup-effect-utils-composition.sh'), 0o755) - git( - checkout, - 'add', - 'README', - 'genie/ci-scripts/prepare-effect-utils-composition.sh', - 'genie/ci-scripts/cleanup-effect-utils-composition.sh', - ) - git(checkout, 'commit', '-m', 'fixture') - const sha = git(checkout, 'rev-parse', 'HEAD') - - writeFileSync( - join(fakeBin, 'nix'), - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - 'printf \'%s|%s\\n\' "$PWD" "$*" >> "$FAKE_NIX_LOG"', - 'if [ "$*" != "build --no-link --print-out-paths .#megarepo" ]; then exit 64; fi', - 'printf \'%s\\n\' "$FAKE_MR_OUT"', - ].join('\n'), - ) - writeFileSync( - join(mrOut, 'bin', 'mr'), - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - 'export AGENT_POLICY_BYPASS=1', - 'fake_root="$(cd "$(dirname "$0")/.." && pwd)"', - 'printf \'%s|%s|%s|%s\\n\' "$PWD" "$MEGAREPO_STORE" "${RUNNER_OS:-unset}" "$*" >> "$fake_root/../mr.log"', - 'if [ "$1" = "--cwd" ]; then', - ' workspace="$2"; shift 2', - ' test "$*" = "apply --worktree-mode tracking --lock-sync off --output ci"', - ' if [ -f "$workspace/.megarepo/composition-generation.json" ]; then exit 0; fi', - ' bare="$(git -C "$workspace" rev-parse --path-format=absolute --git-common-dir)"', - ' stage="${workspace}.member-stage"', - ' git --git-dir="$bare" worktree move "$workspace" "$stage"', - ' mkdir -p "$workspace/repos" "$workspace/.megarepo/bin"', - ' git --git-dir="$bare" worktree move "$stage" "$workspace/repos/effect-utils"', - ' printf \'{}\\n\' > "$workspace/.megarepo/composition-generation.json"', - ' printf \'[cells]\\n\' > "$workspace/.buckconfig"', - ' printf \'#!/usr/bin/env bash\\nexit 0\\n\' > "$workspace/.megarepo/bin/buck2"', - ' chmod +x "$workspace/.megarepo/bin/buck2"', - ' mkdir -p "$MEGAREPO_STORE/reference-effect"', - ' ln -s "$MEGAREPO_STORE/reference-effect" "$workspace/repos/effect"', - ' exit 0', - 'fi', - 'test "$1 $2 $3" = "store worktree new"', - 'repo="$4"; shift 4', - 'ref=; base=; porcelain=0', - 'while [ "$#" -gt 0 ]; do', - ' case "$1" in', - ' --ref) ref="$2"; shift 2 ;;', - ' --base) base="$2"; shift 2 ;;', - ' --porcelain) porcelain=1; shift ;;', - ' *) exit 64 ;;', - ' esac', - 'done', - 'test -n "$repo" && test -n "$ref" && test -n "$base" && test "$porcelain" -eq 1', - 'bare="$MEGAREPO_STORE/github.com/$repo/.bare"', - 'workspace="$MEGAREPO_STORE/github.com/$repo/refs/heads/$ref"', - 'git --git-dir="$bare" update-ref "refs/heads/$ref" "$base"', - 'mkdir -p "$(dirname "$workspace")"', - 'git --git-dir="$bare" worktree add "$workspace" "$ref" >/dev/null', - 'if [ -f "$fake_root/fail" ]; then exit 37; fi', - 'if [ "${FAKE_MR_OUTPUT_MEMBER_ROOT:-0}" = 1 ]; then', - ' printf \'%s\\n\' "$workspace/repos/effect-utils"', - 'else', - ' printf \'%s\\n\' "$workspace"', - 'fi', - ].join('\n'), - ) - chmodSync(join(fakeBin, 'nix'), 0o755) - chmodSync(join(mrOut, 'bin', 'mr'), 0o755) - - const env = { - ...process.env, - AGENT_POLICY_BYPASS: '1', - FAKE_MR_LOG: mrLog, - FAKE_MR_OUT: mrOut, - FAKE_NIX_LOG: nixLog, - GITHUB_ENV: envFile, - GITHUB_JOB: 'unit/job', - GITHUB_RUN_ATTEMPT: '2', - GITHUB_RUN_ID: '100', - GITHUB_WORKSPACE: checkout, - MEGAREPO_STORE: join(runnerTemp, 'megarepo-store/100/2/unit_job'), - EFFECT_UTILS_CI_ORIGIN_URL: checkout, - PATH: `${fakeBin}:${process.env.PATH ?? ''}`, - RUNNER_OS: platform, - RUNNER_TEMP: runnerTemp, - } - return { checkout, env, envFile, mrLog, mrOut, nixLog, root, runnerTemp, sha } - } - - const runComposition = async ( - fixture: ReturnType, - overrides: NodeJS.ProcessEnv = {}, - ) => { - if (overrides.FAKE_MR_FAIL === '1') writeFileSync(join(fixture.mrOut, 'fail'), '') - const { prepareEffectUtilsCompositionStep } = await import( - // oxlint-disable-next-line import/no-dynamic-require - new URL('../../../../../../genie/ci-workflow/setup.ts', import.meta.url).href - ) - return spawnSync('bash', ['-c', prepareEffectUtilsCompositionStep.run], { - cwd: fixture.root, - encoding: 'utf8', - env: { ...fixture.env, ...overrides }, - }) - } - - const cleanupComposition = async ( - fixture: ReturnType, - overrides: NodeJS.ProcessEnv = {}, - ) => { - const { cleanupEffectUtilsCompositionStep } = await import( - // oxlint-disable-next-line import/no-dynamic-require - new URL('../../../../../../genie/ci-workflow/setup.ts', import.meta.url).href - ) - return spawnSync('bash', ['-c', cleanupEffectUtilsCompositionStep.run], { - cwd: fixture.root, - encoding: 'utf8', - env: { ...fixture.env, ...overrides }, - }) - } - - it.each(['Linux', 'macOS'] as const)( - 'keeps checkout immutable and synthesizes the exact owned member on %s', - async (platform) => { - const fixture = makeFixture(platform) - try { - const first = await runComposition(fixture) - expect(first.status, first.stderr).toBe(0) - const branch = 'ci-100-2-unit_job' - const workspace = join( - fixture.runnerTemp, - 'megarepo-store/100/2/unit_job/github.com/overengineeringstudio/effect-utils/refs/heads', - branch, - ) - const member = join(workspace, 'repos/effect-utils') - expect(git(fixture.checkout, 'rev-parse', 'HEAD')).toBe(fixture.sha) - expect(git(fixture.checkout, 'status', '--porcelain=v1', '--untracked-files=all')).toBe('') - expect(git(member, 'rev-parse', 'HEAD')).toBe(fixture.sha) - expect(git(member, 'symbolic-ref', 'HEAD')).toBe(`refs/heads/${branch}`) - expect(git(member, 'merge-base', 'refs/remotes/origin/main', 'HEAD')).toBe(fixture.sha) - expect( - spawnSync('git', ['-C', workspace, 'rev-parse', '--is-inside-work-tree']).status, - ).not.toBe(0) - expect(readFileSync(fixture.nixLog, 'utf8')).toBe( - `${fixture.checkout}|build --no-link --print-out-paths .#megarepo\n`, - ) - // The composition step runs `mr` under `env -i`, which drops PWD, so the - // child shell reports the resolved working directory while every other - // field is the path the runner handed in. Compare the first field in the - // same resolved form; on a filesystem with no symlink above the fixture - // this is the identity. - expect(readFileSync(fixture.mrLog, 'utf8')).toContain( - `${realpathSync(dirname(workspace))}|${join(fixture.runnerTemp, 'megarepo-store/100/2/unit_job')}|unset|store worktree new overengineeringstudio/effect-utils --ref ${branch} --base ${fixture.sha} --porcelain`, - ) - expect(readFileSync(fixture.mrLog, 'utf8')).toContain( - `${realpathSync(dirname(workspace))}|${join(fixture.runnerTemp, 'megarepo-store/100/2/unit_job')}|unset|--cwd ${workspace} apply --worktree-mode tracking --lock-sync off --output ci`, - ) - expect(readFileSync(fixture.envFile, 'utf8')).toContain( - `EFFECT_UTILS_MEMBER_ROOT=${member}\n`, - ) - - const repeated = await runComposition(fixture) - expect(repeated.status, repeated.stderr).toBe(0) - expect(git(member, 'rev-parse', 'HEAD')).toBe(fixture.sha) - - const secondEnv = join(fixture.root, 'github-env-second') - writeFileSync(secondEnv, '') - const secondStore = join(fixture.runnerTemp, 'megarepo-store/100/2/other-job') - const second = await runComposition(fixture, { - GITHUB_ENV: secondEnv, - GITHUB_JOB: 'other-job', - MEGAREPO_STORE: secondStore, - }) - expect(second.status, second.stderr).toBe(0) - const secondMember = join( - fixture.runnerTemp, - 'megarepo-store/100/2/other-job/github.com/overengineeringstudio/effect-utils/refs/heads/ci-100-2-other-job/repos/effect-utils', - ) - expect(git(secondMember, 'rev-parse', 'HEAD')).toBe(fixture.sha) - const secondCleanup = await cleanupComposition(fixture, { - GITHUB_JOB: 'other-job', - MEGAREPO_STORE: secondStore, - }) - expect(secondCleanup.status, secondCleanup.stderr).toBe(0) - const cleanup = await cleanupComposition(fixture) - expect(cleanup.status, cleanup.stderr).toBe(0) - await expect(cleanupComposition(fixture)).resolves.toMatchObject({ status: 0 }) - } finally { - rmSync(fixture.root, { force: true, recursive: true, maxRetries: 10, retryDelay: 20 }) - } - }, - 20_000, - ) - - it('accepts a member-root porcelain result from the release CLI', async () => { - const fixture = makeFixture('Linux') - try { - const result = await runComposition(fixture, { FAKE_MR_OUTPUT_MEMBER_ROOT: '1' }) - expect(result.status, result.stderr).toBe(0) - const member = join( - fixture.runnerTemp, - 'megarepo-store/100/2/unit_job/github.com/overengineeringstudio/effect-utils/refs/heads/ci-100-2-unit_job/repos/effect-utils', - ) - expect(git(member, 'rev-parse', 'HEAD')).toBe(fixture.sha) - await expect(cleanupComposition(fixture)).resolves.toMatchObject({ status: 0 }) - } finally { - rmSync(fixture.root, { force: true, recursive: true, maxRetries: 10, retryDelay: 20 }) - } - }, 20_000) - - it('cleans a direct-final-path worktree after generation fails', async () => { - const fixture = makeFixture('Linux') - try { - const result = await runComposition(fixture, { FAKE_MR_FAIL: '1' }) - expect(result.status).toBe(37) - expect(readFileSync(fixture.envFile, 'utf8')).not.toContain('EFFECT_UTILS_MEMBER_ROOT') - expect(git(fixture.checkout, 'rev-parse', 'HEAD')).toBe(fixture.sha) - expect(git(fixture.checkout, 'status', '--porcelain=v1', '--untracked-files=all')).toBe('') - - const store = fixture.env.MEGAREPO_STORE! - const workspace = join( - store, - 'github.com/overengineeringstudio/effect-utils/refs/heads/ci-100-2-unit_job', - ) - expect(git(workspace, 'symbolic-ref', 'HEAD')).toBe('refs/heads/ci-100-2-unit_job') - - const cleanup = await cleanupComposition(fixture) - expect(cleanup.status, cleanup.stderr).toBe(0) - expect(existsSync(store)).toBe(false) - } finally { - rmSync(fixture.root, { force: true, recursive: true, maxRetries: 10, retryDelay: 20 }) - } - }, 20_000) - - it('refuses cleanup when the direct owned checkout changed branches', async () => { - const fixture = makeFixture('Linux') - try { - const result = await runComposition(fixture, { FAKE_MR_FAIL: '1' }) - expect(result.status).toBe(37) - const store = fixture.env.MEGAREPO_STORE! - const member = join( - store, - 'github.com/overengineeringstudio/effect-utils/refs/heads/ci-100-2-unit_job', - ) - git(member, 'switch', '-c', 'unrelated') +describe('effect-utils standalone CI root', () => { + it('runs every workflow lane from the actions checkout without composition plumbing', () => { + expect(generatedCiWorkflowYamlSource).not.toContain('Prepare effect-utils composition') + expect(generatedCiWorkflowYamlSource).not.toContain('Cleanup effect-utils composition') + expect(generatedCiWorkflowYamlSource).not.toContain('prepare-effect-utils-composition.sh') + expect(generatedCiWorkflowYamlSource).not.toContain('cleanup-effect-utils-composition.sh') + expect(generatedCiWorkflowYamlSource).not.toContain('EFFECT_UTILS_MEMBER_ROOT') + expect(generatedCiWorkflowYamlSource).not.toContain('EFFECT_UTILS_WORKSPACE_ROOT') + expect(generatedCiWorkflowYamlSource).not.toContain('.megarepo/bin/buck2') + expect(generatedCiWorkflowYamlSource).not.toMatch(/^\s+(?:buck2|\.\/[^ ]*buck2)\s/m) + }) - const cleanup = await cleanupComposition(fixture) - expect(cleanup.status).not.toBe(0) - expect(existsSync(member)).toBe(true) - expect(git(member, 'symbolic-ref', 'HEAD')).toBe('refs/heads/unrelated') - } finally { - rmSync(fixture.root, { force: true, recursive: true, maxRetries: 10, retryDelay: 20 }) - } - }, 20_000) + it('keeps the standalone remote-cache proof and cold-GC lane explicit', () => { + const cacheProof = + generatedCiWorkflowYamlSource + .split(' trusted-buck2-remote-cache-proof:\n')[1] + ?.split(/^ [a-z]/m)[0] ?? '' + expect(cacheProof).toContain('Context B is a second standalone root') + expect(cacheProof).toContain('buck="${BUCK2_BIN:?BUCK2_BIN not set}"') + expect(cacheProof).toContain('source_root="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE not set}"') - it('orders every migrated job after composition and keeps the checkout exemptions explicit', () => { - const jobsYaml = generatedCiWorkflowYamlSource.split('\njobs:\n')[1] ?? '' - const blocks = new Map( - Array.from( - jobsYaml.matchAll(/^ ([a-zA-Z0-9_-]+):\n([\s\S]*?)(?=^ [a-zA-Z0-9_-]+:\n|$(?![\s\S]))/gm), - ([, name, body]) => [name!, body!] as const, - ), - ) - const exemptions: Record = { - 'default-ref-policy': true, - 'pr-reviews-resolved': true, - 'source-shape': true, - 'ci-measurements-report': true, - 'notify-alignment': true, - } - expect( - [...blocks.keys()].filter( - (name) => blocks.get(name)?.includes('Prepare effect-utils composition') !== true, - ), - ).toEqual(Object.keys(exemptions)) - for (const [name, block] of blocks) { - const taskIndex = block.indexOf('tasks run ') - if (taskIndex < 0) continue - expect(exemptions[name] === true, name).toBe(false) - const compositionIndex = block.indexOf('Prepare effect-utils composition') - expect(compositionIndex, name).toBeGreaterThanOrEqual(0) - expect(compositionIndex, name).toBeLessThan(taskIndex) - expect(block.indexOf('Cleanup effect-utils composition'), name).toBeGreaterThan(taskIndex) - } - expect(generatedCiWorkflowYamlSource).not.toMatch(/^\s+(?:buck2|\.\/[^ ]*buck2)\s/m) - expect(blocks.has('nix-fod-check')).toBe(false) - expect(blocks.has('nix-check')).toBe(false) - expect(generatedCiWorkflowYamlSource).not.toContain('Evict cached pnpm deps for oxlint-npm') - expect(generatedCiWorkflowYamlSource).not.toContain('.#oxc-config-plugin-pnpm-deps') + const coldGc = + generatedCiWorkflowYamlSource.split(' test-megarepo-cold-gc:\n')[1]?.split(/^ [a-z]/m)[0] ?? + '' + expect(coldGc).toContain('tasks run test:megarepo-cold-gc') }) - it('keeps pull-request source execution credentialless and read-only', () => { + it('keeps pull-request execution credentialless and trusted writes main-only', () => { expect(ciWorkflowSource).toContain("'persist-credentials': false") expect(generatedCiWorkflowYamlSource).toContain('permissions:\n contents: read') - const typecheck = generatedCiWorkflowYamlSource.split(' typecheck:\n')[1] ?? '' - expect(typecheck.indexOf('Prepare effect-utils composition')).toBeLessThan( - typecheck.indexOf('Enable Cachix cache'), - ) - expect(typecheck).toContain("github.ref == 'refs/heads/main'") for (const job of [ 'ci-measurements-report', 'test-integration-notion', @@ -1837,28 +1515,27 @@ describe('effect-utils CI composition workspace', () => { generatedCiWorkflowYamlSource.split(` ${job}:\n`)[1]?.split(/^ [a-z]/m)[0] ?? '' expect(block, job).toContain("github.ref == 'refs/heads/main'") } - expect(prepareEffectUtilsCompositionScriptSource).toContain('env -i \\') - expect(prepareEffectUtilsCompositionScriptSource).not.toContain('GITHUB_TOKEN') - expect(prepareEffectUtilsCompositionScriptSource).toContain( - "'+refs/heads/main:refs/remotes/origin/main'", - ) - const trustedRef = (event: string, ref: string) => - ref === 'refs/heads/main' && (event === 'push' || event === 'workflow_dispatch') - expect(trustedRef('workflow_dispatch', 'refs/heads/feature')).toBe(false) - expect(trustedRef('workflow_dispatch', 'refs/heads/main')).toBe(true) - expect(trustedRef('pull_request', 'refs/heads/main')).toBe(false) + for (const { event, ref, expected } of [ + { event: 'workflow_dispatch', ref: 'refs/heads/feature', expected: false }, + { event: 'workflow_dispatch', ref: 'refs/heads/main', expected: true }, + { event: 'push', ref: 'refs/heads/main', expected: true }, + { event: 'pull_request', ref: 'refs/heads/main', expected: false }, + ] as const) { + const actual = + ref === 'refs/heads/main' && (event === 'push' || event === 'workflow_dispatch') + expect(actual).toBe(expected) + } }) it('keeps the Nix cache stable without projecting an ambient pnpm store', () => { expect(generatedCiWorkflowYamlSource).not.toContain( '${{ runner.temp }}/composition-state/pnpm-store-pure-v1', ) - expect(prepareEffectUtilsCompositionScriptSource).toContain( - 'export XDG_CACHE_HOME="${RUNNER_TEMP:?RUNNER_TEMP not set}/composition-state/nix-cache"', - ) expect(generatedCiWorkflowYamlSource).not.toContain( '${{ runner.temp }}/composition-state/${{ github.run_id }}', ) expect(buckToolchainsSource).not.toContain('store_dir =') + expect(generatedCiWorkflowYamlSource).not.toContain('Evict cached pnpm deps for oxlint-npm') + expect(generatedCiWorkflowYamlSource).not.toContain('.#oxc-config-plugin-pnpm-deps') }) }) diff --git a/packages/@overeng/genie/src/runtime/github-workflow/github-workflow.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/github-workflow.unit.test.ts index 8b0622e955..b95cbba845 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/github-workflow.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/github-workflow.unit.test.ts @@ -660,7 +660,9 @@ describe('GitHub expression validation', () => { }) }) -describe.runIf(hasActionlint)('actionlint integration', () => { +const describeActionlint = hasActionlint === true ? describe : describe.skip + +describeActionlint('actionlint integration', () => { it('passes a clean workflow', async () => { const issues = await getFullValidationIssues({ name: 'CI', diff --git a/packages/@overeng/genie/src/runtime/node/bootstrap-closure-check-cli.ts b/packages/@overeng/genie/src/runtime/node/bootstrap-closure-check-cli.ts index b407ce5c9f..47cb2b6cbf 100644 --- a/packages/@overeng/genie/src/runtime/node/bootstrap-closure-check-cli.ts +++ b/packages/@overeng/genie/src/runtime/node/bootstrap-closure-check-cli.ts @@ -2,12 +2,18 @@ import { existsSync, readdirSync, readFileSync, realpathSync } from 'node:fs' import path from 'node:path' import { parseGeneratorPhase } from '../../core/phase.ts' +import type { BootstrapClosureViolation } from './bootstrap-closure.ts' import { checkBootstrapClosure, formatViolationChain } from './bootstrap-closure.ts' const usage = `Usage: genie-bootstrap-closure-check [--root ] + genie-bootstrap-closure-check [--root ] \\ + --editor-view-root-package-path \\ + --editor-view-package-paths -Checks source-tree // @genie-bootstrap .genie.ts files for runtime-only package imports.` +Checks source-tree // @genie-bootstrap .genie.ts files for runtime-only package imports. +With the editor-view arguments, checks every generator's runtime import closure against the +workspace package views published before genie:check.` const ignoredDiscoveryDirs = new Set([ '.devenv', @@ -22,15 +28,150 @@ const ignoredDiscoveryDirs = new Set([ 'target', ]) +type WorkspacePackage = { + readonly dependencyNames: readonly string[] + readonly name: string + readonly path: string +} + +type EditorViewClosureViolation = { + readonly packageName: string + readonly packagePath: string + readonly violation: BootstrapClosureViolation +} + +const isObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && Array.isArray(value) === false + +const dependencyNamesOf = (manifest: Record): readonly string[] => + ['dependencies', 'devDependencies', 'peerDependencies'].flatMap((field) => { + const dependencies = manifest[field] + return isObject(dependencies) === true ? Object.keys(dependencies) : [] + }) + +const decodePackagePaths = (serialized: string): readonly string[] => { + const decoded: unknown = JSON.parse(serialized) + if ( + Array.isArray(decoded) === false || + decoded.length === 0 || + decoded.some((entry) => typeof entry !== 'string') === true + ) { + throw new Error('--editor-view-package-paths must be a non-empty JSON array of strings') + } + return decoded +} + +const packageNameFromSpecifier = (specifier: string): string => { + const segments = specifier.split('/') + return specifier.startsWith('@') === true ? segments.slice(0, 2).join('/') : segments[0]! +} + +const readWorkspacePackages = (repoRoot: string): readonly WorkspacePackage[] => { + const rootManifest: unknown = JSON.parse( + readFileSync(path.join(repoRoot, 'package.json'), 'utf8'), + ) + if (isObject(rootManifest) === false || Array.isArray(rootManifest.workspaces) === false) { + throw new Error('root package.json must declare a workspaces array') + } + + return rootManifest.workspaces.map((packagePath) => { + if (typeof packagePath !== 'string') { + throw new Error('root package.json workspaces must contain only package paths') + } + const manifest: unknown = JSON.parse( + readFileSync(path.join(repoRoot, packagePath, 'package.json'), 'utf8'), + ) + if (isObject(manifest) === false || typeof manifest.name !== 'string') { + throw new Error(`${packagePath}/package.json must declare a package name`) + } + return { + dependencyNames: dependencyNamesOf(manifest), + name: manifest.name, + path: packagePath, + } + }) +} + +/** Find unresolved first-party runtime imports in the editor views published for Genie. */ +export const findEditorViewClosureViolations = ({ + violations, + workspacePackages, + publishedPackagePaths, + repoRoot, + rootPackagePath, +}: { + readonly violations: readonly BootstrapClosureViolation[] + readonly workspacePackages: readonly WorkspacePackage[] + readonly publishedPackagePaths: readonly string[] + readonly repoRoot: string + readonly rootPackagePath: string +}): readonly EditorViewClosureViolation[] => { + const packageByName = new Map( + workspacePackages.map((workspacePackage) => [workspacePackage.name, workspacePackage]), + ) + const packageByPath = new Map( + workspacePackages.map((workspacePackage) => [workspacePackage.path, workspacePackage]), + ) + const publishedWorkspacePaths = new Set() + for (const packagePath of publishedPackagePaths) { + const workspacePath = packagePath === '.' ? rootPackagePath : packagePath + if (packageByPath.has(workspacePath) === false) { + throw new Error( + `--editor-view-package-paths names an unknown workspace package: ${workspacePath}`, + ) + } + publishedWorkspacePaths.add(workspacePath) + } + const rootPackage = packageByPath.get(rootPackagePath) + if (rootPackage === undefined) { + throw new Error( + `--editor-view-root-package-path names an unknown workspace package: ${rootPackagePath}`, + ) + } + const rootDependencyNames = new Set(rootPackage.dependencyNames) + + return violations.flatMap((violation) => { + const specifierPackageName = packageNameFromSpecifier(violation.specifier) + if (rootDependencyNames.has(specifierPackageName) === true) return [] + const importer = violation.chain[violation.chain.length - 1]! + const importerPackage = workspacePackages.find((workspacePackage) => { + const relative = path.relative(path.join(repoRoot, workspacePackage.path), importer) + return ( + relative === '' || + (relative.startsWith('..') === false && path.isAbsolute(relative) === false) + ) + }) + const requiredPackage = + importerPackage ?? packageByName.get(packageNameFromSpecifier(violation.specifier)) + return requiredPackage === undefined || + publishedWorkspacePaths.has(requiredPackage.path) === true + ? [] + : [ + { + packageName: requiredPackage.name, + packagePath: requiredPackage.path, + violation, + }, + ] + }) +} + const parseArgs = ({ argv, defaultRepoRoot, }: { argv: readonly string[] defaultRepoRoot: string -}): { readonly repoRoot: string; readonly help: boolean } => { +}): { + readonly repoRoot: string + readonly help: boolean + readonly editorViewPackagePaths: readonly string[] | undefined + readonly editorViewRootPackagePath: string | undefined +} => { let repoRoot = defaultRepoRoot let help = false + let editorViewPackagePaths: readonly string[] | undefined + let editorViewRootPackagePath: string | undefined for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]! @@ -47,14 +188,39 @@ const parseArgs = ({ index += 1 continue } + if (arg === '--editor-view-package-paths') { + const value = argv[index + 1] + if (value === undefined || value.length === 0) { + throw new Error('--editor-view-package-paths requires a non-empty JSON array') + } + editorViewPackagePaths = decodePackagePaths(value) + index += 1 + continue + } + if (arg === '--editor-view-root-package-path') { + const value = argv[index + 1] + if (value === undefined || value.length === 0) { + throw new Error('--editor-view-root-package-path requires a non-empty workspace path') + } + editorViewRootPackagePath = value + index += 1 + continue + } throw new Error(`unknown argument: ${arg}`) } + if ((editorViewPackagePaths === undefined) !== (editorViewRootPackagePath === undefined)) { + throw new Error( + '--editor-view-package-paths and --editor-view-root-package-path must be provided together', + ) + } // The walk reports every path as its on-disk identity, so the root the diagnostics are made relative // to has to be that same identity — otherwise a symlinked checkout renders every chain as `../..`. return { repoRoot: existsSync(repoRoot) === true ? realpathSync.native(repoRoot) : repoRoot, help, + editorViewPackagePaths, + editorViewRootPackagePath, } } @@ -93,13 +259,51 @@ export const bootstrapClosureCheckMain = async ({ defaultRepoRoot: string }): Promise => { try { - const { repoRoot, help } = parseArgs({ argv, defaultRepoRoot }) + const { repoRoot, help, editorViewPackagePaths, editorViewRootPackagePath } = parseArgs({ + argv, + defaultRepoRoot, + }) if (help === true) { console.log(usage) return } const allGenieFiles = discoverGenieFiles(repoRoot) + if (editorViewPackagePaths !== undefined && editorViewRootPackagePath !== undefined) { + const result = await checkBootstrapClosure({ + genieFiles: allGenieFiles, + reportAllViolations: true, + }) + const closureViolations = findEditorViewClosureViolations({ + violations: result.violations, + workspacePackages: readWorkspacePackages(repoRoot), + publishedPackagePaths: editorViewPackagePaths, + repoRoot, + rootPackagePath: editorViewRootPackagePath, + }) + if (closureViolations.length > 0) { + console.error( + `✗ editor-view-closure: ${closureViolations.length} generator import(s) require an unpublished workspace package view:\n`, + ) + for (const closureViolation of closureViolations) { + console.error( + ` ${formatViolationChain({ violation: closureViolation.violation, repoRoot })}\n` + + ` missing editor view: ${closureViolation.packagePath} (${closureViolation.packageName})\n`, + ) + } + console.error( + 'Add every listed package path to the editorBootstrapPackagePaths declaration so ' + + 'buck2:editor:bootstrap publishes the complete generator import closure before genie:check.', + ) + process.exit(1) + } + console.log( + `editor-view-closure: OK — ${result.checkedSources.length} .genie.ts runtime import closures ` + + `are covered by ${editorViewPackagePaths.length} bootstrap editor view(s)`, + ) + return + } + const bootstrapFiles = allGenieFiles.filter( (file) => parseGeneratorPhase(readFileSync(file, 'utf8')) === 'bootstrap', ) diff --git a/packages/@overeng/genie/src/runtime/node/bootstrap-closure.ts b/packages/@overeng/genie/src/runtime/node/bootstrap-closure.ts index bd5b0f755f..192f47e914 100644 --- a/packages/@overeng/genie/src/runtime/node/bootstrap-closure.ts +++ b/packages/@overeng/genie/src/runtime/node/bootstrap-closure.ts @@ -300,14 +300,20 @@ const resolveFollowableSpecifier = async ({ } /** - * Walk the transitive runtime import closure of each `.genie.ts` source and report those that reach a - * runtime-only package, with the shortest importer chain to the offending edge. + * Walk the transitive runtime import closure of each `.genie.ts` source and report runtime-only + * package boundaries with their shortest importer chains. + * + * Bootstrap policy needs only the first boundary per source. Editor-bootstrap closure validation + * requests every boundary so it can compare first-party package imports with the published view set. */ export const checkBootstrapClosure = async ({ genieFiles, + reportAllViolations = false, }: { /** Absolute paths of the `.genie.ts` sources to check. */ genieFiles: readonly string[] + /** Report every bare runtime package boundary instead of only the first one per source. */ + reportAllViolations?: boolean }): Promise => { /** Per-file analysis, memoized globally — the runtime import graph is identical across all roots. */ type FileEdges = { @@ -360,15 +366,17 @@ export const checkBootstrapClosure = async ({ return edges } - /** BFS from a root; returns the shortest chain to the first runtime-only specifier, or undefined. */ - const findViolation = async ({ + /** BFS from a root; returns shortest chains to the applicable runtime-only specifier boundaries. */ + const findViolations = async ({ root, session, }: { root: string session: TsFileAnalysisSession - }): Promise => { + }): Promise => { + const found: BootstrapClosureViolation[] = [] const seen = new Set() + const seenSpecifiers = new Set() const queue: (readonly string[])[] = [[root]] let nextIndex = 0 while (nextIndex < queue.length) { @@ -381,14 +389,17 @@ export const checkBootstrapClosure = async ({ // Graph discovery is intentionally serial because the analysis session advances one mutable snapshot. // eslint-disable-next-line no-await-in-loop const { violationSpecifiers, followTargets } = await edgesOf({ file: current, session }) - if (violationSpecifiers.length > 0) { - return { source: root, specifier: violationSpecifiers[0]!, chain } + for (const specifier of violationSpecifiers) { + if (seenSpecifiers.has(specifier) === true) continue + found.push({ source: root, specifier, chain }) + seenSpecifiers.add(specifier) + if (reportAllViolations === false) return found } for (const target of followTargets) { if (seen.has(target) === false) queue.push([...chain, target]) } } - return undefined + return found } // Every path this walk reports — roots included — is the file's on-disk identity, so a chain link is @@ -401,8 +412,7 @@ export const checkBootstrapClosure = async ({ for (const root of sortedGenieFiles) { // Roots share the same mutable analysis snapshot and graph cache, so preserve source order. // eslint-disable-next-line no-await-in-loop - const violation = await findViolation({ root, session }) - if (violation !== undefined) found.push(violation) + found.push(...(await findViolations({ root, session }))) } return found }, diff --git a/packages/@overeng/genie/src/runtime/node/bootstrap-closure.unit.test.ts b/packages/@overeng/genie/src/runtime/node/bootstrap-closure.unit.test.ts index c4647fa3c2..5dd8c47422 100644 --- a/packages/@overeng/genie/src/runtime/node/bootstrap-closure.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/node/bootstrap-closure.unit.test.ts @@ -13,7 +13,10 @@ import path from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { discoverGenieFiles } from './bootstrap-closure-check-cli.ts' +import { + discoverGenieFiles, + findEditorViewClosureViolations, +} from './bootstrap-closure-check-cli.ts' import { canonicalResolvedPath, canonicalSegment, @@ -208,6 +211,22 @@ describe('checkBootstrapClosure', () => { expect(violations[0]!.chain).toEqual([source]) }) + it('can report every bare runtime boundary for editor-view closure validation', async () => { + const dir = makeDir() + const source = write( + dir, + 'all-boundaries.genie.ts', + `import 'effect'\nimport '@scope/runtime'\nexport const ok = true`, + ) + + const { violations } = await checkBootstrapClosure({ + genieFiles: [source], + reportAllViolations: true, + }) + + expect(violations.map(({ specifier }) => specifier)).toEqual(['effect', '@scope/runtime']) + }) + it('does NOT flag node builtins (bare `crypto` and `node:`-prefixed)', async () => { const dir = makeDir() const source = write( @@ -348,6 +367,68 @@ describe('canonicalResolvedPath', () => { }) }) +describe('findEditorViewClosureViolations', () => { + it('names the importing workspace package whose editor view is undeclared', () => { + const violation = { + source: '/repo/source.genie.ts', + specifier: '@overeng/content-address/schema', + chain: ['/repo/source.genie.ts', '/repo/packages/@overeng/otel-contract/src/profile-link.ts'], + } + const rootCoveredViolation = { + source: '/repo/source.genie.ts', + specifier: '@overeng/otel-contract/registry', + chain: ['/repo/source.genie.ts', '/repo/packages/@overeng/ci-tools/src/contract.ts'], + } + const workspacePackages = [ + { + dependencyNames: ['@overeng/otel-contract'], + name: '@overeng/genie', + path: 'packages/@overeng/genie', + }, + { + dependencyNames: ['@overeng/content-address'], + name: '@overeng/otel-contract', + path: 'packages/@overeng/otel-contract', + }, + { + dependencyNames: [], + name: '@overeng/content-address', + path: 'packages/@overeng/content-address', + }, + { + dependencyNames: ['@overeng/otel-contract'], + name: '@overeng/ci-tools', + path: 'packages/@overeng/ci-tools', + }, + ] + + expect( + findEditorViewClosureViolations({ + violations: [violation, rootCoveredViolation], + workspacePackages, + publishedPackagePaths: ['.'], + repoRoot: '/repo', + rootPackagePath: 'packages/@overeng/genie', + }), + ).toEqual([ + { + packageName: '@overeng/otel-contract', + packagePath: 'packages/@overeng/otel-contract', + violation, + }, + ]) + expect( + findEditorViewClosureViolations({ + violations: [violation, rootCoveredViolation], + workspacePackages, + publishedPackagePaths: ['.', 'packages/@overeng/otel-contract'], + repoRoot: '/repo', + rootPackagePath: 'packages/@overeng/genie', + }), + ).toEqual([]) + }) +}) + // The exact-hit branch cannot be staged on a case-insensitive filesystem, so it is proven against the // pure name-selection rule instead — identical on every host. describe('canonicalSegment', () => { diff --git a/packages/@overeng/megarepo/src/cli/commands/composition.integration.test.ts b/packages/@overeng/megarepo/src/cli/commands/composition.integration.test.ts index 20a899a856..d98a06d731 100644 --- a/packages/@overeng/megarepo/src/cli/commands/composition.integration.test.ts +++ b/packages/@overeng/megarepo/src/cli/commands/composition.integration.test.ts @@ -199,6 +199,49 @@ describe('routine composition apply is shape-preserving', () => { }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), ) + it.effect('recognizes only a tracked standalone root without a repos mount', () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem + const fixture = yield* makeLegacyWorkspace + for (const marker of ['.buckroot', '.buckconfig']) { + yield* fs.writeFileString( + EffectPath.unsafe.absoluteFile(NodePath.join(fixture.workspaceRoot, marker)), + '', + ) + } + + const untrackedFailure = yield* preflightCompositionCommand({ + workspaceRoot: EffectPath.unsafe.absoluteDir(`${fixture.workspaceRoot}/`), + compositionEnabled: true, + }).pipe(Effect.flip) + expect(untrackedFailure.reason).toBe('RecreateRequired') + + yield* fixture.git(fixture.workspaceRoot, 'add', '.buckroot', '.buckconfig') + yield* fixture.git( + fixture.workspaceRoot, + 'commit', + '--no-gpg-sign', + '--no-verify', + '-m', + 'track standalone markers', + ) + const identity = yield* preflightCompositionCommand({ + workspaceRoot: EffectPath.unsafe.absoluteDir(`${fixture.workspaceRoot}/`), + compositionEnabled: true, + }) + expect(identity).toBeUndefined() + + yield* fs.makeDirectory( + EffectPath.unsafe.absoluteDir(`${NodePath.join(fixture.workspaceRoot, 'repos')}/`), + ) + const mountedFailure = yield* preflightCompositionCommand({ + workspaceRoot: EffectPath.unsafe.absoluteDir(`${fixture.workspaceRoot}/`), + compositionEnabled: true, + }).pipe(Effect.flip) + expect(mountedFailure.reason).toBe('RecreateRequired') + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ) + it.effect('does not infer composition from an ordinary symlinked member checkout', () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem diff --git a/packages/@overeng/megarepo/src/cli/commands/composition.ts b/packages/@overeng/megarepo/src/cli/commands/composition.ts index 9b95753dcf..11a04c3534 100644 --- a/packages/@overeng/megarepo/src/cli/commands/composition.ts +++ b/packages/@overeng/megarepo/src/cli/commands/composition.ts @@ -5,6 +5,7 @@ import { promisify } from 'node:util' import { Effect, Option, Schema } from 'effect' import * as FileSystem from 'effect/FileSystem' +import type { PlatformError } from 'effect/PlatformError' import type { ChildProcessSpawner } from 'effect/unstable/process/ChildProcessSpawner' import { EffectPath, type AbsoluteDirPath } from '@overeng/effect-path' @@ -186,6 +187,42 @@ export const loadOwnedIdentity = ({ } }).pipe(Effect.mapError(preserveCompositionError)) +const isTrackedStandaloneRoot = ({ + fs, + root, +}: { + readonly fs: FileSystem.FileSystem + readonly root: string +}): Effect.Effect => + Effect.gen(function* () { + const [hasBuckroot, hasBuckconfig, hasRepos] = yield* Effect.all( + [ + fs.exists(EffectPath.unsafe.absoluteFile(NodePath.join(root, '.buckroot'))), + fs.exists(EffectPath.unsafe.absoluteFile(NodePath.join(root, '.buckconfig'))), + fs.exists(EffectPath.unsafe.absoluteDir(`${NodePath.join(root, 'repos')}/`)), + ], + { concurrency: 'unbounded' }, + ) + if (hasBuckroot === false || hasBuckconfig === false || hasRepos === true) return false + + const topLevel = yield* Git.runCommand({ + cwd: root, + args: ['rev-parse', '--show-toplevel'], + }).pipe(Effect.result) + if ( + topLevel._tag === 'Failure' || + NodePath.resolve(topLevel.success) !== NodePath.resolve(root) + ) { + return false + } + + const trackedMarkers = yield* Git.runCommand({ + cwd: root, + args: ['ls-files', '--error-unmatch', '--', '.buckroot', '.buckconfig'], + }).pipe(Effect.result) + return trackedMarkers._tag === 'Success' + }) + /** * Detect a direct registered W independently of P's root config, then validate that the config * still names that exact composed identity. Ordinary Git roots are never inferred as composed. @@ -206,6 +243,7 @@ export const preflightCompositionCommand = ({ const root = workspaceRoot.replace(/\/+$/u, '') const rootGit = EffectPath.unsafe.absoluteFile(NodePath.join(root, '.git')) if ((yield* fs.exists(rootGit)) === true) { + if ((yield* isTrackedStandaloneRoot({ fs, root })) === true) return undefined return compositionEnabled === true ? yield* loadOwnedIdentity({ workspaceRoot }) : undefined } diff --git a/packages/@overeng/megarepo/src/cli/store.integration.test.ts b/packages/@overeng/megarepo/src/cli/store.integration.test.ts index 16d3197d68..ffdab6e71d 100644 --- a/packages/@overeng/megarepo/src/cli/store.integration.test.ts +++ b/packages/@overeng/megarepo/src/cli/store.integration.test.ts @@ -33,7 +33,6 @@ import { createWorkspaceWithLock, getWorktreeCommit, } from '../test-utils/store-setup.ts' -import { Cwd } from './context.ts' import { mrCommand } from './mod.ts' const StoreGcJsonOutput = Schema.Struct({ @@ -89,12 +88,11 @@ const runMrCommand = ({ }), ) - const argv = [...command] - const exit = yield* Cli.Command.runWith(mrCommand, { version: 'test' })(argv).pipe( - Effect.provideService(Cwd, cwd), - Effect.provide(consoleLayer), - Effect.exit, - ) + const exit = yield* Cli.Command.runWith(mrCommand, { version: 'test' })([ + '--cwd', + cwd, + ...command, + ]).pipe(Effect.provide(consoleLayer), Effect.exit) void previousEnv return { diff --git a/packages/@overeng/megarepo/src/composition/apply/composition-apply.integration.test.ts b/packages/@overeng/megarepo/src/composition/apply/composition-apply.integration.test.ts index 5382e89bae..b9f6686fdc 100644 --- a/packages/@overeng/megarepo/src/composition/apply/composition-apply.integration.test.ts +++ b/packages/@overeng/megarepo/src/composition/apply/composition-apply.integration.test.ts @@ -691,7 +691,6 @@ describe('composition apply integration', () => { await value.cleanup() } }) - it('forwards an explicit cache override unchanged to root planning and publication', async () => { const cacheSections: NonNullable = [ { diff --git a/packages/@overeng/megarepo/src/composition/capabilities/capability-projection.ts b/packages/@overeng/megarepo/src/composition/capabilities/capability-projection.ts index 60c51a693b..4e46e99905 100644 --- a/packages/@overeng/megarepo/src/composition/capabilities/capability-projection.ts +++ b/packages/@overeng/megarepo/src/composition/capabilities/capability-projection.ts @@ -6,9 +6,11 @@ import process from 'node:process' import type { BuckMemberCapability } from '../../buck2-manifest.ts' import type { ResolvedCompositionCapability } from './composition-capability-resolver-schema.ts' -type CapabilityProjectionPlatform = 'aarch64-linux' | 'aarch64-macos' | 'x86_64-linux' +/** Supported host tuples for materialized capability projections. */ +export type CapabilityProjectionPlatform = 'aarch64-linux' | 'aarch64-macos' | 'x86_64-linux' -type CapabilityProjectionManifest = { +/** Immutable manifest describing one exact projected capability realization. */ +export type CapabilityProjectionManifest = { readonly closureIdentity: string readonly closureStorePaths: readonly string[] readonly contentDigest: string @@ -20,7 +22,8 @@ type CapabilityProjectionManifest = { readonly toolId: string } -const makeCapabilityProjectionManifest = ({ +/** Converts a resolved capability into its portable projection manifest. */ +export const makeCapabilityProjectionManifest = ({ platform, resolved, }: { @@ -38,15 +41,18 @@ const makeCapabilityProjectionManifest = ({ toolId: resolved.capability.toolId, }) -const capabilityToolBuckBytes = +/** Generated Buck package exposing one projected capability executable and manifest. */ +export const capabilityToolBuckBytes = 'export_file(name = "executable", src = "executable", visibility = ["PUBLIC"])\n' + 'export_file(name = "manifest", src = "manifest.json", visibility = ["PUBLIC"])\n' -const capabilityRootBuckBytes = '# Generated from exact Nix realizations.\n' +/** Generated Buck package marker for the capability projection root. */ +export const capabilityRootBuckBytes = '# Generated from exact Nix realizations.\n' const manifestBytes = (manifest: CapabilityProjectionManifest): string => `${JSON.stringify(manifest)}\n` -const computeCapabilityProjectionGeneration = ( +/** Computes the stable generation identity for an ordered capability file set. */ +export const computeCapabilityProjectionGeneration = ( files: ReadonlyArray<{ readonly path: string; readonly bytes: string }>, ): string => { const framed = files @@ -57,7 +63,8 @@ const computeCapabilityProjectionGeneration = ( return createHash('sha256').update(`${payloadDigest} -\n`).digest('hex') } -const renderCapabilityProjectionDefs = ({ +/** Renders the generated Buck definitions for one capability generation. */ +export const renderCapabilityProjectionDefs = ({ generation, platform, manifests, @@ -79,7 +86,8 @@ const renderCapabilityProjectionDefs = ({ '', ].join('\n') -const projectResolvedCapabilities = async ({ +/** Materializes one immutable generation of resolved capabilities. */ +export const projectResolvedCapabilities = async ({ projectionPath, platform, resolved, diff --git a/packages/@overeng/otel-contract/src/registry-seam.unit.test.ts b/packages/@overeng/otel-contract/src/registry-seam.unit.test.ts index 169c9601c3..f2d305bf12 100644 --- a/packages/@overeng/otel-contract/src/registry-seam.unit.test.ts +++ b/packages/@overeng/otel-contract/src/registry-seam.unit.test.ts @@ -13,8 +13,10 @@ const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url)) const SKIP_DIRS = new Set([ 'node_modules', + 'buck-out', 'dist', 'tmp', + 'target', '.git', '.devenv', '.editor-view', diff --git a/scripts/buck2-cache-posture.integration.test.ts b/scripts/buck2-cache-posture.integration.test.ts new file mode 100644 index 0000000000..39dadd5d52 --- /dev/null +++ b/scripts/buck2-cache-posture.integration.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it } from 'bun:test' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + reconcileStandaloneCachePosture, + standaloneCachePostureConfig, +} from './buck2-cache-posture.ts' + +const temporaryRoots: string[] = [] + +const makeRoot = (): string => { + const root = mkdtempSync(join(tmpdir(), 'buck2-cache-posture-')) + temporaryRoots.push(root) + return root +} + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }) +}) + +describe('standalone Buck cache posture', () => { + it('disables cache reads and uploads only for the exact public-lane opt-out', () => { + expect(standaloneCachePostureConfig({ current: '', env: { BUCK2_NO_REMOTE_CACHE: '1' } })) + .toBe(`# effect-utils standalone cache posture: begin +[buck2] + remote_cache_enabled = false + allow_cache_uploads = false +# effect-utils standalone cache posture: end +`) + + for (const value of [undefined, '0', 'true', ' 1']) + expect( + standaloneCachePostureConfig({ + current: '', + env: { BUCK2_NO_REMOTE_CACHE: value }, + }), + ).toBeUndefined() + }) + + it('preserves local overrides while adding and removing the managed posture atomically', () => { + const root = makeRoot() + const output = join(root, '.buckconfig.local') + const local = `[ui]\n color = true\n` + writeFileSync(output, local) + + reconcileStandaloneCachePosture({ repoRoot: root, env: { BUCK2_NO_REMOTE_CACHE: '1' } }) + expect(readFileSync(output, 'utf8')).toBe(`${local.trimEnd()} + +# effect-utils standalone cache posture: begin +[buck2] + remote_cache_enabled = false + allow_cache_uploads = false +# effect-utils standalone cache posture: end +`) + + reconcileStandaloneCachePosture({ repoRoot: root, env: {} }) + expect(readFileSync(output, 'utf8')).toBe(local) + }) + + it('removes the generated file when a checkout leaves the public trust tier', () => { + const root = makeRoot() + const output = join(root, '.buckconfig.local') + + reconcileStandaloneCachePosture({ repoRoot: root, env: { BUCK2_NO_REMOTE_CACHE: '1' } }) + expect(existsSync(output)).toBeTrue() + + reconcileStandaloneCachePosture({ repoRoot: root, env: { BUCK2_NO_REMOTE_CACHE: '0' } }) + expect(existsSync(output)).toBeFalse() + }) +}) diff --git a/scripts/buck2-cache-posture.ts b/scripts/buck2-cache-posture.ts new file mode 100755 index 0000000000..c2053d204c --- /dev/null +++ b/scripts/buck2-cache-posture.ts @@ -0,0 +1,98 @@ +#!/usr/bin/env -S bun +import { randomUUID } from 'node:crypto' +import { existsSync, lstatSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import process from 'node:process' + +const MANAGED_BEGIN = '# effect-utils standalone cache posture: begin' +const MANAGED_END = '# effect-utils standalone cache posture: end' + +const DISABLED_CACHE_BLOCK = `${MANAGED_BEGIN} +[buck2] + remote_cache_enabled = false + allow_cache_uploads = false +${MANAGED_END}` + +const fail = (message: string): never => { + throw new Error(`standalone Buck cache posture: ${message}`) +} + +const withoutManagedBlock = ( + current: string, +): { readonly content: string; readonly found: boolean } => { + const output: string[] = [] + let inside = false + let found = false + for (const line of current.split(/\r?\n/u)) { + if (line === MANAGED_BEGIN) { + if (inside === true || found === true) fail('duplicate managed block in .buckconfig.local') + inside = true + found = true + continue + } + if (line === MANAGED_END) { + if (inside === false) fail('unmatched managed block end in .buckconfig.local') + inside = false + continue + } + if (inside === false) output.push(line) + } + if (inside === true) fail('unterminated managed block in .buckconfig.local') + return { content: output.join('\n').trimEnd(), found } +} + +/** Derive the standalone checkout's local Buck config from the exact trust-tier opt-out. */ +export const standaloneCachePostureConfig = ({ + current, + env, +}: { + readonly current: string + readonly env: Readonly> +}): string | undefined => { + const withoutManaged = withoutManagedBlock(current) + if (env['BUCK2_NO_REMOTE_CACHE'] !== '1') { + if (withoutManaged.found === false) return current === '' ? undefined : current + return withoutManaged.content === '' ? undefined : `${withoutManaged.content}\n` + } + const unmanaged = withoutManaged.content + return unmanaged === '' + ? `${DISABLED_CACHE_BLOCK}\n` + : `${unmanaged}\n\n${DISABLED_CACHE_BLOCK}\n` +} + +/** Atomically publish or remove only the managed cache posture block. */ +export const reconcileStandaloneCachePosture = ({ + repoRoot, + env, +}: { + readonly repoRoot: string + readonly env: Readonly> +}): void => { + const output = resolve(repoRoot, '.buckconfig.local') + const exists = existsSync(output) + if (exists === true && lstatSync(output).isSymbolicLink() === true) + fail('.buckconfig.local must not be a symbolic link') + const current = exists === true ? readFileSync(output, 'utf8') : '' + const next = standaloneCachePostureConfig({ current, env }) + if (next === undefined) { + if (exists === true) rmSync(output, { force: true }) + return + } + if (next === current) return + const candidate = `${output}.candidate-${randomUUID().replaceAll('-', '')}` + try { + writeFileSync(candidate, next, { flag: 'wx', mode: 0o600 }) + renameSync(candidate, output) + } finally { + rmSync(candidate, { force: true }) + } +} + +if (import.meta.main === true) + try { + const repoRoot = process.argv[2] ?? fail('expected repository root argument') + reconcileStandaloneCachePosture({ repoRoot, env: process.env }) + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } diff --git a/scripts/devenv-task-graph-check.mjs b/scripts/devenv-task-graph-check.mjs index 631df85b34..e0d88ebcd8 100755 --- a/scripts/devenv-task-graph-check.mjs +++ b/scripts/devenv-task-graph-check.mjs @@ -120,12 +120,18 @@ for (const name of [ 'genie:check', 'mr:apply', 'buck2:check', + 'buck2:quick', + 'buck2:all', + 'check:buck2-producer-overlap', 'buck2:typescript:materialize-dist', 'buck2:editor:bootstrap', 'buck2:editor:materialize', 'buck2:editor:authority', 'buck2:editor:publish', 'buck2:editor:check', + 'buck2:editor:publish:restate-effect', + 'buck2:editor:publish:otel-contract', + 'buck2:editor:publish:playwright', 'test:run', 'test:buck2:unit', ]) @@ -169,10 +175,17 @@ try { } const materializer = 'buck2:typescript:materialize-dist' -for (const name of ['check:quick', 'check:all']) { +for (const [checkTask, aggregateTask] of [ + ['check:quick', 'buck2:quick'], + ['check:all', 'buck2:all'], +]) { + ok({ + condition: reaches({ start: checkTask, target: aggregateTask }), + name: `${checkTask} reaches ${aggregateTask}`, + }) ok({ - condition: reaches({ start: name, target: 'buck2:check' }), - name: `${name} reaches the Buck-owned TypeScript gate`, + condition: reaches({ start: checkTask, target: 'check:buck2-producer-overlap' }), + name: `${checkTask} reaches the Buck producer overlap guard`, }) } ok({ @@ -227,10 +240,12 @@ for (const name of [...buck2UnboundedTaskNames, ...buck2ExternalOwnerTaskNames]) }) } for (const name of [ - 'buck2:check', 'buck2:editor:authority', 'buck2:editor:publish', 'buck2:editor:check', + 'buck2:editor:publish:restate-effect', + 'buck2:editor:publish:otel-contract', + 'buck2:editor:publish:playwright', 'buck2:nix-bridge:check', 'lint:check:asset-import-needs-type-reference', 'lint:check:format', @@ -241,19 +256,98 @@ for (const name of [ ...buck2TestLaneTaskNames, ]) { ok({ - condition: reaches({ start: name, target: 'mr:apply' }), - name: `${name} waits for workspace reconciliation and the capability projection`, + condition: reaches({ start: name, target: 'mr:apply' }) === false, + name: `${name} remains standalone`, }) ok({ condition: reaches({ start: name, target: 'genie:check' }), name: `${name} waits for source-side generation freshness`, }) } +for (const name of ['buck2:check', 'buck2:quick', 'buck2:all', 'buck2:nix-bridge:check']) { + ok({ + condition: reaches({ start: name, target: 'mr:apply' }) === false, + name: `${name} remains standalone`, + }) +} ok({ condition: - reaches({ start: 'buck2:editor:bootstrap', target: 'mr:setup' }) === true && + reaches({ start: 'buck2:editor:bootstrap', target: 'mr:setup' }) === false && reaches({ start: 'buck2:editor:bootstrap', target: 'genie:check' }) === false, - name: 'editor bootstrap materializes committed dependencies before freshness without claiming it', + name: 'editor bootstrap reads committed standalone dependencies without mutating projections', +}) + +const scopedPublisherContracts = { + 'buck2:editor:publish:restate-effect': { + consumers: ['test:restate-integration'], + packagePaths: ['packages/@overeng/restate-effect'], + }, + 'buck2:editor:publish:otel-contract': { + consumers: ['weaver:live-check'], + packagePaths: ['packages/@overeng/otel-contract'], + }, + 'buck2:editor:publish:playwright': { + consumers: ['test:pw:tui-react', 'test:pw:utils'], + packagePaths: ['packages/@overeng/tui-react', 'packages/@overeng/utils'], + }, +} +for (const [publisher, { consumers, packagePaths }] of Object.entries(scopedPublisherContracts)) { + const publisherDependencies = [...(dependencies.get(publisher) ?? [])] + ok({ + condition: publisherDependencies.length === 1 && publisherDependencies[0] === 'genie:check', + name: `${publisher} waits directly and only for standalone generator freshness`, + }) + const publisherTask = requireTask(publisher) + const command = publisherTask.command + ok({ + condition: publisherTask.hasExec === true || typeof command === 'string', + name: `${publisher} declares an executable publisher`, + }) + if (typeof command === 'string') { + const commandBody = existsSync(command) === true ? readFileSync(command, 'utf8') : '' + ok({ + condition: + command.includes(publisher.replaceAll(':', '-')) && + commandBody.includes('--packages') && + packagePaths.every((packagePath) => commandBody.includes(`"${packagePath}"`)), + name: `${publisher} has its distinct trace identity and explicit canonical package scope`, + detail: command, + }) + } + const actualConsumers = [...dependencies] + .filter(([, taskDependencies]) => taskDependencies.has(publisher)) + .map(([name]) => name) + .toSorted((a, b) => a.localeCompare(b)) + ok({ + condition: + JSON.stringify(actualConsumers) === + JSON.stringify(consumers.toSorted((a, b) => a.localeCompare(b))), + name: `${publisher} is coalesced across exactly its intended consumers`, + detail: `expected ${consumers.join(', ')}, received ${actualConsumers.join(', ')}`, + }) +} +const fullPublisherTask = requireTask('buck2:editor:publish') +const fullPublisherCommand = fullPublisherTask.command +ok({ + condition: fullPublisherTask.hasExec === true || typeof fullPublisherCommand === 'string', + name: 'whole-workspace editor publication declares an executable fallback', +}) +if (typeof fullPublisherCommand === 'string') { + const fullPublisherCommandBody = + existsSync(fullPublisherCommand) === true ? readFileSync(fullPublisherCommand, 'utf8') : '' + ok({ + condition: + fullPublisherCommand.includes('buck2-editor-publish') && + fullPublisherCommandBody.includes('--packages') === false, + name: 'whole-workspace editor publication retains its unscoped fallback', + detail: fullPublisherCommand, + }) +} +ok({ + condition: + [...(dependencies.get('test:pw:tui-react') ?? [])].join('\n') === + [...(dependencies.get('test:pw:utils') ?? [])].join('\n'), + name: 'both Playwright lanes depend on one canonical union publisher', }) ok({ @@ -279,11 +373,9 @@ const taskSource = (name) => { const editorMaterializeSource = taskSource('buck2:editor:materialize') const orderedMaterializationSteps = [ - 'devenv tasks run mr:setup', 'devenv tasks run buck2:editor:bootstrap --mode single', 'devenv tasks run genie:run --mode single', 'devenv tasks run genie:check --mode single', - 'devenv tasks run mr:apply --mode single', 'devenv tasks run buck2:editor:publish --mode single', ] const orderedMaterializationOffsets = orderedMaterializationSteps.map((step) => @@ -294,7 +386,7 @@ ok({ (offset, index) => offset !== -1 && (index === 0 || offset > orderedMaterializationOffsets[index - 1]), ), - name: 'editor materialization runs bootstrap, generation, freshness, composition, and publication in order', + name: 'editor materialization runs bootstrap, generation, freshness, and publication in order', }) const materializerSource = taskSource(materializer) @@ -307,7 +399,7 @@ ok({ condition: materializerSource.includes(typescriptAuthorityRuntimePath) === true && materializerSource.includes('materialize-dist "$root"') === true && - materializerSource.includes('BUCK2_BIN=') === true, + materializerSource.includes('WORKSPACE_ROOT="$root"') === true, name: 'materializer dispatches the registry-backed TypeScript authority runtime', }) ok({ @@ -323,14 +415,15 @@ ok({ }) ok({ condition: - source.includes('composed_workspace_root()') === true && - source.includes('worktree list --porcelain -z') === true && - source.includes('backlink=') === true && - materializerSource.includes('requires a composed megarepo workspace') === true && - materializerSource.includes('WORKSPACE_ROOT=') === true && + source.includes('typescriptPublicationRootPredicate =') === true && + source.includes('--workspace-root "$root"') === true && + source.includes('--buck2 "$BUCK2_BIN"') === true && + materializerSource.includes('requires a composed megarepo workspace') === false && + materializerSource.includes('WORKSPACE_ROOT="$root"') === true && + materializerSource.includes('BUCK2_BIN="$workspace_root/.megarepo/bin/"buck2') === true && materializerSource.includes('TYPESCRIPT_DIST_MODE=') === false && materializerSource.includes('TSGO_BIN=') === false, - name: 'materializer publishes only from a reciprocal composition root', + name: 'materializer defaults to the standalone root and preserves explicit composed publication', }) const editorViewHelper = source.slice( @@ -349,13 +442,25 @@ ok({ }) const buckCheckSource = taskSource('buck2:check') +const buckQuickSource = taskSource('buck2:quick') +const buckAllSource = taskSource('buck2:all') +const producerOverlapSource = taskSource('check:buck2-producer-overlap') +ok({ + condition: + source.includes('buck2AggregateExec =') === true && + buckCheckSource.includes('audit providers') === true && + buckCheckSource.includes('typescript-authority-runtime.ts') === false && + buckQuickSource.includes('buck2AggregateExec "buck2:quick" "//:quick"') === true && + buckAllSource.includes('buck2AggregateExec "buck2:all" "//:all"') === true && + buckQuickSource.includes('--local-only') === false && + buckAllSource.includes('--local-only') === false, + name: 'Buck check tasks separate provider audit from standalone cache-enabled aggregates', +}) ok({ condition: - buckCheckSource.includes('realpath "$root/../.."') === true && - buckCheckSource.includes('$workspace_root/.megarepo/bin/buck2') === true && - buckCheckSource.includes(typescriptAuthorityRuntimePath) === true && - buckCheckSource.includes('build "$buck"') === true, - name: 'buck2:check resolves the composition wrapper and dispatches the authority runtime', + producerOverlapSource.includes('genie/buck2/producer-overlap.ts') === true && + producerOverlapSource.includes('task-config-devenv-config-task-config') === true, + name: 'producer overlap guard reads the evaluated task registry', }) const buckToolchainSource = readFileSync(`${root}/buck2/toolchains/BUCK`, 'utf8') ok({ @@ -364,6 +469,26 @@ ok({ buckToolchainSource.includes('name = "archive_tool"') === true, name: 'Buck toolchains live in the buck2/toolchains package', }) +const configuredToolchainSource = readFileSync(`${root}/buck2/toolchains/configured.bzl`, 'utf8') +ok({ + condition: + buckToolchainSource.includes('load("@capabilities//:defs.bzl"') === true && + configuredToolchainSource.includes('load("@capabilities//:defs.bzl"') === true, + name: 'capability Starlark loads use external-cell import syntax', +}) +const standaloneBuckConfig = readFileSync(`${root}/.buckconfig`, 'utf8') +const compositionRootSource = readFileSync( + `${root}/packages/@overeng/megarepo/src/composition/root/composition-root.ts`, + 'utf8', +) +ok({ + condition: + standaloneBuckConfig.includes('file_watcher = notify') === true && + standaloneBuckConfig.includes('file_watcher = watchman') === false && + compositionRootSource.includes("lines.push('', '[buck2]', ' file_watcher = watchman')") === + true, + name: 'standalone roots use notify while composed roots retain Watchman', +}) ok({ condition: existsSync(`${root}/toolchains`) === false, name: 'no legacy top-level toolchains directory remains', diff --git a/scripts/editor-view-authority.ts b/scripts/editor-view-authority.ts index 009a30fde7..e7cd891cbe 100755 --- a/scripts/editor-view-authority.ts +++ b/scripts/editor-view-authority.ts @@ -1,11 +1,12 @@ #!/usr/bin/env -S bun -import { readFileSync } from 'node:fs' -import { resolve } from 'node:path' import process from 'node:process' import { pnpmWorkspaceMemberPaths } from '../genie/packages.ts' import { reconcileBuckViews } from '../packages/@overeng/buck2-tools/src/buck-watch.ts' -import type { BuckWatchPlan } from '../packages/@overeng/buck2-tools/src/buck-watch.ts' +import type { + BuckReconcileTiming, + BuckWatchPlan, +} from '../packages/@overeng/buck2-tools/src/buck-watch.ts' import { writeEditorViewAuthority } from '../packages/@overeng/buck2-tools/src/editor-view-authority.ts' import { defaultEditorViewName } from '../packages/@overeng/buck2-tools/src/editor-view.ts' /** Complete source-authoritative editor consumer registry, including the repository root. */ @@ -50,6 +51,59 @@ export const editorViewPlan = ({ left.packagePath === right.packagePath ? 0 : left.packagePath < right.packagePath ? -1 : 1, ), }) +/** Decode and validate the explicit package scope supplied to an editor-view publication. */ +export const decodePublicationPackagePaths = (serialized: string): readonly string[] => { + let decoded: unknown + try { + decoded = JSON.parse(serialized) + } catch (error) { + return fail( + `--packages must be a JSON array: ${error instanceof Error ? error.message : String(error)}`, + ) + } + if (Array.isArray(decoded) === false || decoded.length === 0) + fail('--packages must be a non-empty JSON array of package paths') + const packagePaths = decoded + .map((entry) => + typeof entry === 'string' + ? entry + : fail('--packages must be a non-empty JSON array of package paths'), + ) + .toSorted((left, right) => (left === right ? 0 : left < right ? -1 : 1)) + if (new Set(packagePaths).size !== packagePaths.length) fail('--packages repeats a package path') + for (const packagePath of packagePaths) + if (editorViewPackagePaths.includes(packagePath) === false) + fail(`--packages contains an unregistered editor consumer: ${packagePath}`) + return packagePaths +} + +/** Resolve the editor-view authority and publication scopes for one CLI command. */ +export const resolveEditorViewPackageScope = ({ + command, + authorityPackagePaths, + serializedPublicationPackages, +}: { + readonly command: Command + readonly authorityPackagePaths: readonly string[] + readonly serializedPublicationPackages: string | undefined +}): { + readonly authorityPackagePaths: readonly string[] + readonly publicationPackagePaths: readonly string[] +} => { + if (serializedPublicationPackages === undefined) { + if (command === 'bootstrap') fail('--packages is required with bootstrap') + return { + authorityPackagePaths, + publicationPackagePaths: authorityPackagePaths, + } + } + if (command !== 'publish' && command !== 'bootstrap') + fail('--packages is only valid with publish or bootstrap') + return { + authorityPackagePaths, + publicationPackagePaths: decodePublicationPackagePaths(serializedPublicationPackages), + } +} type Command = 'authority' | 'bootstrap' | 'check' | 'publish' @@ -57,23 +111,19 @@ const fail = (message: string): never => { throw new Error(`editor view authority: ${message}`) } -const commands = new Set(['authority', 'bootstrap', 'check', 'publish']) +type EditorViewTiming = + | BuckReconcileTiming + | { + readonly phase: 'authority' + readonly durationMs: number + } -const bootstrapPackagePaths = (repoRoot: string): readonly string[] => { - const value: unknown = JSON.parse(readFileSync(resolve(repoRoot, 'package.json'), 'utf8')) - if ( - typeof value !== 'object' || - value === null || - !('workspaces' in value) || - Array.isArray(value.workspaces) === false || - value.workspaces.every((entry) => typeof entry === 'string') === false - ) - fail('generated root package.json must declare string workspace paths') - return ['.', ...value.workspaces].toSorted((left, right) => - left === right ? 0 : left < right ? -1 : 1, - ) +const reportTiming = (timing: EditorViewTiming): void => { + process.stderr.write(`[editor-view-timing] ${JSON.stringify(timing)}\n`) } +const commands = new Set(['authority', 'bootstrap', 'check', 'publish']) + const parseCli = (args: readonly string[]) => { const command = args[0] if (commands.has(command as Command) === false) @@ -97,6 +147,7 @@ const parseCli = (args: readonly string[]) => { '--cp', '--mv', '--snapshot-retention', + '--packages', ]) for (const flag of values.keys()) if (allowed.has(flag) === false) fail(`unexpected option: ${flag}`) @@ -114,6 +165,7 @@ const parseCli = (args: readonly string[]) => { cp: admitting === true ? get('--cp') : '', mv: admitting === true ? get('--mv') : '', snapshotRetention: admitting === true ? Number(get('--snapshot-retention')) : 3, + publicationPackages: values.get('--packages'), } } @@ -125,13 +177,22 @@ const main = async (): Promise => { options.snapshotRetention > 32 ) fail('--snapshot-retention must be an integer from 2 through 32') - const packagePaths = - options.command === 'bootstrap' - ? bootstrapPackagePaths(options.repoRoot) - : editorViewPackagePaths + const authorityPackagePaths = editorViewPackagePaths + const packageScope = resolveEditorViewPackageScope({ + command: options.command, + authorityPackagePaths, + serializedPublicationPackages: options.publicationPackages, + }) + // Scope only target construction and publication. Every publisher still refreshes and passes + // freshly proven whole-workspace authority to each selected package view. + const authorityStartedAt = performance.now() const authority = await writeEditorViewAuthority({ ...options, - requiredPackages: packagePaths, + requiredPackages: packageScope.authorityPackagePaths, + }) + reportTiming({ + phase: 'authority', + durationMs: performance.now() - authorityStartedAt, }) if (options.command === 'authority') { process.stdout.write( @@ -139,7 +200,10 @@ const main = async (): Promise => { ) return } - const plan = editorViewPlan({ cell: options.cell, packagePaths }) + const plan = editorViewPlan({ + cell: options.cell, + packagePaths: packageScope.publicationPackagePaths, + }) await reconcileBuckViews({ request: { packagePaths: plan.packages.map(({ packagePath }) => packagePath), @@ -159,6 +223,8 @@ const main = async (): Promise => { cp: options.cp, mv: options.mv, snapshotRetention: options.snapshotRetention, + onTiming: reportTiming, + parallelEditorRoots: options.command === 'bootstrap', }, }) const action = diff --git a/scripts/editor-view-authority.unit.test.ts b/scripts/editor-view-authority.unit.test.ts index a80c1121e9..81823efb92 100644 --- a/scripts/editor-view-authority.unit.test.ts +++ b/scripts/editor-view-authority.unit.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from 'bun:test' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' -import { editorViewPackagePaths, editorViewPlan } from './editor-view-authority.ts' +import { + decodePublicationPackagePaths, + editorViewPackagePaths, + editorViewPlan, + resolveEditorViewPackageScope, +} from './editor-view-authority.ts' describe('editor view authority orchestration', () => { it('derives one deterministic editor publication entry per workspace consumer', () => { @@ -22,4 +29,180 @@ describe('editor view authority orchestration', () => { expect(entry.editor?.target).toBe(`//${entry.packagePath}:editor_inputs`) } }) + + it('canonicalizes an explicit publication scope against the admitted registry', () => { + const serializedPublicationPackages = JSON.stringify([ + 'packages/@overeng/utils', + 'packages/@overeng/tui-react', + ]) + + expect(decodePublicationPackagePaths(serializedPublicationPackages)).toEqual([ + 'packages/@overeng/tui-react', + 'packages/@overeng/utils', + ]) + }) + + it.each([ + ['not JSON', '{', '--packages must be a JSON array'], + ['an empty array', '[]', '--packages must be a non-empty JSON array of package paths'], + [ + 'a non-string entry', + '["packages/@overeng/utils",1]', + '--packages must be a non-empty JSON array of package paths', + ], + [ + 'a duplicate package', + '["packages/@overeng/utils","packages/@overeng/utils"]', + '--packages repeats a package path', + ], + [ + 'an unregistered package', + '["packages/@overeng/not-admitted"]', + '--packages contains an unregistered editor consumer', + ], + ])('rejects %s', (_case, serialized, message) => { + expect(() => decodePublicationPackagePaths(serialized)).toThrow(message) + }) + + it('keeps whole-workspace authority while narrowing only the publication plan', () => { + const scope = resolveEditorViewPackageScope({ + command: 'publish', + authorityPackagePaths: editorViewPackagePaths, + serializedPublicationPackages: JSON.stringify(['packages/@overeng/utils']), + }) + + expect(scope.authorityPackagePaths).toBe(editorViewPackagePaths) + expect(scope.publicationPackagePaths).toEqual(['packages/@overeng/utils']) + expect( + editorViewPlan({ + cell: 'workspace_cell', + packagePaths: scope.publicationPackagePaths, + }).packages.map(({ packagePath }) => packagePath), + ).toEqual(['packages/@overeng/utils']) + }) + + it('bootstraps the declared generator import closure under whole-workspace authority', () => { + const bootstrapPackagePaths = ['.', 'packages/@overeng/otel-contract'] + const scope = resolveEditorViewPackageScope({ + command: 'bootstrap', + authorityPackagePaths: editorViewPackagePaths, + serializedPublicationPackages: JSON.stringify(bootstrapPackagePaths), + }) + + expect(scope.authorityPackagePaths).toBe(editorViewPackagePaths) + expect(scope.publicationPackagePaths).toEqual(bootstrapPackagePaths) + expect( + editorViewPlan({ + cell: 'workspace_cell', + packagePaths: scope.publicationPackagePaths, + }).packages.map(({ packagePath }) => packagePath), + ).toEqual(bootstrapPackagePaths) + }) + + it('retains whole-workspace publication when no explicit scope is provided', () => { + const scope = resolveEditorViewPackageScope({ + command: 'publish', + authorityPackagePaths: editorViewPackagePaths, + serializedPublicationPackages: undefined, + }) + + expect(scope.authorityPackagePaths).toBe(editorViewPackagePaths) + expect(scope.publicationPackagePaths).toBe(editorViewPackagePaths) + }) + + it('requires an explicit bootstrap scope and rejects explicit scopes on read-only commands', () => { + expect(() => + resolveEditorViewPackageScope({ + command: 'bootstrap', + authorityPackagePaths: editorViewPackagePaths, + serializedPublicationPackages: undefined, + }), + ).toThrow('--packages is required with bootstrap') + for (const command of ['authority', 'check'] as const) + expect(() => + resolveEditorViewPackageScope({ + command, + authorityPackagePaths: editorViewPackagePaths, + serializedPublicationPackages: '["packages/@overeng/utils"]', + }), + ).toThrow('--packages is only valid with publish or bootstrap') + }) + + it('rejects an invalid explicit scope before attempting authority or Buck work', () => { + const script = join(dirname(fileURLToPath(import.meta.url)), 'editor-view-authority.ts') + const result = Bun.spawnSync({ + cmd: [ + process.execPath, + script, + 'publish', + '--repo-root', + '/does-not-exist', + '--workspace-root', + '/does-not-exist', + '--cell', + 'workspace_cell', + '--buck2', + '/does-not-exist/buck2', + '--git', + '/does-not-exist/git', + '--output', + '/does-not-exist/authority.json', + '--publisher', + '/does-not-exist/publisher.ts', + '--cp', + '/does-not-exist/cp', + '--mv', + '/does-not-exist/mv', + '--snapshot-retention', + '3', + '--packages', + '["packages/@overeng/not-admitted"]', + ], + stderr: 'pipe', + stdout: 'pipe', + }) + + expect(result.exitCode).toBe(1) + expect(result.stderr.toString()).toContain( + '--packages contains an unregistered editor consumer: packages/@overeng/not-admitted', + ) + }) + + it('rejects a missing bootstrap scope before attempting authority or Buck work', () => { + const script = join(dirname(fileURLToPath(import.meta.url)), 'editor-view-authority.ts') + const result = Bun.spawnSync({ + cmd: [ + process.execPath, + script, + 'bootstrap', + '--repo-root', + '/does-not-exist', + '--workspace-root', + '/does-not-exist', + '--cell', + 'workspace_cell', + '--buck2', + '/does-not-exist/buck2', + '--git', + '/does-not-exist/git', + '--output', + '/does-not-exist/authority.json', + '--publisher', + '/does-not-exist/publisher.ts', + '--cp', + '/does-not-exist/cp', + '--mv', + '/does-not-exist/mv', + '--snapshot-retention', + '3', + ], + stderr: 'pipe', + stdout: 'pipe', + }) + + expect(result.exitCode).toBe(1) + expect(result.stderr.toString()).toBe( + 'editor view authority: --packages is required with bootstrap\n', + ) + }) })