diff --git a/.github/workflows/CICD.md b/.github/workflows/CICD.md new file mode 100644 index 0000000000..ad5deb0acc --- /dev/null +++ b/.github/workflows/CICD.md @@ -0,0 +1,140 @@ +# CI/CD Flows + +## PR Quality Gates (ci.yml) + +Trigger: pull_request to develop or master + +``` + ┌──────────────────┐ + │ PR opened │ + └────────┬─────────┘ + │ + ┌────────▼─────────┐ + │ fmt --all │ + └────────┬─────────┘ + │ + ┌───────────▼──────────┐ + │ clippy --all-targets │ + └───┬───┬───┬───┬───┬──┘ + │ │ │ │ │ + ┌───────────────┘ │ │ │ └────────────────┐ + │ ┌───────────┘ │ └───────────┐ │ + ▼ ▼ ▼ ▼ ▼ + ┌──────────┐ ┌──────────┐ ┌───────────┐ ┌─────────┐ ┌──────────┐ + │ test │ │ security │ │ semgrep │ │benchmark│ │ doc │ + │ ubuntu │ │ cargo │ │ AST-aware │ │ >=80% │ │ review │ + │ windows │ │ audit │ │ diff-only │ │ savings │ │ ai agent │ + │ macos │ │ patterns │ │ │ │ │ │ │ + └────┬─────┘ └────┬─────┘ └─────┬─────┘ └────┬────┘ └────┬─────┘ + │ │ │ │ │ + └────────────┴─────────┬───┴─────────────┴────────────┘ + │ + ┌──────────▼─────────┐ + │ All must pass │ + │ to merge │ + └────────────────────┘ + + + DCO check (independent, develop PRs only) + + Dependabot (weekly: Cargo deps + GitHub Actions) +``` + +## Merge to develop — pre-release (cd.yml) + +Trigger: push to develop | workflow_dispatch (not master) | Concurrency: cancel-in-progress + +``` + ┌──────────────────┐ + │ push to develop │ + │ OR dispatch │ + └────────┬─────────┘ + │ + ┌────────▼──────────────────┐ + │ pre-release │ + │ compute next version │ + │ from conventional commits │ + │ tag = v{next}-rc.{run} │ + └────────┬──────────────────┘ + │ + ┌────────▼──────────────────┐ + │ release.yml │ + │ prerelease = true │ + └────────┬──────────────────┘ + │ + ┌────────▼──────────────────┐ + │ Build │ + │ 5 platforms + DEB + RPM │ + └────────┬──────────────────┘ + │ + ┌────────▼──────────────────┐ + │ GitHub Release │ + │ (pre-release badge) │ + │ │ + │ Discord: SKIPPED │ + │ Homebrew: SKIPPED │ + └──────────────────────────┘ +``` + +## Merge to master — stable release (cd.yml) + +Trigger: push to master (only) | Concurrency: never cancelled + +``` + ┌──────────────────┐ + │ push to master │ + └────────┬─────────┘ + │ + ┌────────▼──────────────────┐ + │ release-please │ + │ analyze conventional │ + │ commits │ + └────────┬──────────────────┘ + │ + ┌────┴────────────────┐ + │ │ + no release release created + │ │ + ▼ ▼ + ┌──────────────┐ ┌───────────────────────┐ + │ create/update│ │ release.yml │ + │ release PR │ │ prerelease = false │ + └──────────────┘ └───────────┬───────────┘ + │ + ┌────────────▼────────────┐ + │ Build │ + │ 5 platforms + DEB + RPM │ + └────────────┬────────────┘ + │ + ┌────────────▼────────────┐ + │ GitHub Release │ + │ (stable, "Latest" badge) │ + └──┬─────────┬─────────┬──┘ + │ │ │ + ▼ ▼ ▼ + Discord Homebrew latest + notify tap update tag +``` + +## Manual release (release.yml) + +Trigger: workflow_dispatch + +``` + ┌────────────────────────┐ + │ workflow_dispatch │ + │ inputs: tag, prerelease │ + └───────────┬────────────┘ + │ + ┌───────────▼────────────┐ + │ Full build pipeline │ + │ 5 platforms + DEB + RPM │ + └───────────┬────────────┘ + │ + ┌──────┴──────┐ + │ │ + prerelease=false prerelease=true + │ │ + ▼ ▼ + Discord pre-release + Homebrew badge only + latest tag +``` diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000000..0ed2bc0fcf --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,155 @@ +name: CD + +on: + workflow_dispatch: + push: + branches: [develop, master] + +concurrency: + group: cd-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +permissions: + contents: write + pull-requests: write + +jobs: + # ═══════════════════════════════════════════════ + # DEVELOP PATH: Pre-release + # ═══════════════════════════════════════════════ + + pre-release: + if: >- + github.ref == 'refs/heads/develop' + || (github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/master') + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Compute version from commits like release please + id: tag + run: | + LATEST_TAG=$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' --sort=-version:refname | grep -v '-' | head -1) + if [ -z "$LATEST_TAG" ]; then + echo "::error::No stable release tag found" + exit 1 + fi + LATEST_VERSION="${LATEST_TAG#v}" + echo "Latest release: $LATEST_TAG" + + # ── Analyse conventional commits since that tag ── + COMMITS=$(git log "${LATEST_TAG}..HEAD" --format="%s") + HAS_BREAKING=$(echo "$COMMITS" | grep -cE '^[a-z]+(\(.+\))?!:' || true) + HAS_FEAT=$(echo "$COMMITS" | grep -cE '^feat(\(.+\))?:' || true) + HAS_FIX=$(echo "$COMMITS" | grep -cE '^fix(\(.+\))?:' || true) + echo "Commits since ${LATEST_TAG} — breaking=$HAS_BREAKING feat=$HAS_FEAT fix=$HAS_FIX" + + # ── Compute next version (matches release-please observed behaviour) ── + # Pre-1.0 with bump-minor-pre-major: breaking → minor, feat → minor, fix → patch + IFS='.' read -r MAJOR MINOR PATCH <<< "$LATEST_VERSION" + if [ "$MAJOR" -eq 0 ]; then + if [ "$HAS_BREAKING" -gt 0 ] || [ "$HAS_FEAT" -gt 0 ]; then + MINOR=$((MINOR + 1)); PATCH=0 # breaking or feat → minor + else + PATCH=$((PATCH + 1)) # fix only → patch + fi + else + if [ "$HAS_BREAKING" -gt 0 ]; then + MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 # breaking → major + elif [ "$HAS_FEAT" -gt 0 ]; then + MINOR=$((MINOR + 1)); PATCH=0 # feat → minor + else + PATCH=$((PATCH + 1)) # fix → patch + fi + fi + VERSION="${MAJOR}.${MINOR}.${PATCH}" + TAG="dev-${VERSION}-rc.${{ github.run_number }}" + + echo "Next version: $VERSION (from $LATEST_VERSION)" + echo "Pre-release tag: $TAG" + + # Safety: fail if this exact tag already exists + if git ls-remote --tags origin "refs/tags/${TAG}" | grep -q .; then + echo "::error::Tag ${TAG} already exists" + exit 1 + fi + + echo "tag=$TAG" >> $GITHUB_OUTPUT + + build-prerelease: + name: Build pre-release + needs: pre-release + if: needs.pre-release.outputs.tag != '' + uses: ./.github/workflows/release.yml + with: + tag: ${{ needs.pre-release.outputs.tag }} + prerelease: true + permissions: + contents: write + secrets: inherit + + # ═══════════════════════════════════════════════ + # MASTER PATH: Full release + # ═══════════════════════════════════════════════ + + release-please: + if: github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + outputs: + release_created: ${{ steps.release.outputs.release_created }} + tag_name: ${{ steps.release.outputs.tag_name }} + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + - uses: googleapis/release-please-action@v4 + id: release + with: + release-type: rust + package-name: rtk + token: ${{ steps.app-token.outputs.token }} + + build-release: + name: Build and upload release assets + needs: release-please + if: ${{ needs.release-please.outputs.release_created == 'true' }} + uses: ./.github/workflows/release.yml + with: + tag: ${{ needs.release-please.outputs.tag_name }} + permissions: + contents: write + secrets: inherit + + update-latest-tag: + name: Update 'latest' tag + needs: [release-please, build-release] + if: ${{ needs.release-please.outputs.release_created == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Update latest tag + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -fa latest -m "Latest stable release (${{ needs.release-please.outputs.tag_name }})" + git push origin latest --force diff --git a/.github/workflows/ci-self-hosted.yml b/.github/workflows/ci-self-hosted.yml new file mode 100644 index 0000000000..eb05d7da83 --- /dev/null +++ b/.github/workflows/ci-self-hosted.yml @@ -0,0 +1,99 @@ +name: CI (self-hosted) + +# Routes the compile-heavy cargo test pass to a self-hosted +# Linux/X64 runner. Triggered only on pushes to in-repo branches and on +# manual dispatch — NEVER on `pull_request`, because fork PRs from +# outside contributors must not be able to execute code on the +# self-hosted box (they hit the existing `ubuntu-latest` ci.yml jobs +# instead, where GitHub sandboxes the work). +# +# Belt-and-braces: GitHub repo setting "Require approval for all +# outside collaborators" must also be enabled in Settings -> Actions +# -> General so even the cloud-hosted workflows don't fire on a fork +# PR without manual approval. + +on: + push: + branches: + - develop + - main + - 'feat/**' + - 'fix/**' + - 'harden/**' + - 'polish/**' + - 'perf/**' + - 'docs/**' + - 'ci/**' + workflow_dispatch: + +# Tighten the default GITHUB_TOKEN to read-only. Per-job permissions can +# override if needed (none of these jobs write to the repo). +permissions: + contents: read + +concurrency: + group: self-hosted-${{ github.ref }} + cancel-in-progress: true + +# All third-party actions pinned to commit SHAs (not tags) so a +# compromised tag re-point cannot poison the self-hosted runner. +# Version comments reflect the tag/branch the SHA resolved from at +# pinning time — update via Dependabot or manual re-resolve. + +jobs: + test: + name: cargo test (self-hosted) + runs-on: [self-hosted, Linux, X64] + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable channel ref + + - name: Cargo cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + shared-key: self-hosted-stable + + - name: cargo test --bin contextcrawler + run: cargo test --bin contextcrawler --no-fail-fast + + - name: Upload tee logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rtk-tee-logs-${{ github.run_id }} + path: ~/.local/share/rtk/tee/ + if-no-files-found: ignore + retention-days: 7 + + clippy: + name: cargo clippy (self-hosted) + runs-on: [self-hosted, Linux, X64] + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable channel ref + with: + components: clippy + + - name: Cargo cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + shared-key: self-hosted-stable + + # Clippy runs in visible-but-non-fatal mode while the codebase + # works through new doc/MSRV lints introduced by post-1.80 + # toolchains. Re-tighten to `-- -D warnings` after lint cleanup + # lands (tracked separately). + - name: cargo clippy + run: cargo clippy --bin contextcrawler --all-features diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..992dea78e6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,389 @@ +name: CI + +on: + pull_request: + branches: [develop, master] + +permissions: + contents: read + pull-requests: read + +env: + CARGO_TERM_COLOR: always + +jobs: + # ─── Fast gates (fail early, save CI minutes) ─── + + check-test-presence: + name: test presence + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 50 + - name: Check filter modules have tests + run: | + git fetch origin "${{ github.base_ref }}" --depth=1 || true + bash scripts/check-test-presence.sh "origin/${{ github.base_ref }}" + + fmt: + name: fmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - run: cargo fmt --all -- --check + + clippy: + name: clippy + needs: fmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - run: cargo clippy --all-targets + + # ─── Parallel gates (all need code to compile) ─── + + test: + name: test (${{ matrix.os }}) + needs: clippy + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo test --all + + security: + name: Security Scan + needs: clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + + - name: Install cargo-audit + run: cargo install cargo-audit + + - name: Cargo Audit (CVE check) + run: | + echo "## Security Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Dependency Vulnerabilities" >> $GITHUB_STEP_SUMMARY + if cargo audit 2>&1 | tee audit.log; then + echo "No known vulnerabilities detected" >> $GITHUB_STEP_SUMMARY + else + echo "Vulnerabilities found:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat audit.log >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "::warning::Dependency vulnerabilities detected - review required" + fi + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Critical files check + run: | + echo "### Critical Files Modified" >> $GITHUB_STEP_SUMMARY + CRITICAL=$(git diff --name-only origin/master...HEAD | grep -E "(runner|summary|tracking|init|pnpm_cmd|container)\.rs|Cargo\.toml|workflows/.*\.yml" || true) + if [ -n "$CRITICAL" ]; then + echo "**HIGH RISK**: The following critical files were modified:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "$CRITICAL" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Required Actions:**" >> $GITHUB_STEP_SUMMARY + echo "- [ ] Manual security review by 2 maintainers" >> $GITHUB_STEP_SUMMARY + echo "- [ ] Verify no shell injection vectors" >> $GITHUB_STEP_SUMMARY + echo "- [ ] Check input validation remains intact" >> $GITHUB_STEP_SUMMARY + echo "::warning::Critical RTK files modified - enhanced review required" + else + echo "No critical files modified" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Dangerous patterns scan + run: | + echo "### Dangerous Code Patterns" >> $GITHUB_STEP_SUMMARY + PATTERNS=$(git diff origin/master...HEAD | grep -E "Command::new\(\"sh\"|Command::new\(\"bash\"|\.env\(\"LD_PRELOAD|\.env\(\"PATH|reqwest::|std::net::|TcpStream|UdpSocket|unsafe \{|\.unwrap\(\) |panic!\(|todo!\(|unimplemented!\(" || true) + if [ -n "$PATTERNS" ]; then + echo "**Potentially dangerous patterns detected:**" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo "$PATTERNS" | head -30 >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Security Concerns:**" >> $GITHUB_STEP_SUMMARY + echo "$PATTERNS" | grep -q "Command::new" && echo "- Shell command execution detected" >> $GITHUB_STEP_SUMMARY || true + echo "$PATTERNS" | grep -q "\.env\(\"" && echo "- Environment variable manipulation" >> $GITHUB_STEP_SUMMARY || true + echo "$PATTERNS" | grep -q "reqwest::\|std::net::\|TcpStream\|UdpSocket" && echo "- Network operations added" >> $GITHUB_STEP_SUMMARY || true + echo "$PATTERNS" | grep -q "unsafe" && echo "- Unsafe code blocks" >> $GITHUB_STEP_SUMMARY || true + echo "$PATTERNS" | grep -q "\.unwrap\(\)\|panic!\(" && echo "- Panic-inducing code" >> $GITHUB_STEP_SUMMARY || true + echo "::warning::Dangerous code patterns detected - manual review required" + else + echo "No dangerous patterns detected" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + - name: New dependencies check + run: | + echo "### Dependencies Changes" >> $GITHUB_STEP_SUMMARY + if git diff origin/master...HEAD Cargo.toml | grep -E "^\+.*=" | grep -v "^\+\+\+" > new_deps.txt; then + echo "**New dependencies added:**" >> $GITHUB_STEP_SUMMARY + echo '```toml' >> $GITHUB_STEP_SUMMARY + cat new_deps.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Required Actions:**" >> $GITHUB_STEP_SUMMARY + echo "- [ ] Audit each new dependency on crates.io" >> $GITHUB_STEP_SUMMARY + echo "- [ ] Check maintainer reputation and download counts" >> $GITHUB_STEP_SUMMARY + echo "- [ ] Verify no typosquatting (e.g., 'reqwest' vs 'request')" >> $GITHUB_STEP_SUMMARY + echo "::warning::New dependencies require supply chain audit" + else + echo "No new dependencies added" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Clippy security lints + run: | + echo "### Clippy Security Lints" >> $GITHUB_STEP_SUMMARY + if cargo clippy --all-targets -- -W clippy::unwrap_used -W clippy::panic -W clippy::expect_used 2>&1 | tee clippy.log | grep -E "warning:|error:"; then + echo "Security-related lints triggered:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E "warning:|error:" clippy.log | head -20 >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "::warning::Clippy security lints failed" + else + echo "All security lints passed" >> $GITHUB_STEP_SUMMARY + fi + + - name: Summary verdict + run: | + echo "---" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Security Review Verdict" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**This is an automated security scan. A human maintainer must:**" >> $GITHUB_STEP_SUMMARY + echo "1. Review all warnings above" >> $GITHUB_STEP_SUMMARY + echo "2. Verify PR intent matches actual code changes" >> $GITHUB_STEP_SUMMARY + echo "3. Check for subtle backdoors or logic bombs" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**For high-risk PRs (critical files modified):**" >> $GITHUB_STEP_SUMMARY + echo "- Require approval from 2 maintainers" >> $GITHUB_STEP_SUMMARY + echo "- Test in isolated environment before merge" >> $GITHUB_STEP_SUMMARY + + semgrep: + name: semgrep security scan + needs: clippy + runs-on: ubuntu-latest + container: + image: semgrep/semgrep + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: semgrep scan --config .semgrep.yml --baseline-commit ${{ github.event.pull_request.base.sha }} --error + + benchmark: + name: benchmark + needs: clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + + - name: Build rtk + run: cargo build --release + + - name: Install system tools + run: sudo apt-get install -y tree + + - name: Install Python tools + run: pip install ruff pytest mypy + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "stable" + + - name: Install Go tools + run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + + - name: Run benchmark + run: ./scripts/benchmark.sh + + # ─── AI Doc Review: develop PRs only ─── + + doc-review: + name: doc review + if: github.base_ref == 'develop' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Gather PR context + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUM=${{ github.event.pull_request.number }} + gh pr diff "$PR_NUM" --name-only > changed_files.txt + gh pr diff "$PR_NUM" | head -c 12000 > diff.txt + gh pr view "$PR_NUM" --json title,body --jq '"PR Title: \(.title)\nPR Description: \(.body)"' > pr_info.txt + + - name: Build prompt files + run: | + # System prompt + cat <<'EOF' > system_prompt.txt + You are a documentation reviewer for the RTK project. + You will receive the project's CONTRIBUTING.md (which contains the documentation rules), the PR info, changed files, and diff. + Your job: based ONLY on the documentation rules in CONTRIBUTING.md, decide if the PR includes the required documentation updates. + + IMPORTANT: + - CI/CD changes, test-only changes, and refactors with no user-facing impact do NOT require doc updates. + - Be practical, not pedantic. Small obvious fixes don't need CHANGELOG entries. + - Only flag missing docs when there is a clear user-facing change. + EOF + + # User prompt: concatenate files (no printf, no variable expansion issues) + { + cat pr_info.txt + echo "" + echo "---" + echo "CONTRIBUTING.md:" + cat CONTRIBUTING.md + echo "" + echo "---" + echo "Changed files:" + cat changed_files.txt + echo "" + echo "---" + echo "Diff (may be truncated):" + cat diff.txt + } > user_prompt.txt + + - name: AI documentation review + env: + ANTHROPIC_API_KEY: ${{ secrets.RTK_DOCS_ANTHROPIC_KEY }} + run: | + echo "## Documentation Review (AI)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "::warning::ANTHROPIC_API_KEY not configured — skipping AI doc review" + echo "Skipped: ANTHROPIC_API_KEY secret not configured." >> $GITHUB_STEP_SUMMARY + exit 0 + fi + + echo "::group::Preparing API request" + echo "System prompt: $(wc -c < system_prompt.txt) bytes" + echo "User prompt: $(wc -c < user_prompt.txt) bytes" + SYSTEM_JSON=$(jq -Rs . < system_prompt.txt) + USER_JSON=$(jq -Rs . < user_prompt.txt) + echo "::endgroup::" + + echo "::group::Calling Claude API (claude-sonnet-4-6)" + RESPONSE=$(curl -s -w "\n%{http_code}" https://api.anthropic.com/v1/messages \ + -H "content-type: application/json" \ + -H "x-api-key: $ANTHROPIC_API_KEY" \ + -H "anthropic-version: 2023-06-01" \ + -d "{ + \"model\": \"claude-sonnet-4-6\", + \"max_tokens\": 1024, + \"messages\": [{\"role\": \"user\", \"content\": $USER_JSON}], + \"system\": $SYSTEM_JSON, + \"output_config\": { + \"format\": { + \"type\": \"json_schema\", + \"schema\": { + \"type\": \"object\", + \"properties\": { + \"status\": {\"type\": \"string\", \"enum\": [\"PASS\", \"FAIL\"]}, + \"reasoning\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}}, + \"files_to_update\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}} + }, + \"required\": [\"status\", \"reasoning\", \"files_to_update\"], + \"additionalProperties\": false + } + } + } + }") + + HTTP_CODE=$(echo "$RESPONSE" | tail -1) + BODY=$(echo "$RESPONSE" | sed '$d') + echo "HTTP status: $HTTP_CODE" + echo "::endgroup::" + + if [ "$HTTP_CODE" != "200" ]; then + echo "::warning::Claude API returned HTTP $HTTP_CODE — skipping doc review" + echo "Skipped: API error (HTTP $HTTP_CODE)" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "$BODY" | head -10 >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + exit 0 + fi + + # Parse structured JSON response + REVIEW_JSON=$(echo "$BODY" | jq -r '.content[0].text // empty') + + if [ -z "$REVIEW_JSON" ]; then + echo "::warning::Empty response from Claude API — skipping doc review" + echo "Skipped: empty API response" >> $GITHUB_STEP_SUMMARY + echo "Raw response:" + echo "$BODY" | head -20 + exit 0 + fi + + echo "::group::AI Review Result" + echo "$REVIEW_JSON" | jq . + echo "::endgroup::" + + STATUS=$(echo "$REVIEW_JSON" | jq -r '.status') + REASONING=$(echo "$REVIEW_JSON" | jq -r '.reasoning[]' 2>/dev/null) + FILES=$(echo "$REVIEW_JSON" | jq -r '.files_to_update[]' 2>/dev/null) + + echo "### Verdict: ${STATUS}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ -n "$REASONING" ]; then + echo "**Reasoning:**" >> $GITHUB_STEP_SUMMARY + echo "$REASONING" | while IFS= read -r line; do + echo "- $line" >> $GITHUB_STEP_SUMMARY + done + echo "" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$STATUS" = "FAIL" ] && [ -n "$FILES" ]; then + echo "**Files to update:**" >> $GITHUB_STEP_SUMMARY + echo "$FILES" | while IFS= read -r f; do + echo "- \`$f\`" >> $GITHUB_STEP_SUMMARY + done + echo "" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$STATUS" = "PASS" ]; then + echo "Documentation review passed." + elif [ "$STATUS" = "FAIL" ]; then + echo "::error::Documentation review failed — see summary for details" + exit 1 + else + echo "::warning::Unexpected status '${STATUS}' — skipping" + echo "Unexpected AI response status: ${STATUS}" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/next-release.yml b/.github/workflows/next-release.yml new file mode 100644 index 0000000000..e7535f8642 --- /dev/null +++ b/.github/workflows/next-release.yml @@ -0,0 +1,126 @@ +name: Update Next Release PR + +on: + pull_request: + types: [closed] + branches: [develop] + +permissions: + contents: read + pull-requests: write + +jobs: + update-next-release: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Update Next Release PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_URL: ${{ github.event.pull_request.html_url }} + PR_BODY: ${{ github.event.pull_request.body }} + REPO: ${{ github.repository }} + ALLOWED_REPOS: "rtk-ai/rtk" + run: | + set -euo pipefail + + URL_PATTERN="" + for repo in $ALLOWED_REPOS; do + URL_PATTERN="${URL_PATTERN}|https://github\\.com/${repo}/issues/[0-9]+" + done + URL_PATTERN="${URL_PATTERN#|}" + + if printf '%s' "$PR_TITLE" | grep -qiE '^feat'; then + SECTION="Feats" + elif printf '%s' "$PR_TITLE" | grep -qiE '^fix'; then + SECTION="Fix" + else + SECTION="Other" + fi + + ISSUE_REFS="" + if [ -n "$PR_BODY" ]; then + ISSUE_REFS=$(echo "$PR_BODY" \ + | grep -oiE "(closes|fixes|resolves):?\s+#[0-9]+|${URL_PATTERN}" \ + | grep -oE '#[0-9]+|issues/[0-9]+' \ + | sed 's|issues/|#|' \ + | sort -u \ + || true) + fi + + ENTRY="- ${PR_TITLE} [#${PR_NUMBER}](${PR_URL})" + if [ -n "$ISSUE_REFS" ]; then + CLOSES_PARTS="" + while IFS= read -r ref; do + [ -z "$ref" ] && continue + NUM="${ref#\#}" + ISSUE_URL="https://github.com/${REPO}/issues/${NUM}" + if [ -n "$CLOSES_PARTS" ]; then + CLOSES_PARTS="${CLOSES_PARTS}, [${ref}](${ISSUE_URL}) (to verify)" + else + CLOSES_PARTS="Closes [${ref}](${ISSUE_URL}) (to verify)" + fi + done <<< "$ISSUE_REFS" + ENTRY="${ENTRY} — ${CLOSES_PARTS}" + fi + + NEXT_PR=$(gh pr list \ + --repo "$REPO" \ + --label next-release \ + --base master \ + --head develop \ + --state open \ + --json number,body \ + --jq '.[0] // empty') + + NEXT_PR_NUMBER="" + if [ -n "$NEXT_PR" ]; then + NEXT_PR_NUMBER=$(echo "$NEXT_PR" | jq -r '.number') + fi + + if [ -z "$NEXT_PR_NUMBER" ]; then + TEMPLATE="### Feats + + ### Fix + + ### Other" + + PR_CREATE_URL=$(gh pr create \ + --repo "$REPO" \ + --base master \ + --head develop \ + --title "Next Release" \ + --label next-release \ + --body "$TEMPLATE") + + NEXT_PR_NUMBER=$(echo "$PR_CREATE_URL" | grep -oE '/pull/[0-9]+' | grep -oE '[0-9]+') + CURRENT_BODY="$TEMPLATE" + else + CURRENT_BODY=$(echo "$NEXT_PR" | jq -r '.body') + fi + + SECTION_HEADER="### ${SECTION}" + export ENTRY + if echo "$CURRENT_BODY" | grep -qF "$SECTION_HEADER"; then + UPDATED_BODY=$(echo "$CURRENT_BODY" | awk -v section="$SECTION_HEADER" ' + $0 == section { + print + print ENVIRON["ENTRY"] + next + } + { print } + ') + else + UPDATED_BODY="${CURRENT_BODY} + + ${SECTION_HEADER} + ${ENTRY}" + fi + + gh pr edit "$NEXT_PR_NUMBER" \ + --repo "$REPO" \ + --body "$UPDATED_BODY" + + echo "Updated Next Release PR #${NEXT_PR_NUMBER} — added entry to ### ${SECTION}" diff --git a/.github/workflows/pr-target-check.yml b/.github/workflows/pr-target-check.yml new file mode 100644 index 0000000000..ac3ec13666 --- /dev/null +++ b/.github/workflows/pr-target-check.yml @@ -0,0 +1,48 @@ +name: PR Target Branch Check + +on: + pull_request_target: + types: [opened, edited] + +jobs: + check-target: + runs-on: ubuntu-latest + permissions: {} + # Skip develop→master PRs (maintainer releases) + if: >- + github.event.pull_request.base.ref == 'master' && + github.event.pull_request.head.ref != 'develop' + steps: + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ secrets.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-pull-requests: write + + - name: Add wrong-base label and comment + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + script: | + const pr = context.payload.pull_request; + + // Add label + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + labels: ['wrong-base'] + }); + + // Post comment + const body = `Automatic message from CI checks : It seems like this branch is targeting the wrong branch, any contribution should target develop branch. + + See [CONTRIBUTING.md](https://github.com/rtk-ai/rtk/blob/master/CONTRIBUTING.md) for details.`; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: body + }); diff --git a/.gitignore b/.gitignore index 55c6f9f309..e5a4f1e677 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,18 @@ claudedocs # ContextCrawler: personal/AI workspace — never publish local/ .claude/ -.github/ +# Peer-review / intent-diff scratch (never commit) +.agy-intent-diff.patch +# Stray playwright-mcp dependency lockfile (not used by this Rust crate) +package-lock.json +# Playwright MCP browser-session captures +.playwright-mcp/ +# Python bytecode (hermes hooks) +__pycache__/ +*.pyc +# .github: ignore everything except shipped workflows +.github/* +!.github/workflows/ .rtk/ # bench harness output (tests/harness_standalone.rs) — see issue #29 diff --git a/docs/audits/HANDOVER-2026-05-22.md b/docs/audits/HANDOVER-2026-05-22.md deleted file mode 100644 index 812b6b1e86..0000000000 --- a/docs/audits/HANDOVER-2026-05-22.md +++ /dev/null @@ -1,101 +0,0 @@ -# Session Handover — 2026-05-22 - -State for the next session. Read top to bottom, then pick up at the checklist. - -## Where things stand - -`develop` is at `396838d`, clean, all work merged. Three bodies of work this session: - -1. **md-min markdown minifier — PARKED.** Built a tiered markdown stripper + spec - (brainstorm → peer-reviewed spec → TDD build). Real-tokenizer measurement - (`o200k_base`, 9 docs) showed only 1–3% lossless savings — BPE already - compresses whitespace for free. Killed on the data. PR #123 closed (not - merged); branch `feat/md-min-viability` kept as a parked artifact. Recorded - in project memory (`md-min-parked.md`) so it isn't re-proposed. - -2. **Upstream RTK sync.** RTK (`upstream` remote) was 20 commits ahead. Triaged: - ~2 PRs of real value, rest churn. Merged #124 (init copilot data-loss fix) - and #125 (log/wc/git-status/.env filter correctness). Ruff-cap port abandoned - — our fork already caps that output. - -3. **E2E review remediation — the main work.** A 4-agent review - (`docs/audits/2026-05-22-e2e-review.md`) found 5 top-tier + 13 IMPORTANT + - ~15 NICE. Remediation status below. - -## E2E audit remediation status - -**Top tier — 5/5 fixed and merged** (#126–#129): -- #126 startup SIGABRT (33 crashes/week) — SIGPIPE→SIG_DFL. -- #127 grep filter rejecting standard flags (62% of all parse-failures). -- #128 dotnet argv checker — MSBuild RCE flags; review caught 2 bypasses. -- #129 permission matching — token-aware + substitution-aware; 22-probe - adversarial review, no Deny bypass survived. - -**IMPORTANT tier — 11/13 fixed and merged** (#130–#133): -- #130 security: integrity binary-hook check, supply-chain budget/cap, - iso8601 misparse, downgrades O(n²) (SEC-I1–I4). -- #131 argv checkers: rake + node executed-config (CMD-I1/I2). -- #132 analytics accuracy: weighted avg, quota horizon, char tokens, - ccusage date (AN-I1–I4). -- #133 perf: VACUUM off the `record()` hot path (PERF-I1). - -**STILL OPEN — deferred deliberately:** -- **READ-I1** — the `read` filter delivers 13.7% savings (1,209 of 1,653 runs - at literal 0%, 15.5M-token sink). Not a batch patch — needs a diagnose-first - pass: is the filter not engaging, or does it need a redesign? Treat like - md-min — its own scoped effort with measurement. -- **GIT-I1** — `git log` filter delivers 1.9% over 790 runs vs the documented - 80%. Diagnose first: which flag variants bypass the filter. -- **~15 NICE-tier findings** — listed in the audit doc's "Feature / uplift - ideas" + the NICE sub-sections. Not triaged for action. -- **2 pre-existing follow-up bugs** noted in #127/#129 PR bodies: grep - single-file colon mis-display; `split_on_operators` paren-depth (no Deny - bypass — over-conservative Allow drops only). - -## Environment notes (important — these bit during the session) - -- **Commit signing**: 1Password SSH agent holds the key, but this kind of - session's `SSH_AUTH_SOCK` defaults to the empty macOS launchd agent. Git - SSH-signing reads `SSH_AUTH_SOCK` (ignores `~/.ssh/config` IdentityAgent). - Prefix every commit with: - `SSH_AUTH_SOCK="$HOME/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock"` - or signing fails. `git log %G?` shows `N` because no local - `gpg.ssh.allowedSignersFile` is configured — the commits ARE signed - (`git cat-file commit | grep gpgsig`); GitHub verifies server-side. - Commits earlier in the session (#121/#122, md-min, #124/#125) are unsigned - — agent was locked then; not re-signing merged history. -- **Full test suite**: `cargo test --bins` run normally dies silently - (exit 144, 0-byte output) — a sandbox/background interaction. Working - technique: `cargo test --bins --no-run` (build the test binary), then run - the binary directly: - `CARGO_PKG_NAME=contextcrawler CARGO_MANIFEST_DIR="$PWD" --test-threads=8`. - The two CARGO env vars are required — `is_test_context()` needs them or 3 - `core::tracking` tests fail (they assert test-DB redirection). Find the - binary: newest `target/debug/deps/contextcrawler-*` executable (no extension). -- **Sandboxed cargo** fails silently — pass `dangerouslyDisableSandbox` for - build/test. -- **CI** on this repo is CodeQL + Analyze (rust/python/js) only — it does NOT - run `cargo test`. The full suite must be validated locally. Current - `develop` tip: 2,533 tests pass, 0 fail, clippy clean. -- **Remotes**: `origin` = the fork (`thehoff/contextcrawler`), `upstream` = - RTK (`rtk-ai/rtk`, push disabled via `no_push`). Push needs the gh - credential helper: `git -c credential.helper='!gh auth git-credential' push`. -- **Workflow that worked well**: brainstorm→spec→plan→subagent-driven TDD with - two-stage review per task; for batch fixes, group by concern into branches, - one implementer subagent per branch, code-reviewer subagent per branch. - The adversarial code review repeatedly caught real bypasses the first - implementation missed (dotnet ×2, rake glued-form, integrity foreign-path). - Cap parallel agents at 3–4. - -## Next-session checklist - -1. Decide the deferred filter-uplift pair (READ-I1, GIT-I1). Recommended: - one diagnose-first agent per filter — find *why* savings are low before - committing to a fix vs a redesign. `git log` is likely the quicker win. -2. Or take the NICE tier from `docs/audits/2026-05-22-e2e-review.md` — has - data-grounded feature ideas (`gain --weak-filters`, missing `ps`/`rsync`/`du` - filters, stop logging 5,252 supply-chain skip records = 11.5MB JSONL noise). -3. Optional housekeeping: configure `gpg.ssh.allowedSignersFile` so - `git log --show-signature` verifies locally (GitHub already does). -4. The parked `feat/md-min-viability` branch can be deleted if md-min is - definitively dead — currently kept as a research artifact. diff --git a/src/core/mod.rs b/src/core/mod.rs index 01317e9425..f1cc2475f5 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -5,6 +5,7 @@ pub mod constants; pub mod display_helpers; pub mod filter; pub mod runner; +pub mod secret_redact; pub mod stream; pub mod tee; pub mod telemetry; diff --git a/src/core/secret_redact.rs b/src/core/secret_redact.rs new file mode 100644 index 0000000000..51417a5776 --- /dev/null +++ b/src/core/secret_redact.rs @@ -0,0 +1,263 @@ +//! Redact common secret shapes from a shell command string before it lands in +//! an audit log on disk. Belt-and-braces: the gate logs are user-readable and +//! sit at predictable paths, so any token captured verbatim is a leak. +//! +//! Conservative on purpose. The goal is to scrub obvious credentials +//! (`Authorization` headers, GitHub PATs, env-var assignments to +//! `*TOKEN`/`*KEY`/`*SECRET`/etc., URL basic-auth) without mangling normal +//! shell commands. False negatives are preferred over false positives that +//! corrupt the diagnostic value of the log. +//! +//! Known limitations (intentional false negatives): +//! - Bare-shape secrets like `T=<40-hex>` (one-letter alias to a token) are +//! not redacted: a name-only heuristic can't safely distinguish a token +//! value from a git SHA without context. If a downstream consumer reuses +//! the alias in an `Authorization` header within the same cmd string, the +//! header-side match still scrubs that occurrence. + +use lazy_static::lazy_static; +use regex::Regex; +use std::borrow::Cow; + +lazy_static! { + /// Each entry is `(pattern, replacement)`. Order matters: more specific + /// patterns run first so a generic match doesn't shadow a structured one. + static ref PATTERNS: Vec<(Regex, &'static str)> = vec![ + // 1. URL basic-auth (`https://user:password@host`). + // Redact both segments — the username can leak who the request was for. + ( + Regex::new(r"(?Phttps?://)[^:\s/@]+:[^@\s/]+@").unwrap(), + "${scheme}:@", + ), + // 2. `Authorization: token ` and `Authorization: Bearer `. + ( + Regex::new(r"(?i)(?PAuthorization\s*:\s*(?:token|bearer)\s+)\S+").unwrap(), + "${hdr}", + ), + // 3. GitHub PATs by prefix shape. Whole match (prefix+value) is the secret. + ( + Regex::new(r"\b(?:gh[opsu]_|github_pat_)[A-Za-z0-9_]{16,}").unwrap(), + "", + ), + // 4. Inline env-var assignment to credential-shaped names. Case-insensitive + // to catch `TEA_TOKEN=`, `my_secret=`, `Api_Key=`. Suffix list is the + // allow-redact set; anything else (e.g. PATH, HOME) is untouched. + // Note: matches `(?:_|^)NAME` to avoid eating substrings like "BROKEN". + // Value is anything up to whitespace/quote/semicolon. + ( + Regex::new( + r#"(?xi) + (?P + \b + [a-z][a-z0-9_]*? + _(?:token|key|secret|password|pat|apikey|auth) + | + \b(?:token|key|secret|password|pat|apikey|auth) + ) + = + (?P[^\s'";]+) + "# + ) + .unwrap(), + "${name}=", + ), + // 5. git-credential-helper format inside a piped string: + // `protocol=http\nhost=...\nusername=...\npassword=` + // The literal `\n` puts the secret name mid-word from the regex + // engine's POV, so `\b` (pattern 4) doesn't anchor. Match the + // escape-prefix explicitly and preserve it. + ( + Regex::new( + r#"(?xi) + (?P\\n|\\r) + (?Ppassword|token|secret|auth) + = + (?P[^\s'";\\]+) + "# + ) + .unwrap(), + "${pfx}${name}=", + ), + // 5. CLI flags carrying credentials. + // `--token foo`, `--token=foo`, `--password=foo`, `--api-key foo`, etc. + // Space-separated or `=`-attached. Captures the whole flag-name segment + // so it's preserved in the replacement. + ( + Regex::new( + r#"(?xi) + (?P + --(?:auth[-_])?(?:token|password|api[-_]?key|secret) + [\s=]+ + ) + (?P[^\s'";]+) + "# + ) + .unwrap(), + "${flag}", + ), + ]; +} + +/// Redact secrets from `cmd`. Returns `Cow::Borrowed` if nothing matched +/// (zero-copy fast path) and `Cow::Owned` if any replacement was applied. +pub fn redact(cmd: &str) -> Cow<'_, str> { + let mut current: Cow<'_, str> = Cow::Borrowed(cmd); + for (re, replacement) in PATTERNS.iter() { + let after = re.replace_all(¤t, *replacement); + if let Cow::Owned(s) = after { + current = Cow::Owned(s); + } + } + current +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_secrets_returns_borrowed() { + let cmd = "git status -sb && cargo test --workspace"; + match redact(cmd) { + Cow::Borrowed(s) => assert_eq!(s, cmd), + Cow::Owned(_) => panic!("expected zero-copy on a benign cmd"), + } + } + + #[test] + fn authorization_token_header_is_redacted() { + let cmd = r#"curl -H "Authorization: token 147dd871c9edab5848377af412b6575bca133169" https://x/api"#; + let out = redact(cmd); + assert!(!out.contains("147dd871"), "token leaked: {}", out); + assert!(out.contains("Authorization: token "), "header malformed: {}", out); + } + + #[test] + fn authorization_bearer_header_is_redacted() { + let cmd = r#"curl -H "Authorization: Bearer eyJ.abc.def" https://api"#; + let out = redact(cmd); + assert!(!out.contains("eyJ.abc.def")); + assert!(out.contains("Authorization: Bearer ")); + } + + #[test] + fn github_pat_prefixes_are_redacted() { + for prefix in &["ghp_", "gho_", "ghs_", "ghu_", "github_pat_"] { + let token = format!("{}abcdef0123456789ABCdef", prefix); + let cmd = format!("gh auth login --with-token {}", token); + let out = redact(&cmd); + assert!(!out.contains(&token), "leaked {}: {}", prefix, out); + assert!(out.contains("")); + } + } + + #[test] + fn env_var_token_assignment_is_redacted() { + let cmd = "TEA_TOKEN=147dd871c9edab5848377af412b6575bca133169 tea repos list"; + let out = redact(cmd); + assert!(!out.contains("147dd871"), "leaked: {}", out); + assert!(out.contains("TEA_TOKEN=")); + // Rest of the command is preserved. + assert!(out.contains("tea repos list")); + } + + #[test] + fn env_var_bare_token_assignment_is_redacted() { + let cmd = "TOKEN=abc123 do_thing"; + let out = redact(cmd); + assert!(!out.contains("abc123")); + assert!(out.contains("TOKEN=")); + } + + #[test] + fn env_var_lowercase_secret_is_redacted() { + let cmd = "my_secret=hunter2 ./run"; + let out = redact(cmd); + assert!(!out.contains("hunter2")); + } + + #[test] + fn path_and_home_are_not_redacted() { + let cmd = "PATH=/usr/bin:/bin HOME=/Users/x ./tool"; + let out = redact(cmd); + assert_eq!(out, cmd, "innocuous env vars must not be touched"); + } + + #[test] + fn cli_flag_token_space_separated_is_redacted() { + let cmd = "myapp --token deadbeef123 --verbose"; + let out = redact(cmd); + assert!(!out.contains("deadbeef123")); + assert!(out.contains("")); + assert!(out.contains("--verbose")); + } + + #[test] + fn cli_flag_token_equals_is_redacted() { + let cmd = "myapp --auth-token=deadbeef123 --verbose"; + let out = redact(cmd); + assert!(!out.contains("deadbeef123")); + } + + #[test] + fn cli_flag_password_is_redacted() { + let cmd = "psql --password=letmein -h db.example"; + let out = redact(cmd); + assert!(!out.contains("letmein")); + } + + #[test] + fn url_basic_auth_is_redacted() { + let cmd = "git clone https://user:supersecret@github.com/foo/bar.git"; + let out = redact(cmd); + assert!(!out.contains("supersecret")); + assert!(!out.contains("user:supersecret@")); + assert!(out.contains(":@github.com")); + } + + #[test] + fn redaction_is_idempotent() { + let cmd = concat!( + r#"TEA_TOKEN=147dd871 curl -H "Authorization: token abc" "#, + r#"https://user:pw@host/x && app --token x ghp_abcdef0123456789ABCDEF"# + ); + let once = redact(cmd).to_string(); + let twice = redact(&once).to_string(); + assert_eq!(once, twice, "redactor must be idempotent"); + // Smoke-check that every secret was caught at least once. + for needle in &["147dd871", "abc\"", "supersecret", "pw@", "ghp_abcdef"] { + assert!(!once.contains(needle), "leaked {:?}: {}", needle, once); + } + } + + #[test] + fn multiple_secrets_in_one_cmd_all_redacted() { + let cmd = "TEA_TOKEN=aaa MY_API_KEY=bbb curl -H 'Authorization: token ccc' https://x"; + let out = redact(cmd); + for needle in &["aaa", "bbb", "ccc"] { + assert!(!out.contains(needle), "leaked {}: {}", needle, out); + } + } + + #[test] + fn git_credential_helper_password_redacted() { + // git credential-osxkeychain / credential-store / cache feed a stream + // like `protocol=...\nhost=...\nusername=...\npassword=` over a + // pipe. The literal `\n` defeats `\b`-anchored env-var matching. + let cmd = r#"printf "protocol=http\nhost=gitea.example\nusername=alice\npassword=147dd871c9edab5848377af412b6575bca133169\n\n" | git credential-store"#; + let out = redact(cmd); + assert!(!out.contains("147dd871"), "leaked: {}", out); + assert!(out.contains(r"\npassword=")); + // Preserves benign neighbouring assignments. + assert!(out.contains(r"\nusername=alice")); + assert!(out.contains(r"\nhost=gitea.example")); + } + + #[test] + fn benign_text_with_token_word_unchanged() { + // "token" appearing in prose, not as a credential, must not trigger anything. + let cmd = "echo 'how token rotation works'"; + let out = redact(cmd); + assert_eq!(out, cmd); + } +} diff --git a/src/hooks/supply_chain_gate.rs b/src/hooks/supply_chain_gate.rs index fcc86ca5ab..eed50a3291 100644 --- a/src/hooks/supply_chain_gate.rs +++ b/src/hooks/supply_chain_gate.rs @@ -1571,32 +1571,102 @@ fn read_body(resp: ureq::Response) -> Result, String> { Ok(buf) } +/// Per-attempt HTTP timeout. With one retry on transient errors, total +/// wall-clock per call is bounded by `2 * HTTP_ATTEMPT_TIMEOUT + +/// HTTP_RETRY_BACKOFF`. Kept well under `CHECK_WALL_BUDGET` so a single +/// slow package can't blow the whole install's budget. +const HTTP_ATTEMPT_TIMEOUT: StdDuration = StdDuration::from_secs(5); +const HTTP_MAX_RETRIES: u32 = 1; +const HTTP_RETRY_BACKOFF: StdDuration = StdDuration::from_millis(250); + +/// Lightweight tag for classifying an HTTP failure. Separated from +/// `ureq::Error` so the policy can be unit-tested without constructing a +/// real `ureq::Response`/`ureq::Transport`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HttpErrTag { + Status(u16), + Transport, +} + +/// Should we retry after a failure of shape `tag`? Yes for transport-level +/// failures (DNS hiccup, connection reset, read timeout) and 5xx responses +/// (registry unhealthy, transient overload). No for 4xx — those signal a +/// terminal problem with the request (package missing, auth, rate-limit) +/// where an immediate retry just adds load and won't change the outcome. +fn is_retryable_http_err_tag(tag: HttpErrTag) -> bool { + match tag { + HttpErrTag::Status(code) => (500..600).contains(&code), + HttpErrTag::Transport => true, + } +} + +fn is_retryable_http_err(e: &ureq::Error) -> bool { + let tag = match e { + ureq::Error::Status(code, _) => HttpErrTag::Status(*code), + ureq::Error::Transport(_) => HttpErrTag::Transport, + }; + is_retryable_http_err_tag(tag) +} + fn http_get_json(url: &str) -> Result { - let mut req = ureq::get(url) - .set("User-Agent", "contextcrawler-supply-chain-gate/0.1") - .timeout(StdDuration::from_secs(8)); - // Request npm's abbreviated metadata where applicable — ~100x smaller. - // PyPI ignores the header, so it is safe to send unconditionally for npm - // hosts only. - if url.starts_with("https://registry.npmjs.org/") { - req = req.set("Accept", NPM_ABBREVIATED_ACCEPT); - } - let resp = req - .call() - .map_err(|e| format!("HTTP {}: {}", url, e))?; - let buf = read_body(resp)?; - serde_json::from_slice(&buf).map_err(|e| e.to_string()) + let mut last_err: Option = None; + for attempt in 0..=HTTP_MAX_RETRIES { + if attempt > 0 { + std::thread::sleep(HTTP_RETRY_BACKOFF); + } + let mut req = ureq::get(url) + .set("User-Agent", "contextcrawler-supply-chain-gate/0.1") + .timeout(HTTP_ATTEMPT_TIMEOUT); + // Request npm's abbreviated metadata where applicable — ~100x smaller. + // PyPI ignores the header, so it is safe to send unconditionally for npm + // hosts only. + if url.starts_with("https://registry.npmjs.org/") { + req = req.set("Accept", NPM_ABBREVIATED_ACCEPT); + } + match req.call() { + Ok(resp) => { + let buf = read_body(resp)?; + return serde_json::from_slice(&buf).map_err(|e| e.to_string()); + } + Err(e) => { + let retryable = is_retryable_http_err(&e); + last_err = Some(format!("HTTP {}: {}", url, e)); + if !retryable { + break; + } + } + } + } + Err(last_err.unwrap_or_else(|| format!("HTTP {}: no attempts", url))) } fn http_post_json(url: &str, body: &Value) -> Result { - let resp = ureq::post(url) - .set("User-Agent", "contextcrawler-supply-chain-gate/0.1") - .set("Content-Type", "application/json") - .timeout(StdDuration::from_secs(8)) - .send_string(&body.to_string()) - .map_err(|e| format!("HTTP {}: {}", url, e))?; - let buf = read_body(resp)?; - serde_json::from_slice(&buf).map_err(|e| e.to_string()) + let payload = body.to_string(); + let mut last_err: Option = None; + for attempt in 0..=HTTP_MAX_RETRIES { + if attempt > 0 { + std::thread::sleep(HTTP_RETRY_BACKOFF); + } + let resp = ureq::post(url) + .set("User-Agent", "contextcrawler-supply-chain-gate/0.1") + .set("Content-Type", "application/json") + .timeout(HTTP_ATTEMPT_TIMEOUT) + .send_string(&payload); + match resp { + Ok(r) => { + let buf = read_body(r)?; + return serde_json::from_slice(&buf).map_err(|e| e.to_string()); + } + Err(e) => { + let retryable = is_retryable_http_err(&e); + last_err = Some(format!("HTTP {}: {}", url, e)); + if !retryable { + break; + } + } + } + } + Err(last_err.unwrap_or_else(|| format!("HTTP {}: no attempts", url))) } /// Resolve (version, publish_time) for the package. If `pinned` is Some, use @@ -1870,6 +1940,47 @@ fn cache_put(eco: Ecosystem, pkg: &str, version: &str, publish: &DateTime) } } +/// Does `cmd` open with a leading `NAME=VALUE` assignment (possibly preceded +/// by sibling assignments and whitespace), and is that assignment for `name` +/// with a value in `allowed`? +/// +/// Used so `CONTEXTCRAWLER_SUPPLY_CHAIN=off pip install x` bypasses the +/// gate the same way the user reads the hint suggests. Only the *leading* +/// run of assignments counts — once we see a non-assignment token, the +/// rest of the cmd is ignored. (Mid-cmd `&& FOO=bar baz` does not bypass.) +/// +/// Conservative on value parsing: unquoted values are anything up to the +/// next whitespace; quoted values are not supported in v1 (the bypass +/// values we care about are short bareword tokens like `off`/`0`/`false`). +fn cmd_has_leading_assignment(cmd: &str, name: &str, allowed: &[&str]) -> bool { + let mut rest = cmd.trim_start(); + while !rest.is_empty() { + // Pull off the next whitespace-delimited token. + let tok_end = rest + .find(char::is_whitespace) + .unwrap_or(rest.len()); + let token = &rest[..tok_end]; + // POSIX-shape assignment: NAME=VALUE where NAME is identifier-safe. + let Some(eq_idx) = token.find('=') else { + // First non-assignment token ends the leading run. + return false; + }; + let (n, rhs) = (&token[..eq_idx], &token[eq_idx + 1..]); + if n.is_empty() + || !n.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + || !n.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + { + return false; + } + if n == name && allowed.iter().any(|v| *v == rhs) { + return true; + } + // Advance past this assignment + any whitespace before the next token. + rest = rest[tok_end..].trim_start(); + } + false +} + // --------------------------------------------------------------------------- // Public entry points // --------------------------------------------------------------------------- @@ -1883,6 +1994,18 @@ pub fn check(cmd: &str) -> Verdict { if std::env::var("CONTEXTCRAWLER_SUPPLY_CHAIN").as_deref() == Ok("off") { return Verdict::Skip; } + // Also honour an inline leading-prefix bypass, which is what the gate's + // own error messages suggest (`Overrides: rerun with + // CONTEXTCRAWLER_SUPPLY_CHAIN=off …`). Without this branch the user + // sees the hint, runs the suggested form, and is still blocked — the + // inline assignment scopes to the subprocess, not to this hook. See #181. + if cmd_has_leading_assignment( + cmd, + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off", "0", "false", "no"], + ) { + return Verdict::Skip; + } let installs = detect_installs(cmd); if installs.is_empty() { @@ -2132,11 +2255,13 @@ pub fn log_event(cmd: &str, verdict: &Verdict) { Verdict::Block(f) | Verdict::Ask(f) => serde_json::to_string(f).unwrap_or_default(), _ => "[]".to_string(), }; + // Scrub credentials before the cmd lands on disk. See issue #180. + let safe_cmd = crate::core::secret_redact::redact(cmd); let record = format!( r#"{{"ts":"{}","verdict":"{}","cmd":{},"findings":{}}}"#, Utc::now().to_rfc3339(), kind, - serde_json::to_string(cmd).unwrap_or_else(|_| "\"\"".into()), + serde_json::to_string(safe_cmd.as_ref()).unwrap_or_else(|_| "\"\"".into()), findings ); if let Ok(mut f) = fs::OpenOptions::new().create(true).append(true).open(&path) { @@ -3759,4 +3884,159 @@ mod tests { // muddied if cd ever gains install-shaped argv). assert!(!command_head_is_data_utility("cd foo")); } + + #[test] + fn inline_bypass_simple() { + // The exact form the gate's own error message tells users to run. + assert!(cmd_has_leading_assignment( + "CONTEXTCRAWLER_SUPPLY_CHAIN=off pip install starlette==0.49.1", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off"], + )); + } + + #[test] + fn inline_bypass_value_variants() { + for v in &["off", "0", "false", "no"] { + let cmd = format!("CONTEXTCRAWLER_SUPPLY_CHAIN={} pip install x", v); + assert!( + cmd_has_leading_assignment( + &cmd, + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off", "0", "false", "no"], + ), + "should recognise value {}", + v + ); + } + } + + #[test] + fn inline_bypass_with_sibling_assignments() { + // Real-world: `FOO=bar CONTEXTCRAWLER_SUPPLY_CHAIN=off cmd`. + assert!(cmd_has_leading_assignment( + "FOO=bar CONTEXTCRAWLER_SUPPLY_CHAIN=off pip install x", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off"], + )); + // Whitespace tolerance. + assert!(cmd_has_leading_assignment( + " FOO=bar CONTEXTCRAWLER_SUPPLY_CHAIN=off pip install x", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off"], + )); + } + + #[test] + fn inline_bypass_mid_cmd_does_not_count() { + // Bypass must be PREFIX, not mid-cmd. `&&` is not a sibling + // assignment, so once we hit `pip` the leading run ends. + assert!(!cmd_has_leading_assignment( + "pip install x && CONTEXTCRAWLER_SUPPLY_CHAIN=off other", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off"], + )); + } + + #[test] + fn inline_bypass_wrong_value_does_not_count() { + // Defensive: user typed `=on` thinking it enables — must NOT bypass. + assert!(!cmd_has_leading_assignment( + "CONTEXTCRAWLER_SUPPLY_CHAIN=on pip install x", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off", "0", "false", "no"], + )); + } + + #[test] + fn inline_bypass_value_must_match_exactly() { + // `=offsuffix` is not `=off`. + assert!(!cmd_has_leading_assignment( + "CONTEXTCRAWLER_SUPPLY_CHAIN=offsuffix pip install x", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off"], + )); + } + + #[test] + fn inline_bypass_invalid_identifier_breaks_run() { + // `1NAME=` is not a valid identifier — should stop the leading run. + assert!(!cmd_has_leading_assignment( + "1NAME=x CONTEXTCRAWLER_SUPPLY_CHAIN=off pip install x", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off"], + )); + } + + #[test] + fn inline_bypass_empty_cmd() { + assert!(!cmd_has_leading_assignment( + "", + "CONTEXTCRAWLER_SUPPLY_CHAIN", + &["off"], + )); + } + + #[test] + fn http_err_5xx_is_retryable() { + for code in [500u16, 502, 503, 504, 599] { + assert!( + is_retryable_http_err_tag(HttpErrTag::Status(code)), + "5xx must be retryable: {}", + code + ); + } + } + + #[test] + fn http_err_4xx_is_not_retryable() { + // 404 = package doesn't exist; 401/403 = auth; 429 = rate-limit + // (we don't retry that here — would just add load against the same + // limiter; a separate cooldown could be future work). + for code in [400u16, 401, 403, 404, 422, 429] { + assert!( + !is_retryable_http_err_tag(HttpErrTag::Status(code)), + "4xx must NOT be retryable: {}", + code + ); + } + } + + #[test] + fn http_err_2xx_and_3xx_are_not_retryable() { + // Defensive: a 2xx/3xx shouldn't ever reach the classifier + // (ureq returns Ok for those), but if it did, treat as terminal — + // we don't want a bug to spin retries on a successful response. + for code in [200u16, 201, 204, 301, 302, 304] { + assert!(!is_retryable_http_err_tag(HttpErrTag::Status(code))); + } + } + + #[test] + fn http_err_transport_is_retryable() { + // DNS hiccups, TCP resets, read timeouts — the cases the user's + // 5-unavailables-in-a-day issue traced back to. + assert!(is_retryable_http_err_tag(HttpErrTag::Transport)); + } + + #[test] + fn http_retry_constants_within_budget() { + // Per-attempt timeout × (1 + max_retries) + backoff × max_retries + // must comfortably fit inside CHECK_WALL_BUDGET so a single slow + // call cannot push the whole install past the deadline. + let worst_call = HTTP_ATTEMPT_TIMEOUT + .saturating_mul(1 + HTTP_MAX_RETRIES) + .saturating_add(HTTP_RETRY_BACKOFF.saturating_mul(HTTP_MAX_RETRIES)); + assert!( + worst_call < CHECK_WALL_BUDGET, + "worst-case per-call ({:?}) must be less than CHECK_WALL_BUDGET ({:?})", + worst_call, + CHECK_WALL_BUDGET + ); + // And the budget can still service at least two slow packages. + assert!( + worst_call * 2 < CHECK_WALL_BUDGET.saturating_add(StdDuration::from_secs(5)), + "budget must still cover ≥2 retried calls in one check" + ); + } } diff --git a/src/hooks/tirith_gate.rs b/src/hooks/tirith_gate.rs index 188013d780..b12168d7b7 100644 --- a/src/hooks/tirith_gate.rs +++ b/src/hooks/tirith_gate.rs @@ -148,20 +148,28 @@ pub fn log_downgrade(cmd: &str, reason: &'static str, tirith_json: Option<&str>) } let path = dir.join("downgrades.jsonl"); + // Scrub credentials before the cmd lands on disk. See issue #180. + let safe_cmd = crate::core::secret_redact::redact(cmd); + // The tirith blob frequently echoes the command (and any inline + // credentials) back in its findings, so scrub it too. The redactor is + // structure-preserving — it only touches matched substrings — so the + // surrounding JSON shape remains valid. + let safe_tirith = tirith_json.map(|j| crate::core::secret_redact::redact(j)); + let timestamp = chrono::Utc::now().to_rfc3339(); - let record = match tirith_json { + let record = match safe_tirith.as_deref() { Some(json) => format!( r#"{{"ts":"{}","reason":"{}","cmd":{},"tirith":{}}}"#, timestamp, reason, - json_escape(cmd), + json_escape(&safe_cmd), json.trim(), ), None => format!( r#"{{"ts":"{}","reason":"{}","cmd":{}}}"#, timestamp, reason, - json_escape(cmd), + json_escape(&safe_cmd), ), }; @@ -192,6 +200,159 @@ pub fn tirith_binary_path() -> Option { } } +/// Run the `contextcrawler security --scrub-logs` action: scan both the +/// tirith downgrade log and the supply-chain event log, deep-redact every +/// string field via `core::secret_redact::redact`, and rewrite atomically. +/// A timestamped backup is written alongside each rewritten file. With +/// `dry_run`, no file is touched — only counts are reported. +/// +/// Returns the process exit code (0 on success). +pub fn run_scrub_logs(dry_run: bool) -> anyhow::Result { + let data_dir = dirs::data_local_dir().ok_or_else(|| { + anyhow::anyhow!("could not resolve data_local_dir (XDG_DATA_HOME or platform equivalent)") + })?; + let log_dir = data_dir.join("contextcrawler"); + let report = scrub_logs_in(&log_dir, dry_run)?; + println!( + "ContextCrawler audit log scrub — {}", + if dry_run { "DRY RUN" } else { "live" } + ); + println!("════════════════════════════════════════════════════════════"); + println!("log dir: {}", log_dir.display()); + println!(); + for f in &report.files { + if f.skipped { + println!(" {}: not present, skipping", f.name); + continue; + } + println!( + " {}: lines={} changed={} unparseable={}", + f.name, f.total, f.changed, f.unparseable + ); + if let Some(bak) = &f.backup_path { + println!(" backup: {}", bak.display()); + } + } + println!(); + println!( + "Summary: {} lines processed, {} changed{}", + report.grand_total, + report.grand_changed, + if dry_run { " (no files written)" } else { "" } + ); + Ok(0) +} + +/// Per-file outcome from a scrub pass. +#[derive(Debug, Default)] +pub struct ScrubFileReport { + pub name: String, + pub skipped: bool, + pub total: usize, + pub changed: usize, + pub unparseable: usize, + pub backup_path: Option, +} + +#[derive(Debug, Default)] +pub struct ScrubReport { + pub files: Vec, + pub grand_total: usize, + pub grand_changed: usize, +} + +/// Core of `run_scrub_logs`, lifted out so tests can drive it against a +/// tempdir instead of the real `~/Library/Application Support/contextcrawler`. +pub fn scrub_logs_in( + log_dir: &std::path::Path, + dry_run: bool, +) -> anyhow::Result { + use crate::core::secret_redact::redact; + use chrono::Utc; + use serde_json::Value; + use std::io::{BufRead, BufReader, Write}; + + fn deep_redact(v: &mut Value) { + match v { + Value::String(s) => { + let r = redact(s); + if let std::borrow::Cow::Owned(new) = r { + *s = new; + } + } + Value::Array(a) => a.iter_mut().for_each(deep_redact), + Value::Object(m) => m.values_mut().for_each(deep_redact), + _ => {} + } + } + + let targets = ["downgrades.jsonl", "supply_chain.jsonl"]; + let stamp = Utc::now().format("%Y%m%d-%H%M%S").to_string(); + let mut report = ScrubReport::default(); + + for name in &targets { + let mut entry = ScrubFileReport { + name: (*name).to_string(), + ..Default::default() + }; + let path = log_dir.join(name); + if !path.exists() { + entry.skipped = true; + report.files.push(entry); + continue; + } + let src = std::fs::File::open(&path)?; + let reader = BufReader::new(src); + let tmp_path = path.with_extension("jsonl.scrub-tmp"); + let mut out_buf: Vec = Vec::new(); + for line in reader.lines() { + let line = line?; + if line.is_empty() { + out_buf.extend_from_slice(b"\n"); + continue; + } + entry.total += 1; + match serde_json::from_str::(&line) { + Ok(mut v) => { + let before = v.clone(); + deep_redact(&mut v); + if v != before { + entry.changed += 1; + } + let serialised = + serde_json::to_string(&v).unwrap_or_else(|_| line.clone()); + out_buf.extend_from_slice(serialised.as_bytes()); + out_buf.push(b'\n'); + } + Err(_) => { + entry.unparseable += 1; + let r = redact(&line); + if r.as_ref() != line { + entry.changed += 1; + } + out_buf.extend_from_slice(r.as_bytes()); + out_buf.push(b'\n'); + } + } + } + report.grand_total += entry.total; + report.grand_changed += entry.changed; + if !dry_run { + { + let mut tmp = std::fs::File::create(&tmp_path)?; + tmp.write_all(&out_buf)?; + tmp.sync_all()?; + } + let bak = path.with_file_name(format!("{}.bak-{}", name, stamp)); + std::fs::copy(&path, &bak)?; + std::fs::rename(&tmp_path, &path)?; + entry.backup_path = Some(bak); + } + report.files.push(entry); + } + Ok(report) +} + /// Returns the path where downgrade events are appended (whether or not /// the file exists yet). Mirrors the resolution in `log_downgrade`. pub fn downgrades_log_path() -> Option { @@ -478,4 +639,69 @@ mod tests { "last record must be the most recent line" ); } + + #[test] + fn scrub_logs_in_strips_credentials_and_backs_up() { + let dir = tempfile::TempDir::new().unwrap(); + let down = dir.path().join("downgrades.jsonl"); + let supply = dir.path().join("supply_chain.jsonl"); + std::fs::write( + &down, + concat!( + r#"{"ts":"2026-05-26T00:00:00Z","reason":"tirith_block","cmd":"TEA_TOKEN=147dd871c9edab5848377af412b6575bca133169 curl -H \"Authorization: token 147dd871c9edab5848377af412b6575bca133169\" https://x","tirith":{"action":"block","evidence":[{"raw":"Authorization: token 147dd871c9edab5848377af412b6575bca133169"}]}}"#, + "\n", + ), + ) + .unwrap(); + std::fs::write( + &supply, + concat!( + r#"{"ts":"2026-05-26T00:00:00Z","verdict":"skip","cmd":"echo MY_API_KEY=abc123 && curl -H 'Authorization: Bearer eyJ.tok.en' https://x","findings":[]}"#, + "\n", + ), + ) + .unwrap(); + + let report = scrub_logs_in(dir.path(), false).unwrap(); + assert_eq!(report.grand_total, 2); + assert_eq!(report.grand_changed, 2, "both lines must be scrubbed"); + assert!(report.files.iter().all(|f| f.backup_path.is_some())); + + let down_new = std::fs::read_to_string(&down).unwrap(); + let supply_new = std::fs::read_to_string(&supply).unwrap(); + assert!(!down_new.contains("147dd871"), "leaked: {}", down_new); + assert!(!down_new.contains("Authorization: token 147")); + assert!(!supply_new.contains("abc123")); + assert!(!supply_new.contains("eyJ.tok.en")); + assert!(supply_new.contains("MY_API_KEY=")); + // Surrounding JSON shape preserved. + assert!(down_new.contains(r#""reason""#)); + assert!(supply_new.contains(r#""verdict""#)); + } + + #[test] + fn scrub_logs_in_dry_run_does_not_write() { + let dir = tempfile::TempDir::new().unwrap(); + let down = dir.path().join("downgrades.jsonl"); + let leaky = concat!( + r#"{"ts":"x","reason":"r","cmd":"TEA_TOKEN=abc curl https://x"}"#, + "\n", + ); + std::fs::write(&down, leaky).unwrap(); + let before = std::fs::read_to_string(&down).unwrap(); + let report = scrub_logs_in(dir.path(), true).unwrap(); + let after = std::fs::read_to_string(&down).unwrap(); + assert_eq!(before, after, "dry-run must not mutate the file"); + assert_eq!(report.grand_changed, 1, "dry-run still reports counts"); + assert!(report.files.iter().all(|f| f.backup_path.is_none())); + } + + #[test] + fn scrub_logs_in_skips_missing_files() { + let dir = tempfile::TempDir::new().unwrap(); + let report = scrub_logs_in(dir.path(), false).unwrap(); + assert!(report.files.iter().all(|f| f.skipped)); + assert_eq!(report.grand_total, 0); + assert_eq!(report.grand_changed, 0); + } } diff --git a/src/main.rs b/src/main.rs index 0901e3d5a4..0a5891f6b1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -740,6 +740,14 @@ enum Commands { /// Emit machine-readable JSON instead of the human-readable dashboard #[arg(long)] json: bool, + /// Scrub credentials from existing audit logs in place. + /// Writes a `.bak-` backup alongside each rewritten file. + /// Pair with `--dry-run` to preview without writing. + #[arg(long, conflicts_with_all = ["all", "json"])] + scrub_logs: bool, + /// With `--scrub-logs`: report what would change without rewriting. + #[arg(long, requires = "scrub_logs")] + dry_run: bool, }, /// Ruff linter/formatter with compact output @@ -4263,7 +4271,13 @@ fn run_cli() -> Result { 0 } - Commands::Security { all, json } => hooks::tirith_gate::run_security_dashboard(all, json)?, + Commands::Security { all, json, scrub_logs, dry_run } => { + if scrub_logs { + hooks::tirith_gate::run_scrub_logs(dry_run)? + } else { + hooks::tirith_gate::run_security_dashboard(all, json)? + } + } }; Ok(code)