diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 22529caa..f0817478 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,19 +7,16 @@ updates: day: monday time: "04:00" timezone: Asia/Shanghai - open-pull-requests-limit: 10 + open-pull-requests-limit: 3 labels: - dependencies - go commit-message: prefix: deps groups: - golang-x: + all-go-dependencies: patterns: - - "golang.org/x/*" - chainreactors: - patterns: - - "github.com/chainreactors/*" + - "*" - package-ecosystem: github-actions directory: "/" @@ -28,8 +25,13 @@ updates: day: monday time: "04:30" timezone: Asia/Shanghai + open-pull-requests-limit: 1 labels: - dependencies - github-actions commit-message: prefix: deps + groups: + all-github-actions: + patterns: + - "*" diff --git a/.github/native/README.md b/.github/native/README.md index ef02592b..325bfcbc 100644 --- a/.github/native/README.md +++ b/.github/native/README.md @@ -1,16 +1,16 @@ # Recorder native SDK -AIScan uses a two-stage build so ordinary full builds do not compile FFmpeg and x264. +AIScan keeps the native recorder SDK separate from normal product builds. 1. Maintainers run the `recorder-native-sdk` workflow after changing `versions.env` or the native build configuration. It builds the pinned sources, creates relocatable static SDK archives, writes SHA-256 sidecars, and publishes the assets to the versioned GitHub release. -2. Users and product release jobs run `make full`. Its `record-native` prerequisite downloads the matching platform archive once, verifies it, and installs it below `.cache/record-native` before the Go/CGO link step. Pull-request CI falls back to the same pinned source builder when a new versioned SDK release has not been published yet. +2. SDK and record-tool developers run `make record` when they need the optional backend. It downloads the matching platform archive once, verifies it, installs it below `.cache/record-native`, and builds `aiscan-record`. Default `make full` and product release jobs do not fetch or link this SDK. Supported bundles are `linux-amd64`, `linux-arm64`, and `windows-amd64`. FFmpeg and x264 are static, so the distributed executable does not require separate FFmpeg/x264 installation. Operating-system libraries remain external dependencies: Linux uses glibc and X11/XCB; Windows uses system DLLs. The source builder uses an explicit component allowlist (capture input, H.264 encoder, MP4 muxer, and file output only), and packaging rejects static-library sets larger than 16 MiB by default. The Makefile is the public build interface: ```bash -make full # fetch SDK and build aiscan-full +make record # fetch SDK and build aiscan-record make record-native # fetch and verify SDK only make record-native-source # build SDK from pinned sources make record-native-source record-native-package @@ -31,8 +31,16 @@ Environment overrides: - `AISCAN_RECORD_PREFIX`: SDK install/cache directory. - `AISCAN_RECORD_NATIVE_URL`: release or mirror base URL containing the archive and `.sha256` sidecar. - `AISCAN_RECORD_OFFLINE=1`: forbid downloads and require an already cached matching SDK. -- `AISCAN_RECORD_BUILD_FROM_SOURCE=1`: make `make full` or `build.sh -p full` use the pinned source builders instead of downloading an SDK. +- `AISCAN_RECORD_BUILD_FROM_SOURCE=1`: make `make record` or `make record-native` use the pinned source builder instead of downloading an SDK. - `RECORD_ARCH`: target architecture for Makefile SDK targets (defaults to `go env GOARCH`). - `RECORD_NATIVE_OUTPUT`: package output directory (defaults to `dist/native`). When native inputs or flags change, increment `RECORD_NATIVE_VERSION` and `RECORD_NATIVE_RELEASE` together before publishing. Do not replace an existing SDK version with incompatible contents. + +## macOS CGO cross-build + +The release workflow builds both standard and full macOS binaries on an Ubuntu runner. Standard uses the normal pure-Go `CGO_ENABLED=0` cross-build. Full uses the Zig C/C++ driver with a pinned macOS SDK, `CGO_ENABLED=1`, and external Go linking so the bundled Darwin `libcstx` and RE2 archives can link against `Security`, `CoreFoundation`, `libresolv`, and libc++. + +The SDK version, checksum, Zig version, and minimum deployment target are pinned in `versions.env`. The SDK archive is downloaded from the versioned `joseluisq/macosx-sdks` release and verified before extraction. Native recording remains supported only by the Linux and Windows recorder SDK bundles above. + +No macOS GitHub Actions runner is used. Linux can validate the generated Mach-O format and architecture, but it cannot execute the release binary; runtime smoke coverage remains the responsibility of downstream macOS users or a separately authorized external test environment. diff --git a/.github/native/versions.env b/.github/native/versions.env index 6e89c9ab..a76a5392 100644 --- a/.github/native/versions.env +++ b/.github/native/versions.env @@ -6,3 +6,7 @@ FFMPEG_COMMIT=8ae0b34901ba60a802f183ee75a250a9fc3e09a5 FFMPEG_REPOSITORY=https://github.com/FFmpeg/FFmpeg.git X264_COMMIT=0480cb05fa188d37ae87e8f4fd8f1aea3711f7ee X264_REPOSITORY=https://github.com/mirror/x264.git +MACOS_CROSS_ZIG_VERSION=0.14.1 +MACOS_CROSS_SDK_VERSION=14.5 +MACOS_CROSS_SDK_SHA256=6e146275d19f027faa2e8354da5e0267513abf013b8f16ad65a231653a2b1c5d +MACOS_CROSS_DEPLOYMENT_TARGET=11.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5653a77..47f3b9c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,78 +15,67 @@ concurrency: cancel-in-progress: true jobs: - # ── Fast gates (independent, no deps) ────────────────────────── - - lint: + checks: runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v6 with: - fetch-depth: 0 submodules: recursive - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - cache: true - - - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v9.2.1 - with: - version: v2.12.2 - args: --timeout=5m --build-tags "re2_cgo re2_static" - - tidy: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true + cache: ${{ env.ACT != 'true' }} - name: Check go mod tidy run: | - cp go.mod go.mod.orig - cp go.sum go.sum.orig - go mod tidy - if ! diff -q go.mod go.mod.orig >/dev/null 2>&1; then - echo "::error::go.mod is not tidy. Run 'go mod tidy' and commit the result." - diff go.mod.orig go.mod || true + cp go.mod "$RUNNER_TEMP/go.mod.before" + cp go.sum "$RUNNER_TEMP/go.sum.before" + for attempt in 1 2 3; do + if go mod tidy; then + break + fi + if [[ "$attempt" == "3" ]]; then + exit 1 + fi + echo "go mod tidy failed, retrying ($attempt/3)..." + sleep $((attempt * 5)) + done + if ! cmp -s go.mod "$RUNNER_TEMP/go.mod.before" || \ + ! cmp -s go.sum "$RUNNER_TEMP/go.sum.before"; then + echo "::error::go.mod or go.sum is not tidy. Run 'go mod tidy' and commit the result." + diff -u "$RUNNER_TEMP/go.mod.before" go.mod || true + diff -u "$RUNNER_TEMP/go.sum.before" go.sum | head -30 || true exit 1 fi - if ! diff -q go.sum go.sum.orig >/dev/null 2>&1; then - echo "::error::go.sum is not tidy. Run 'go mod tidy' and commit the result." - diff go.sum.orig go.sum | head -30 || true + + - name: Check AOP module tidy + working-directory: aop + run: | + cp go.mod "$RUNNER_TEMP/aop.go.mod.before" + cp go.sum "$RUNNER_TEMP/aop.go.sum.before" + for attempt in 1 2 3; do + if go mod tidy; then + break + fi + if [[ "$attempt" == "3" ]]; then + exit 1 + fi + echo "aop go mod tidy failed, retrying ($attempt/3)..." + sleep $((attempt * 5)) + done + if ! cmp -s go.mod "$RUNNER_TEMP/aop.go.mod.before" || \ + ! cmp -s go.sum "$RUNNER_TEMP/aop.go.sum.before"; then + echo "::error::aop/go.mod or aop/go.sum is not tidy. Run 'cd aop && go mod tidy' and commit the result." + diff -u "$RUNNER_TEMP/aop.go.mod.before" go.mod || true + diff -u "$RUNNER_TEMP/aop.go.sum.before" go.sum | head -30 || true exit 1 fi - quality: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - name: Check dependency layers, registered skips, and repository debt - run: go test -count=1 ./core/deps + run: go test -count=1 ./internal/repositorytest ./core/deps - name: Ensure the standard CLI does not depend on libcstx run: | @@ -99,117 +88,116 @@ jobs: - name: Run go vet run: go vet ./... + - name: Compile public scanner regressions + run: | + go test -run '^$' -tags "full integration re2_cgo re2_static" ./tools + - name: Check whitespace and submodule pins run: | git diff --check HEAD - git diff --exit-code --submodule=diff - git submodule foreach --recursive 'test -z "$(git status --porcelain)"' + git diff --ignore-submodules=dirty --exit-code -- \ + .gitmodules templates web/frontend/cyber-ui - # ── Unit tests (depends on tidy) ────────────────────────────── + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v9.2.1 + with: + version: v2.12.2 + args: --timeout=8m --build-tags "re2_cgo re2_static" + skip-cache: ${{ env.ACT == 'true' }} test: runs-on: ubuntu-22.04 - needs: [tidy, quality] + needs: checks steps: - name: Checkout uses: actions/checkout@v6 with: - fetch-depth: 0 submodules: recursive - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - cache: true - + cache: ${{ env.ACT != 'true' }} - name: Generate embedded resources run: go generate ./core/resources/... + - name: Run AOP module tests + working-directory: aop + run: | + go test -race -count=1 \ + -coverprofile=coverage.out \ + -covermode=atomic \ + ./... + - name: Run unit tests with coverage run: | - go test -tags "re2_cgo re2_static" -race -count=1 -timeout 5m \ + test_args=(-timeout 5m) + if [[ "${ACT:-}" == "true" ]]; then + test_args=(-timeout 10m -p 4) + fi + go test -tags "re2_cgo re2_static" -race -count=1 "${test_args[@]}" \ -coverprofile=coverage.out \ -covermode=atomic \ ./... + - name: Enforce coverage floor + run: | + coverage="$(go tool cover -func=coverage.out | awk '/^total:/ {gsub("%", "", $3); print $3}')" + if [[ -z "$coverage" ]]; then + echo "::error::unable to read total coverage" + exit 1 + fi + if ! awk -v coverage="$coverage" 'BEGIN { exit !(coverage + 0 >= 50.0) }'; then + echo "::error::total coverage ${coverage}% is below the 50.0% floor" + exit 1 + fi + echo "Total coverage ${coverage}% meets the 50.0% floor" + + - name: Check generated resources are committed + run: git diff --exit-code -- core/resources/template.go + - name: Display coverage summary if: always() run: | if [ -f coverage.out ]; then - echo "### Total coverage" + echo "### Root module coverage" go tool cover -func=coverage.out | tail -1 echo "" echo "### Per-package coverage (top 20)" go tool cover -func=coverage.out | grep -E '^[a-z]' | sort -t$'\t' -k3 -rn | head -20 fi + if [ -f aop/coverage.out ]; then + echo "" + echo "### AOP module coverage" + go tool cover -func=aop/coverage.out | tail -1 + fi - name: Upload coverage artifact - if: always() + if: ${{ always() && env.ACT != 'true' }} uses: actions/upload-artifact@v7 with: name: coverage-report - path: coverage.out + path: | + coverage.out + aop/coverage.out retention-days: 14 - # ── Proxy & TMux tool tests (depends on tidy) ───────────────── - - tool-tests: - runs-on: ubuntu-22.04 - needs: tidy - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - - - name: Run proxy tool tests - run: | - go test -tags "re2_cgo re2_static" -race -count=1 -timeout 5m -v \ - ./tools/proxy/ - - - name: Run tmux command tests - run: | - go test -tags "re2_cgo re2_static" -race -count=1 -timeout 5m -v \ - -run 'Tmux|BashProxy' \ - ./pkg/commands/ - - - name: Run PTY interactive session tests - run: | - go test -tags "re2_cgo re2_static" -race -count=1 -timeout 5m -v \ - -run 'MultiRound|SendCtrlC' \ - ./agent/tmux/ - - - name: Run agent tmux integration tests - run: | - go test -tags "re2_cgo re2_static" -race -count=1 -timeout 5m -v \ - -run 'AgentTmux' \ - ./agent/ - windows-test: runs-on: windows-2022 - needs: [tidy, quality] + needs: test steps: - name: Checkout uses: actions/checkout@v6 with: - fetch-depth: 0 submodules: recursive - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - cache: true + cache: ${{ env.ACT != 'true' }} - name: Set up mingw for libcstx shell: bash @@ -225,52 +213,20 @@ jobs: env: CGO_ENABLED: "1" - race-stress: - runs-on: ubuntu-22.04 - needs: [tidy, quality] - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - - name: Repeat agent and runner concurrency tests - run: | - go test -race -count=20 -timeout 15m \ - -run 'Test(ConcurrentEmitWhileRegistering|SetProviderRaceWithRun|ResetDoesNotAllowConcurrentPrompt|StreamingProviderEmitsMessageUpdates)$' \ - ./agent/... - go test -race -count=20 -timeout 15m \ - -run 'Test(StdioSameSessionFIFOOrder|StdioSessionsRunConcurrently|StdioDrainWaitsForInFlightAndQueued|RuntimeSessionDirectLoopUsesSessionScheduler|RuntimeSessionRejectsRequestsPastPendingLimit|SessionContextCancellationStopsActiveRun|ActiveRunSteersAsyncInputWithoutSecondLifecycle)$' \ - ./pkg/runner/... - - - name: Repeat web SSE, cancellation, and reload concurrency tests - run: | - go test -race -count=20 -timeout 20m \ - -run 'Test(BroadcastAOPEventPersistsRawEnvelope|ServeSSEWithSnapshotSubscribesBeforeReadingSnapshot|ServeSSEWithSnapshotDropsQueuedSnapshotDuplicates|SessionEventsReplayHasNoSideEffects|SessionEventsResumesAfterLastEventID|CancelRemoteScanStopsAgentAndPreservesCanceledStatus|CancelQueuedScanDoesNotWaitForConcurrencySlot|CancelTaskUsesControlChannelWhenTaskQueueIsFull|CancelTaskWaitsForSaturatedControlChannel|CompleteJobCannotOverwriteCanceledScan|BroadcastConfigReload|BroadcastConfigReloadWaitsBehindCancellationFrames|HandleConfigReloadResultUpdatesAgentStatus|SaveConfigBuildFailureKeepsCommittedConfigAndCurrentApp|SaveConfigCommitFailureClosesCandidateAndKeepsCurrentApp|SaveConfigSerializesConcurrentCandidates)$' \ - ./pkg/web - scanner-functional: runs-on: ubuntu-22.04 - needs: tidy + needs: test steps: - name: Checkout uses: actions/checkout@v6 with: - fetch-depth: 0 submodules: recursive - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - cache: true + cache: ${{ env.ACT != 'true' }} - name: Run scanner functional regressions run: | @@ -303,20 +259,19 @@ jobs: headless-record-replay-e2e: runs-on: ubuntu-22.04 - needs: tidy + needs: test timeout-minutes: 10 steps: - name: Checkout uses: actions/checkout@v6 with: - fetch-depth: 0 submodules: recursive - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - cache: true + cache: ${{ env.ACT != 'true' }} - name: Set up Chrome uses: browser-actions/setup-chrome@v2 @@ -347,57 +302,26 @@ jobs: -run '^TestE2EKatanaDeepRendersAuthenticatedSPA$' \ ./tools/scan - # ── Generated templates tests (depends on tidy) ─────────────── - - generated-test: - runs-on: ubuntu-22.04 - needs: tidy - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - - - name: Run go generate for templates - run: go generate ./core/resources/... - - - name: Run resources tests - run: | - go test -tags "re2_cgo re2_static" -race -count=1 -timeout 5m \ - ./core/resources/... - - - name: Check generated resources are committed - run: git diff --exit-code - - protobuf-generated: + e2e: runs-on: ubuntu-22.04 - needs: tidy + needs: test steps: - name: Checkout uses: actions/checkout@v6 with: - fetch-depth: 0 submodules: recursive - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - cache: true + cache: ${{ env.ACT != 'true' }} - name: Set up Node.js uses: actions/setup-node@v6 with: node-version: 22 - cache: npm + cache: ${{ env.ACT != 'true' && 'npm' || '' }} cache-dependency-path: web/frontend/package-lock.json - name: Set up protoc 35.1 @@ -413,46 +337,18 @@ jobs: run: go run ./cmd/gen - name: Check generated protobuf bindings are committed - run: git diff --exit-code - - # ── E2E tests (depends on test) ─────────────────────────────── - - e2e: - runs-on: ubuntu-22.04 - needs: test - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: 22 - cache: npm - cache-dependency-path: web/frontend/package-lock.json + run: | + git diff --exit-code -- \ + pkg/rpc \ + pkg/types \ + web/frontend/src/gen \ + web/frontend/cyber-ui/packages/aop/src/gen/aop - name: Build embedded frontend run: | - npm --prefix web/frontend ci npm --prefix web/frontend run build test -s web/static/index.html - - name: Upload embedded frontend - uses: actions/upload-artifact@v7 - with: - name: embedded-frontend - path: web/static - retention-days: 1 - - name: Install Playwright Chromium working-directory: web/frontend run: npx playwright install --with-deps chromium @@ -467,177 +363,16 @@ jobs: corepack pnpm install --frozen-lockfile corepack pnpm --filter @cyber/viewer test - - name: Run e2e tests + - name: Run backend E2E tests run: | go test -race -count=1 -timeout 10m \ -tags "e2e re2_cgo re2_static" \ -v \ ./pkg/web - # ── Standard build: pure Go, no libcstx/CGO ────────────────── - - build-standard: - runs-on: ubuntu-22.04 - needs: test - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - - name: Generate embedded resources - run: go generate ./core/resources - - - name: Build all standard platforms - run: | - for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do - IFS='/' read -r goos goarch <<< "$target" - echo " compile ${goos}/${goarch}" - suffix="" - [[ "$goos" == "windows" ]] && suffix=".exe" - CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \ - go build -trimpath -tags "forceposix emptytemplates noembed osusergo netgo" \ - -ldflags "-s -w" -buildvcs=false \ - -o "dist/standard_${goos}_${goarch}${suffix}" ./cmd/aiscan - done - test -f dist/standard_windows_amd64.exe - test -f dist/standard_windows_arm64.exe - ls -lh dist/ - - - name: Upload standard binaries - uses: actions/upload-artifact@v7 - with: - name: aiscan-standard - path: dist/standard_* - if-no-files-found: error - retention-days: 7 - - # ── Full build: native libcstx/CGO on supported platforms ──── - - build-full: - needs: [test, e2e] - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: bash - strategy: - fail-fast: false - matrix: - include: - - id: linux-amd64 - runner: ubuntu-22.04 - goos: linux - goarch: amd64 - - id: linux-arm64 - runner: ubuntu-24.04-arm - goos: linux - goarch: arm64 - - id: darwin-amd64 - runner: macos-15-intel - goos: darwin - goarch: amd64 - - id: darwin-arm64 - runner: macos-15 - goos: darwin - goarch: arm64 - - id: windows-amd64 - runner: windows-2022 - goos: windows - goarch: amd64 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - - name: Set up mingw for libcstx - if: runner.os == 'Windows' - run: echo "C:/msys64/mingw64/bin" >> "$GITHUB_PATH" - - - name: Install recorder SDK link dependencies on Linux - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y build-essential nasm yasm pkg-config \ - libxcb1-dev libxcb-shm0-dev libxcb-shape0-dev libxcb-xfixes0-dev - - - name: Install recorder SDK link dependencies on Windows - if: runner.os == 'Windows' - run: | - C:/msys64/usr/bin/bash.exe -lc \ - "pacman -S --noconfirm --needed git diffutils make nasm yasm pkgconf mingw-w64-x86_64-toolchain" - - - name: Prepare static FFmpeg and x264 recorder SDK - if: runner.os != 'macOS' - run: | - if [[ "${RUNNER_OS}" == "Windows" ]]; then - platform=windows - C:/msys64/usr/bin/bash.exe -lc "cd '${GITHUB_WORKSPACE}'; make record-native RECORD_ARCH='${{ matrix.goarch }}' || make record-native-source RECORD_ARCH='${{ matrix.goarch }}'" - else - platform=linux - make record-native RECORD_ARCH='${{ matrix.goarch }}' || make record-native-source RECORD_ARCH='${{ matrix.goarch }}' - fi - bash .github/native/sdk.sh env "${platform}" '${{ matrix.goarch }}' >> "${GITHUB_ENV}" - - - name: Verify recorder SDK link environment - if: runner.os != 'macOS' - run: pkg-config --modversion libavcodec - - - name: Download embedded frontend - uses: actions/download-artifact@v7 - with: - name: embedded-frontend - path: web/static - - - name: Generate embedded resources - run: go generate ./core/resources - - - name: Build full ${{ matrix.id }} - run: | - suffix="" - [[ "${{ matrix.goos }}" == "windows" ]] && suffix=".exe" - CGO_ENABLED=1 GOOS="${{ matrix.goos }}" GOARCH="${{ matrix.goarch }}" \ - go build -trimpath \ - -tags "forceposix emptytemplates noembed osusergo netgo full sqlite record_ffmpeg re2_cgo re2_static" \ - -ldflags "-s -w" -buildvcs=false \ - -o "dist/full_${{ matrix.goos }}_${{ matrix.goarch }}${suffix}" ./cmd/aiscan - - - name: Verify recorder libraries are statically linked - if: runner.os != 'macOS' - run: | - set -euo pipefail - binary="$(find dist -maxdepth 1 -type f -name 'full_*' -print -quit)" - test -n "${binary}" - if [[ "${RUNNER_OS}" == "Windows" ]]; then - if objdump -p "${binary}" | grep -Eiq 'DLL Name:.*(libav|x264|libwinpthread)'; then - echo "recorder library remained dynamically linked" >&2 - exit 1 - fi - else - if ldd "${binary}" | grep -Eiq '(libav|libx264)'; then - echo "recorder library remained dynamically linked" >&2 - exit 1 - fi - fi - - - name: Upload full ${{ matrix.id }} binary - uses: actions/upload-artifact@v7 - with: - name: aiscan-full-${{ matrix.id }} - path: dist/full_* - if-no-files-found: error - retention-days: 7 + release-verify: + needs: [windows-test, scanner-functional, headless-record-replay-e2e, e2e] + uses: ./.github/workflows/release-build.yml + with: + tag: v0.0.0-ci.${{ github.run_id }} + ref: ${{ github.sha }} diff --git a/.github/workflows/go-release.yml b/.github/workflows/go-release.yml index 893719a5..6e3cc92e 100644 --- a/.github/workflows/go-release.yml +++ b/.github/workflows/go-release.yml @@ -20,22 +20,6 @@ on: required: false default: false type: boolean - workflow_call: - inputs: - tag: - description: 'Release tag' - required: true - type: string - target: - description: 'Branch or commit to tag if the tag does not exist' - required: false - default: master - type: string - prerelease: - description: 'Publish immediately as a prerelease instead of creating a draft' - required: false - default: false - type: boolean permissions: contents: write @@ -44,20 +28,12 @@ concurrency: group: release-${{ inputs.tag != '' && inputs.tag || github.ref_name }} cancel-in-progress: false -# --------------------------------------------------------------------------- -# One frontend bundle + standard/full native build matrix → draft release -# --------------------------------------------------------------------------- - jobs: - - # ── Resolve the tag once, share with all jobs ─────────────────── prepare: - # Nightly tag pushes also match v*.*.*. The nightly workflow invokes this - # workflow explicitly with prerelease=true, so ignore the duplicate push. - if: github.event_name != 'push' || !contains(github.ref_name, '-nightly.') runs-on: ubuntu-22.04 outputs: tag: ${{ steps.tag.outputs.tag }} + ref: ${{ steps.tag.outputs.ref }} steps: - name: Checkout uses: actions/checkout@v6 @@ -82,15 +58,17 @@ jobs: echo "Invalid release tag: ${TAG}" >&2 exit 1 fi - if [[ "${TAG}" == *nightly* && "${{ inputs.prerelease }}" != "true" ]]; then - echo "Nightly tags require prerelease=true: ${TAG}" >&2 - exit 1 - fi - git fetch --force --tags origin if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then - echo "Tag ${TAG} exists" + if [[ -n "${{ inputs.tag }}" ]]; then + tag_commit="$(git rev-parse "refs/tags/${TAG}^{commit}")" + target_commit="$(git rev-parse "${TARGET}^{commit}")" + if [[ "${tag_commit}" != "${target_commit}" ]]; then + echo "Tag ${TAG} points to ${tag_commit}, expected ${target_commit}" >&2 + exit 1 + fi + fi else if [[ -z "${{ inputs.tag }}" ]]; then echo "Tag ${TAG} was expected to exist for a tag push event" >&2 @@ -98,228 +76,21 @@ jobs: fi git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout --force "${TARGET}" - git tag -a "${TAG}" -m "Release ${TAG}" + git tag -a "${TAG}" "${TARGET}" -m "Release ${TAG}" git push origin "refs/tags/${TAG}" fi echo "tag=${TAG}" >> "${GITHUB_OUTPUT}" + echo "ref=refs/tags/${TAG}" >> "${GITHUB_OUTPUT}" - # ── Build the embedded frontend once for every full target ───── - frontend: - needs: prepare - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: refs/tags/${{ needs.prepare.outputs.tag }} - fetch-depth: 0 - submodules: recursive - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: 22 - cache: npm - cache-dependency-path: web/frontend/package-lock.json - - - name: Build embedded frontend - run: | - npm --prefix web/frontend ci - npm --prefix web/frontend run build - test -s web/static/index.html - test -n "$(find web/static/assets -type f -size +0c -print -quit)" - - - name: Upload embedded frontend - uses: actions/upload-artifact@v7 - with: - name: embedded-frontend - path: web/static - retention-days: 1 - - # ── Parallel build matrix ─────────────────────────────────────── build: - needs: [prepare, frontend] - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: bash - strategy: - fail-fast: false - matrix: - include: - - id: aiscan - profile: standard - runner: ubuntu-22.04 - main: ./cmd/aiscan - binary: aiscan - tags: "forceposix emptytemplates noembed osusergo netgo" - targets: "linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64" - cgo: "0" - - id: aiscan-full-linux-amd64 - profile: full - runner: ubuntu-22.04 - main: ./cmd/aiscan - binary: aiscan-full - tags: "forceposix emptytemplates noembed osusergo netgo full sqlite record_ffmpeg re2_cgo re2_static" - targets: "linux/amd64" - cgo: "1" - - id: aiscan-full-linux-arm64 - profile: full - runner: ubuntu-24.04-arm - main: ./cmd/aiscan - binary: aiscan-full - tags: "forceposix emptytemplates noembed osusergo netgo full sqlite record_ffmpeg re2_cgo re2_static" - targets: "linux/arm64" - cgo: "1" - - id: aiscan-full-darwin-amd64 - profile: full - runner: macos-15-intel - main: ./cmd/aiscan - binary: aiscan-full - tags: "forceposix emptytemplates noembed osusergo netgo full sqlite record_ffmpeg re2_cgo re2_static" - targets: "darwin/amd64" - cgo: "1" - - id: aiscan-full-darwin-arm64 - profile: full - runner: macos-15 - main: ./cmd/aiscan - binary: aiscan-full - tags: "forceposix emptytemplates noembed osusergo netgo full sqlite re2_cgo re2_static" - targets: "darwin/arm64" - cgo: "1" - - id: aiscan-full-windows-amd64 - profile: full - runner: windows-2022 - main: ./cmd/aiscan - binary: aiscan-full - tags: "forceposix emptytemplates noembed osusergo netgo full sqlite record_ffmpeg re2_cgo re2_static" - targets: "windows/amd64" - cgo: "1" - - env: - GORELEASER_CURRENT_TAG: ${{ needs.prepare.outputs.tag }} - - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: refs/tags/${{ needs.prepare.outputs.tag }} - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: true - - - name: Set up mingw for libcstx - if: runner.os == 'Windows' - run: echo "C:/msys64/mingw64/bin" >> "$GITHUB_PATH" - - - name: Install recorder SDK link dependencies on Linux - if: matrix.profile == 'full' && runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y build-essential nasm yasm pkg-config \ - libxcb1-dev libxcb-shm0-dev libxcb-shape0-dev libxcb-xfixes0-dev - - - name: Install recorder SDK link dependencies on Windows - if: matrix.profile == 'full' && runner.os == 'Windows' - run: | - C:/msys64/usr/bin/bash.exe -lc \ - "pacman -S --noconfirm --needed git diffutils make nasm yasm pkgconf mingw-w64-x86_64-toolchain" - - - name: Prepare static FFmpeg and x264 recorder SDK - if: matrix.profile == 'full' && runner.os != 'macOS' - run: | - if [[ "${RUNNER_OS}" == "Windows" ]]; then - platform=windows - C:/msys64/usr/bin/bash.exe -lc \ - "cd '${GITHUB_WORKSPACE}'; make record-native RECORD_ARCH=amd64 || make record-native-source RECORD_ARCH=amd64" - else - platform=linux - make record-native RECORD_ARCH="$(go env GOARCH)" || \ - make record-native-source RECORD_ARCH="$(go env GOARCH)" - fi - bash .github/native/sdk.sh env "${platform}" "$(go env GOARCH)" >> "${GITHUB_ENV}" - - - name: Verify recorder SDK link environment - if: matrix.profile == 'full' && runner.os != 'macOS' - run: pkg-config --modversion libavcodec - - - name: Download embedded frontend - if: matrix.profile == 'full' - uses: actions/download-artifact@v7 - with: - name: embedded-frontend - path: web/static - - - name: Generate embedded resources - run: go generate ./core/resources - - - name: Build binaries - shell: bash - run: | - set -euo pipefail - TARGETS="${{ matrix.targets }}" - TAGS="${{ matrix.tags }}" - BINARY="${{ matrix.binary }}" - MAIN="${{ matrix.main }}" - VERSION="${GORELEASER_CURRENT_TAG#v}" - OUTDIR="dist/build" - mkdir -p "${OUTDIR}" - - for target in $TARGETS; do - IFS='/' read -r goos goarch <<< "$target" - suffix=""; [[ "$goos" == "windows" ]] && suffix=".exe" - out="${OUTDIR}/${BINARY}_${goos}_${goarch}${suffix}" - echo " compiling ${goos}/${goarch} → ${out}" - CGO_ENABLED="${{ matrix.cgo }}" GOOS="$goos" GOARCH="$goarch" \ - go build -trimpath -tags "$TAGS" \ - -ldflags "-s -w -X github.com/chainreactors/aiscan/core/config.Version=${VERSION}" \ - -buildvcs=false \ - -o "$out" "$MAIN" - done - - if [[ -x "${OUTDIR}/${BINARY}_linux_amd64" ]]; then - version_output="$("${OUTDIR}/${BINARY}_linux_amd64" --version)" - test "$version_output" = "aiscan v${VERSION}" - fi - - echo "=== Binaries ===" - ls -lh "${OUTDIR}/" - - - name: Verify recorder libraries are statically linked - if: matrix.profile == 'full' && runner.os != 'macOS' - run: | - set -euo pipefail - binary="$(find dist/build -maxdepth 1 -type f -name 'aiscan-full_*' -print -quit)" - test -n "${binary}" - if [[ "${RUNNER_OS}" == "Windows" ]]; then - if objdump -p "${binary}" | grep -Eiq 'DLL Name:.*(libav|x264|libwinpthread)'; then - echo "recorder library remained dynamically linked" >&2 - exit 1 - fi - else - if ldd "${binary}" | grep -Eiq '(libav|libx264)'; then - echo "recorder library remained dynamically linked" >&2 - exit 1 - fi - fi - - - name: Upload artifacts - uses: actions/upload-artifact@v7 - with: - name: release-${{ matrix.id }} - path: dist/build/* - retention-days: 1 + needs: prepare + uses: ./.github/workflows/release-build.yml + with: + tag: ${{ needs.prepare.outputs.tag }} + ref: ${{ needs.prepare.outputs.ref }} - # ── Package release ───────────────────────────────────────────── - package: + release: needs: [prepare, build] runs-on: ubuntu-22.04 env: @@ -328,145 +99,40 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: - ref: refs/tags/${{ needs.prepare.outputs.tag }} + ref: ${{ needs.prepare.outputs.ref }} fetch-depth: 0 - - name: Download all artifacts - uses: actions/download-artifact@v7 - with: - pattern: release-* - path: dist/build - merge-multiple: true - - - name: Install pinned UPX - env: - UPX_VERSION: 5.2.0 - UPX_SHA256: 3db5d3294707439db97866feab8d75d800f028f48481a40547411824da4288a1 - run: | - set -euo pipefail - archive="${RUNNER_TEMP}/upx-${UPX_VERSION}-amd64_linux.tar.xz" - curl -fsSL \ - "https://github.com/upx/upx/releases/download/v${UPX_VERSION}/upx-${UPX_VERSION}-amd64_linux.tar.xz" \ - -o "${archive}" - echo "${UPX_SHA256} ${archive}" | sha256sum -c - - tar -xJf "${archive}" -C "${RUNNER_TEMP}" - echo "UPX_BIN=${RUNNER_TEMP}/upx-${UPX_VERSION}-amd64_linux/upx" >> "${GITHUB_ENV}" - - - name: Compress Windows amd64 binaries with UPX - run: | - set -euo pipefail - "${UPX_BIN}" --version - for f in dist/build/*_windows_amd64.exe; do - [ -f "$f" ] || continue - "${UPX_BIN}" "$f" - "${UPX_BIN}" -t "$f" - done - - - name: Package archives - run: | - set -euo pipefail - mkdir -p dist/release - for f in dist/build/*; do - [ -f "$f" ] || continue - base=$(basename "$f") - archive_name="${base%.exe}" - binary="${base%%_*}" - inner_name="$binary" - [[ "$base" == *.exe ]] && inner_name="${binary}.exe" - - tmpdir=$(mktemp -d) - cp "$f" "${tmpdir}/${inner_name}" - cp README.md "${tmpdir}/" 2>/dev/null || true - cp -r docs "${tmpdir}/" 2>/dev/null || true - (cd "$tmpdir" && zip -r - .) > "${GITHUB_WORKSPACE}/dist/release/${archive_name}.zip" - rm -rf "$tmpdir" - done - ls -lh dist/release - - - name: Generate checksums - run: | - cd dist/release - sha256sum *.zip > aiscan_checksums.txt - cat aiscan_checksums.txt - - - name: Upload release bundle - uses: actions/upload-artifact@v7 - with: - name: release-bundle - path: dist/release/* - retention-days: 1 - - # UPX can report a structurally valid PE that still crashes at process - # startup. Run the exact packaged Windows binaries before publishing them. - verify-windows-release: - needs: [prepare, package] - runs-on: windows-2022 - steps: - name: Download release bundle uses: actions/download-artifact@v7 with: name: release-bundle path: dist/release - - name: Smoke test packaged Windows binaries - shell: pwsh - env: - TAG: ${{ needs.prepare.outputs.tag }} + - name: Generate changelog + id: changelog + shell: bash run: | - $ErrorActionPreference = 'Stop' - $expectedVersion = "aiscan $env:TAG" - $cases = @( - @{ Archive = 'aiscan_windows_amd64.zip'; Binary = 'aiscan.exe'; Full = $false }, - @{ Archive = 'aiscan-full_windows_amd64.zip'; Binary = 'aiscan-full.exe'; Full = $true } - ) - - foreach ($case in $cases) { - $destination = Join-Path $env:RUNNER_TEMP ([IO.Path]::GetFileNameWithoutExtension($case.Archive)) - Expand-Archive -LiteralPath (Join-Path 'dist/release' $case.Archive) -DestinationPath $destination - $binary = Join-Path $destination $case.Binary - $version = (& $binary --version | Out-String).Trim() - if ($LASTEXITCODE -ne 0) { - throw "$($case.Binary) --version exited with $LASTEXITCODE" - } - if ($version -ne $expectedVersion) { - throw "$($case.Binary) reported '$version', expected '$expectedVersion'" - } - if ($case.Full) { - & $binary web --help | Out-Null - if ($LASTEXITCODE -ne 0) { - throw "$($case.Binary) web --help exited with $LASTEXITCODE" + awk -v heading="## ${TAG}" ' + /^## / { + if (found) exit + if ($0 == heading || index($0, heading " ") == 1) { + found = 1 + next } } - } - - # ── Publish release ───────────────────────────────────────────── - release: - needs: [prepare, package, verify-windows-release] - runs-on: ubuntu-22.04 - env: - TAG: ${{ needs.prepare.outputs.tag }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: refs/tags/${{ needs.prepare.outputs.tag }} - fetch-depth: 0 + found { print } + ' docs/changelog.md > /tmp/changelog.md - - name: Download release bundle - uses: actions/download-artifact@v7 - with: - name: release-bundle - path: dist/release - - - name: Generate changelog - id: changelog - run: | - prev_tag=$(git tag --sort=-v:refname \ - | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + prev_tag=$(git tag --merged "${TAG}^" --sort=-version:refname \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z][0-9A-Za-z.-]*)?$' \ + | grep -Ev 'nightly|^v0\.0\.0-ci\.' \ | grep -Fxv "${TAG}" \ | head -1 || true) - if [ -n "$prev_tag" ]; then - git log --pretty=format:"- %s" "${prev_tag}..${TAG}" --no-merges | grep -v "^- ${TAG}$" | grep -v "^- docs" > /tmp/changelog.md || true + if [ -s /tmp/changelog.md ]; then + echo "Using curated notes from docs/changelog.md" + elif [ -n "$prev_tag" ]; then + git log --pretty=format:"- %s" "${prev_tag}..${TAG}" --no-merges \ + | grep -v "^- ${TAG}$" | grep -v "^- docs" > /tmp/changelog.md || true else git log --pretty=format:"- %s" -20 --no-merges > /tmp/changelog.md || true fi @@ -475,8 +141,8 @@ jobs: - name: Create or update release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash run: | - # Delete existing release if any (replace mode) gh release delete "${TAG}" --yes 2>/dev/null || true release_flags=(--draft) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index c10d99e5..00000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: nightly - -on: - schedule: - - cron: '0 16 * * *' # UTC 16:00 = CST 00:00 - workflow_dispatch: - -permissions: - contents: write - -concurrency: - group: nightly - cancel-in-progress: false - -jobs: - prepare: - runs-on: ubuntu-22.04 - outputs: - tag: ${{ steps.nightly.outputs.tag }} - target: ${{ steps.nightly.outputs.target }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - submodules: recursive - - - name: Set nightly tag - id: nightly - run: | - DATE=$(date -u +%Y%m%d) - TAG="v0.0.0-nightly.${DATE}" - echo "TAG=${TAG}" >> "${GITHUB_ENV}" - echo "tag=${TAG}" >> "${GITHUB_OUTPUT}" - echo "target=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" - - - name: Create nightly tag - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git tag -f "$TAG" - git push --force origin "$TAG" - - release: - needs: prepare - uses: ./.github/workflows/go-release.yml - with: - tag: ${{ needs.prepare.outputs.tag }} - target: ${{ needs.prepare.outputs.target }} - prerelease: true - secrets: inherit - - cleanup: - needs: [prepare, release] - runs-on: ubuntu-22.04 - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.prepare.outputs.tag }} - steps: - - name: Delete superseded nightly releases - run: | - gh release list --limit 50 --json tagName \ - | jq -r '.[] | select(.tagName | startswith("v0.0.0-nightly.")) | .tagName' \ - | while read -r tag; do - if [[ "${tag}" == "${TAG}" ]]; then - continue - fi - echo "Deleting release ${tag}" - gh release delete "${tag}" --yes --cleanup-tag || true - done diff --git a/.github/workflows/record-native.yml b/.github/workflows/record-native.yml index 2f6b13e3..f69d4484 100644 --- a/.github/workflows/record-native.yml +++ b/.github/workflows/record-native.yml @@ -99,5 +99,5 @@ jobs: else gh release create "${RECORD_NATIVE_RELEASE}" dist/native/* \ --title "AIScan recorder native SDK ${RECORD_NATIVE_VERSION}" \ - --notes "Prebuilt static FFmpeg ${FFMPEG_TAG} and x264 ${X264_COMMIT} SDKs used by AIScan full builds." + --notes "Prebuilt static FFmpeg ${FFMPEG_TAG} and x264 ${X264_COMMIT} SDKs for optional AIScan record tool builds." fi diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml new file mode 100644 index 00000000..d80c3f44 --- /dev/null +++ b/.github/workflows/release-build.yml @@ -0,0 +1,392 @@ +name: release-build + +on: + workflow_call: + inputs: + tag: + description: 'Version injected into the binaries' + required: true + type: string + ref: + description: 'Commit or tag to build' + required: true + type: string + +permissions: + contents: read + +concurrency: + group: release-build-${{ inputs.ref }} + cancel-in-progress: false + +# --------------------------------------------------------------------------- +# One frontend bundle + standard/full/runner build matrix -> verified bundle +# --------------------------------------------------------------------------- + +jobs: + # ── Build the embedded frontend once for every full target ───── + frontend: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + submodules: recursive + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + cache-dependency-path: web/frontend/package-lock.json + + - name: Build embedded frontend + run: | + npm --prefix web/frontend ci + npm --prefix web/frontend run build + test -s web/static/index.html + test -n "$(find web/static/assets -type f -size +0c -print -quit)" + + - name: Upload embedded frontend + uses: actions/upload-artifact@v7 + with: + name: embedded-frontend + path: web/static + retention-days: 1 + + # ── Parallel build matrix ─────────────────────────────────────── + build: + needs: frontend + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - id: aiscan + profile: standard + runner: ubuntu-22.04 + main: ./cmd/aiscan + binary: aiscan + tags: "forceposix emptytemplates noembed osusergo netgo" + targets: "linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64" + cgo: "0" + - id: runner + profile: runner + runner: ubuntu-22.04 + main: ./cmd/runner + binary: runner + tags: "" + targets: "linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64" + cgo: "0" + - id: aiscan-full-linux-amd64 + profile: full + runner: ubuntu-22.04 + main: ./cmd/aiscan + binary: aiscan-full + tags: "forceposix emptytemplates noembed osusergo netgo full sqlite re2_cgo re2_static" + targets: "linux/amd64" + cgo: "1" + - id: aiscan-full-linux-arm64 + profile: full + runner: ubuntu-24.04-arm + main: ./cmd/aiscan + binary: aiscan-full + tags: "forceposix emptytemplates noembed osusergo netgo full sqlite re2_cgo re2_static" + targets: "linux/arm64" + cgo: "1" + - id: aiscan-full-darwin + profile: full + runner: ubuntu-22.04 + main: ./cmd/aiscan + binary: aiscan-full + tags: "forceposix emptytemplates noembed osusergo netgo full sqlite re2_cgo re2_static" + targets: "darwin/amd64 darwin/arm64" + cgo: "1" + cross: darwin + - id: aiscan-full-windows-amd64 + profile: full + runner: windows-2022 + main: ./cmd/aiscan + binary: aiscan-full + tags: "forceposix emptytemplates noembed osusergo netgo full sqlite re2_cgo re2_static" + targets: "windows/amd64" + cgo: "1" + + env: + GORELEASER_CURRENT_TAG: ${{ inputs.tag }} + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + submodules: recursive + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Read macOS cross-toolchain versions + if: matrix.cross == 'darwin' + id: macos-cross + run: | + source .github/native/versions.env + echo "zig-version=${MACOS_CROSS_ZIG_VERSION}" >> "$GITHUB_OUTPUT" + echo "sdk-version=${MACOS_CROSS_SDK_VERSION}" >> "$GITHUB_OUTPUT" + echo "sdk-sha256=${MACOS_CROSS_SDK_SHA256}" >> "$GITHUB_OUTPUT" + echo "deployment-target=${MACOS_CROSS_DEPLOYMENT_TARGET}" >> "$GITHUB_OUTPUT" + + - name: Set up Zig for macOS CGO cross-compilation + if: matrix.cross == 'darwin' + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 + with: + version: ${{ steps.macos-cross.outputs.zig-version }} + cache-key: macos-cgo-${{ matrix.id }} + + - name: Install pinned macOS SDK + if: matrix.cross == 'darwin' + env: + SDK_VERSION: ${{ steps.macos-cross.outputs.sdk-version }} + SDK_SHA256: ${{ steps.macos-cross.outputs.sdk-sha256 }} + DEPLOYMENT_TARGET: ${{ steps.macos-cross.outputs.deployment-target }} + run: | + set -euo pipefail + sdk_dir="${RUNNER_TEMP}/macos-sdk" + archive="${sdk_dir}/MacOSX${SDK_VERSION}.sdk.tar.xz" + mkdir -p "${sdk_dir}" + for attempt in 1 2 3; do + if curl --retry 3 --retry-all-errors --retry-delay 2 -fsSL \ + "https://github.com/joseluisq/macosx-sdks/releases/download/${SDK_VERSION}/MacOSX${SDK_VERSION}.sdk.tar.xz" \ + -o "${archive}"; then + break + fi + if [[ "${attempt}" == "3" ]]; then + exit 1 + fi + done + echo "${SDK_SHA256} ${archive}" | sha256sum -c - + tar -xJf "${archive}" -C "${sdk_dir}" + sdk_root="${sdk_dir}/MacOSX${SDK_VERSION}.sdk" + test -d "${sdk_root}/System/Library/Frameworks" + test -f "${sdk_root}/usr/lib/libresolv.tbd" + echo "MACOS_SDKROOT=${sdk_root}" >> "${GITHUB_ENV}" + echo "MACOSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET}" >> "${GITHUB_ENV}" + + - name: Set up mingw for libcstx + if: runner.os == 'Windows' + run: echo "C:/msys64/mingw64/bin" >> "$GITHUB_PATH" + + - name: Download embedded frontend + if: matrix.profile == 'full' + uses: actions/download-artifact@v7 + with: + name: embedded-frontend + path: web/static + + - name: Generate embedded resources + run: go generate ./core/resources + + - name: Build binaries + shell: bash + run: | + set -euo pipefail + TARGETS="${{ matrix.targets }}" + TAGS="${{ matrix.tags }}" + BINARY="${{ matrix.binary }}" + MAIN="${{ matrix.main }}" + VERSION="${GORELEASER_CURRENT_TAG#v}" + OUTDIR="dist/build" + mkdir -p "${OUTDIR}" + + for target in $TARGETS; do + IFS='/' read -r goos goarch <<< "$target" + suffix=""; [[ "$goos" == "windows" ]] && suffix=".exe" + out="${OUTDIR}/${BINARY}_${goos}_${goarch}${suffix}" + echo " compiling ${goos}/${goarch} → ${out}" + + link_flags="-s -w -X github.com/chainreactors/aiscan/core/config.Version=${VERSION}" + if [[ "$goos" == "darwin" && "${{ matrix.cgo }}" == "1" ]]; then + test -n "${MACOS_SDKROOT:-}" + case "$goarch" in + amd64) zig_target=x86_64-macos ;; + arm64) zig_target=aarch64-macos ;; + *) echo "unsupported Darwin architecture ${goarch}" >&2; exit 1 ;; + esac + cross_flags="-target ${zig_target} -isysroot ${MACOS_SDKROOT} -F${MACOS_SDKROOT}/System/Library/Frameworks -L${MACOS_SDKROOT}/usr/lib -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" + export CC="zig cc ${cross_flags}" + export CXX="zig c++ ${cross_flags}" + link_flags="-linkmode external ${link_flags}" + else + unset CC CXX + fi + + build_args=(-trimpath) + if [[ -n "$TAGS" ]]; then + build_args+=(-tags "$TAGS") + fi + CGO_ENABLED="${{ matrix.cgo }}" GOOS="$goos" GOARCH="$goarch" \ + go build "${build_args[@]}" \ + -ldflags "${link_flags}" \ + -buildvcs=false \ + -o "$out" "$MAIN" + + if [[ "$goos" == "darwin" ]]; then + file_info="$(file -b "$out")" + echo " ${file_info}" + case "$goarch" in + amd64) grep -Eq 'Mach-O 64-bit.*x86_64' <<< "$file_info" ;; + arm64) grep -Eq 'Mach-O 64-bit.*arm64' <<< "$file_info" ;; + esac + fi + done + + if [[ -x "${OUTDIR}/${BINARY}_linux_amd64" ]]; then + version_output="$("${OUTDIR}/${BINARY}_linux_amd64" --version)" + if [[ "${{ matrix.profile }}" == "runner" ]]; then + test "$version_output" = "runner v${VERSION}" + else + test "$version_output" = "aiscan v${VERSION}" + fi + fi + + echo "=== Binaries ===" + ls -lh "${OUTDIR}/" + + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: release-${{ matrix.id }} + path: dist/build/* + retention-days: 1 + + # ── Package release ───────────────────────────────────────────── + package: + needs: build + runs-on: ubuntu-22.04 + env: + TAG: ${{ inputs.tag }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Download all artifacts + uses: actions/download-artifact@v7 + with: + pattern: release-* + path: dist/build + merge-multiple: true + + - name: Install pinned UPX + env: + UPX_VERSION: 5.2.0 + UPX_SHA256: 3db5d3294707439db97866feab8d75d800f028f48481a40547411824da4288a1 + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/upx-${UPX_VERSION}-amd64_linux.tar.xz" + curl -fsSL \ + "https://github.com/upx/upx/releases/download/v${UPX_VERSION}/upx-${UPX_VERSION}-amd64_linux.tar.xz" \ + -o "${archive}" + echo "${UPX_SHA256} ${archive}" | sha256sum -c - + tar -xJf "${archive}" -C "${RUNNER_TEMP}" + echo "UPX_BIN=${RUNNER_TEMP}/upx-${UPX_VERSION}-amd64_linux/upx" >> "${GITHUB_ENV}" + + - name: Compress Windows amd64 binaries with UPX + run: | + set -euo pipefail + "${UPX_BIN}" --version + for f in dist/build/*_windows_amd64.exe; do + [ -f "$f" ] || continue + "${UPX_BIN}" "$f" + "${UPX_BIN}" -t "$f" + done + + - name: Package archives + run: | + set -euo pipefail + mkdir -p dist/release + for f in dist/build/*; do + [ -f "$f" ] || continue + base=$(basename "$f") + archive_name="${base%.exe}" + binary="${base%%_*}" + inner_name="$binary" + [[ "$base" == *.exe ]] && inner_name="${binary}.exe" + + tmpdir=$(mktemp -d) + cp "$f" "${tmpdir}/${inner_name}" + cp README.md "${tmpdir}/" 2>/dev/null || true + cp -r docs "${tmpdir}/" 2>/dev/null || true + (cd "$tmpdir" && zip -r - .) > "${GITHUB_WORKSPACE}/dist/release/${archive_name}.zip" + rm -rf "$tmpdir" + done + ls -lh dist/release + + - name: Generate checksums + run: | + cd dist/release + sha256sum *.zip > aiscan_checksums.txt + cat aiscan_checksums.txt + + - name: Upload release bundle + uses: actions/upload-artifact@v7 + with: + name: release-bundle + path: dist/release/* + retention-days: 1 + + # UPX can report a structurally valid PE that still crashes at process + # startup. Run the exact packaged Windows binaries before publishing them. + verify-windows-release: + needs: package + runs-on: windows-2022 + steps: + - name: Download release bundle + uses: actions/download-artifact@v7 + with: + name: release-bundle + path: dist/release + + - name: Smoke test packaged Windows binaries + shell: pwsh + env: + TAG: ${{ inputs.tag }} + run: | + $ErrorActionPreference = 'Stop' + $expectedVersion = "aiscan $env:TAG" + $cases = @( + @{ Archive = 'aiscan_windows_amd64.zip'; Binary = 'aiscan.exe'; Full = $false }, + @{ Archive = 'aiscan-full_windows_amd64.zip'; Binary = 'aiscan-full.exe'; Full = $true }, + @{ Archive = 'runner_windows_amd64.zip'; Binary = 'runner.exe'; Full = $false; Runner = $true } + ) + + foreach ($case in $cases) { + $destination = Join-Path $env:RUNNER_TEMP ([IO.Path]::GetFileNameWithoutExtension($case.Archive)) + Expand-Archive -LiteralPath (Join-Path 'dist/release' $case.Archive) -DestinationPath $destination + $binary = Join-Path $destination $case.Binary + $version = (& $binary --version | Out-String).Trim() + if ($LASTEXITCODE -ne 0) { + throw "$($case.Binary) --version exited with $LASTEXITCODE" + } + $caseExpectedVersion = if ($case.Runner) { "runner $env:TAG" } else { $expectedVersion } + if ($version -ne $caseExpectedVersion) { + throw "$($case.Binary) reported '$version', expected '$caseExpectedVersion'" + } + if ($case.Full) { + & $binary web --help | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "$($case.Binary) web --help exited with $LASTEXITCODE" + } + } + } diff --git a/.github/workflows/scanner-regression.yml b/.github/workflows/scanner-regression.yml index b7c255b0..9991a022 100644 --- a/.github/workflows/scanner-regression.yml +++ b/.github/workflows/scanner-regression.yml @@ -22,17 +22,55 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: - fetch-depth: 0 submodules: recursive - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - cache: true + cache: ${{ env.ACT != 'true' }} - name: Run bounded public scanner regressions run: | go test -tags "full integration re2_cgo re2_static" -count=1 -timeout 8m -v \ -run 'Test(ScannerPublicIntegration|FullScannerPublicIntegration)$' \ ./tools + + race-stress: + runs-on: ubuntu-22.04 + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: ${{ env.ACT != 'true' }} + + - name: Repeat agent and runner concurrency tests + run: | + go test -race -count=20 -timeout 15m \ + -run 'Test(ConcurrentEmitWhileRegistering|SetProviderRaceWithRun|ResetDoesNotAllowConcurrentPrompt|StreamingProviderEmitsMessageUpdates)$' \ + ./agent/... + go test -race -count=20 -timeout 15m \ + -run 'Test(StdioSameSessionFIFOOrder|StdioSessionsRunConcurrently|StdioDrainWaitsForInFlightAndQueued|RuntimeSessionDirectLoopUsesSessionScheduler|RuntimeSessionRejectsRequestsPastPendingLimit|SessionContextCancellationStopsActiveRun|ActiveRunSteersAsyncInputWithoutSecondLifecycle)$' \ + ./pkg/runner/... + + - name: Repeat web SSE, cancellation, and reload concurrency tests + run: | + go test -race -count=20 -timeout 20m \ + -run 'Test(BroadcastAOPEventPersistsRawEnvelope|ServeSSEWithSnapshotSubscribesBeforeReadingSnapshot|ServeSSEWithSnapshotDropsQueuedSnapshotDuplicates|SessionEventsReplayHasNoSideEffects|SessionEventsResumesAfterLastEventID|CancelRemoteScanStopsAgentAndPreservesCanceledStatus|CancelQueuedScanDoesNotWaitForConcurrencySlot|CancelTaskUsesControlChannelWhenTaskQueueIsFull|CancelTaskWaitsForSaturatedControlChannel|CompleteJobCannotOverwriteCanceledScan|BroadcastConfigReload|BroadcastConfigReloadWaitsBehindCancellationFrames|HandleConfigReloadResultUpdatesAgentStatus|SaveConfigBuildFailureKeepsCommittedConfigAndCurrentApp|SaveConfigCommitFailureClosesCandidateAndKeepsCurrentApp|SaveConfigSerializesConcurrentCandidates)$' \ + ./pkg/web + + - name: Fuzz AOP envelope decoding + working-directory: aop + run: | + go test -run '^$' \ + -fuzz '^FuzzEnvelopeBinaryRoundTrip$' \ + -fuzztime 30s \ + -timeout 2m \ + . diff --git a/.gitignore b/.gitignore index 0ff26c24..f337c8b7 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ web/frontend/test-results/ community.yaml # Local runtime state / operator artifacts +.aiscan/ /aiscan-deploy.yaml /*.log /.claude/ diff --git a/.goreleaser.yml b/.goreleaser.yml index 31c56200..399505be 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -21,9 +21,6 @@ builds: goarch: - amd64 - arm64 - ignore: - - goos: windows - goarch: arm64 flags: - -trimpath tags: @@ -39,12 +36,33 @@ builds: gcflags: - all=-trimpath={{.Env.GOPATH}} + - id: runner + main: ./cmd/runner + binary: runner + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + flags: + - -trimpath + ldflags: + - -s -w -X github.com/chainreactors/aiscan/core/config.Version={{.Version}} + asmflags: + - all=-trimpath={{.Env.GOPATH}} + gcflags: + - all=-trimpath={{.Env.GOPATH}} + - id: aiscan-full main: ./cmd/aiscan binary: "{{ .ProjectName }}-full" env: # Full links the native libcstx runtime. The official release workflow - # builds each target on a matching native runner. + # cross-compiles Darwin with Zig and a pinned macOS SDK on Linux. - CGO_ENABLED=1 goos: - linux @@ -91,6 +109,15 @@ archives: - src: README.md - src: docs/* + - id: runner + ids: [runner] + name_template: "runner_{{ .Os }}_{{ .Arch }}" + formats: + - zip + files: + - src: README.md + - src: docs/* + - id: aiscan-full ids: [aiscan-full] name_template: "{{ .ProjectName }}-full_{{ .Os }}_{{ .Arch }}" diff --git a/Makefile b/Makefile index 8cc4495d..41185e9c 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,13 @@ endif STANDARD_BIN ?= $(BIN_DIR)/aiscan$(EXE) FULL_BIN ?= $(BIN_DIR)/aiscan-full$(EXE) +RECORD_BIN ?= $(BIN_DIR)/aiscan-record$(EXE) +RUNNER_BIN ?= $(BIN_DIR)/runner$(EXE) # Standard/full match release artifacts. STANDARD_TAGS := forceposix emptytemplates noembed osusergo netgo -FULL_TAGS := forceposix emptytemplates noembed osusergo netgo full sqlite record_ffmpeg re2_cgo re2_static +FULL_TAGS := forceposix emptytemplates noembed osusergo netgo full sqlite re2_cgo re2_static +RECORD_TAGS := $(FULL_TAGS) record_ffmpeg BUILD_FLAGS := -trimpath -buildvcs=false GO_LDFLAGS ?= -s -w @@ -48,12 +51,14 @@ RECORD_EXTRA_LDFLAGS := endif RECORD_BUILD_ENV := PKG_CONFIG="$(RECORD_PKG_CONFIG)" PKG_CONFIG_PATH="$(RECORD_PREFIX)/lib/pkgconfig" CGO_CFLAGS="-I$(RECORD_PREFIX)/include" CGO_LDFLAGS="-L$(RECORD_PREFIX)/lib $(RECORD_EXTRA_LDFLAGS)" -.PHONY: help prepare frontend proto-gen standard record-native record-native-source record-native-package full web-build web-run web all clean +.PHONY: help prepare frontend proto-gen standard runner full record record-native record-native-source record-native-package web-build web-run web all clean help: @echo "AIScan build targets:" @echo " make / make standard Build the standard AIScan edition" + @echo " make runner Build the tag-free runner binary" @echo " make full Build frontend, then build the full edition" + @echo " make record Build the record-enabled edition (supported platforms only)" @echo " make web Build the full edition and start the Web UI" @echo " make frontend Build only web/frontend into web/static" @echo " make record-native Download the prebuilt FFmpeg/x264 recorder SDK" @@ -80,7 +85,11 @@ standard: prepare CGO_ENABLED=0 $(GO) build $(BUILD_FLAGS) -ldflags "$(GO_LDFLAGS)" -tags "$(STANDARD_TAGS)" -o "$(STANDARD_BIN)" ./cmd/aiscan @echo "Built standard edition: $(STANDARD_BIN)" -# The full binary embeds web/static, so frontend must finish first. +runner: prepare + CGO_ENABLED=0 $(GO) build $(BUILD_FLAGS) -ldflags "$(GO_LDFLAGS)" -o "$(RUNNER_BIN)" ./cmd/runner + @echo "Built runner: $(RUNNER_BIN)" + +# Full and record-enabled binaries embed web/static, so frontend must finish first. record-native: ifeq ($(RECORD_PLATFORM),unsupported) @echo "record native backend is not supported on this platform" @@ -106,10 +115,20 @@ else "$(BASH)" ".github/native/sdk.sh" package "$(RECORD_PLATFORM)" "$(RECORD_ARCH)" "$(RECORD_NATIVE_OUTPUT)" endif -full: frontend record-native prepare - $(RECORD_BUILD_ENV) CGO_ENABLED=1 $(GO) build $(BUILD_FLAGS) -ldflags "$(GO_LDFLAGS)" -tags "$(FULL_TAGS)" -o "$(FULL_BIN)" ./cmd/aiscan +full: frontend prepare + CGO_ENABLED=1 $(GO) build $(BUILD_FLAGS) -ldflags "$(GO_LDFLAGS)" -tags "$(FULL_TAGS)" -o "$(FULL_BIN)" ./cmd/aiscan @echo "Built full edition: $(FULL_BIN)" +ifeq ($(RECORD_PLATFORM),unsupported) +record: + @echo "record native backend is not supported on this platform" >&2 + @exit 1 +else +record: frontend record-native prepare + $(RECORD_BUILD_ENV) CGO_ENABLED=1 $(GO) build $(BUILD_FLAGS) -ldflags "$(GO_LDFLAGS)" -tags "$(RECORD_TAGS)" -o "$(RECORD_BIN)" ./cmd/aiscan + @echo "Built record-enabled edition: $(RECORD_BIN)" +endif + web-build: full web-run: @@ -118,7 +137,7 @@ web-run: web: full "$(FULL_BIN)" web --addr "$(WEB_ADDR)" $(if $(strip $(WEB_TOKEN)),--token "$(WEB_TOKEN)",) -all: standard full +all: standard runner full clean: - rm -f "$(STANDARD_BIN)" "$(FULL_BIN)" + rm -f "$(STANDARD_BIN)" "$(FULL_BIN)" "$(RECORD_BIN)" "$(RUNNER_BIN)" diff --git a/README.md b/README.md index 058c7adb..a943bc58 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,14 @@ From [GitHub Releases](https://github.com/chainreactors/aiscan/releases/latest): | Edition | Description | | --- | --- | | **aiscan** | Standard — scan/agent/gogo/spray/zombie/neutron/proton/arsenal | -| **aiscan-full** | Full — adds playwright, passive recon, katana, and native recording on supported Windows/Linux systems | +| **aiscan-full** | Full — adds Web, playwright, passive recon, and katana | +| **runner** | Single tag-free remote tool node | -| OS | Arch | Standard | Full | -| --- | --- | --- | --- | -| Linux | amd64 / arm64 | `aiscan_linux_.zip` | `aiscan-full_linux_.zip` | -| macOS | Intel / Apple Silicon | `aiscan_darwin_.zip` | `aiscan-full_darwin_.zip` | -| Windows | amd64 / arm64 | `aiscan_windows_.zip` | `aiscan-full_windows_amd64.zip` | +| OS | Arch | Standard | Full | Runner | +| --- | --- | --- | --- | --- | +| Linux | amd64 / arm64 | `aiscan_linux_.zip` | `aiscan-full_linux_.zip` | `runner_linux_.zip` | +| macOS | Intel / Apple Silicon | `aiscan_darwin_.zip` | `aiscan-full_darwin_.zip` | `runner_darwin_.zip` | +| Windows | amd64 / arm64 | `aiscan_windows_.zip` | `aiscan-full_windows_amd64.zip` | `runner_windows_.zip` | ```bash # Linux @@ -103,6 +104,7 @@ The Web console stores sessions, scans, assets, findings, and configuration in git clone https://github.com/chainreactors/aiscan.git && cd aiscan make # standard edition +make runner # tag-free remote tool runner make full # frontend + full edition ``` @@ -110,9 +112,9 @@ The standalone agent executable is no longer a maintained build or release target. Reference wiring remains in `examples/agent` and can be run manually with `go run ./examples/agent --help`. `make full` requires Node.js/npm and a working CGO toolchain; it builds the frontend first so the latest `web/static` -assets are embedded into the binary. On supported Windows/Linux targets it -also downloads and verifies the pinned recorder SDK; use -`make record-native-source` to build that SDK from pinned sources instead. +assets are embedded into the binary. The native `record` tool is not included +in the default full build; SDK and tool developers can build it explicitly with +`make record`, as described in [docs/record.md](docs/record.md). ```bash make web WEB_ADDR=127.0.0.1:18081 WEB_TOKEN=local-dev # full build + Web UI @@ -168,6 +170,8 @@ RE2, Abseil, libstdc++, libgcc, or winpthread DLLs. - playwright — headless Chromium sessions, screenshots, network capture - katana — web crawler with standard/headless/hybrid engines - passive — cyberspace search (FOFA, Hunter, Shodan) + +**Optional SDK tools** - record — native desktop/window screenshots and H.264/MP4 recording (Windows and Linux X11) **Utilities** diff --git a/README_CN.md b/README_CN.md index a19732fb..faeb3430 100644 --- a/README_CN.md +++ b/README_CN.md @@ -42,7 +42,7 @@ aiscan agent --base-url "https://api.deepseek.com" --api-key "sk-..." --model de | 版本 | 说明 | | --- | --- | | **aiscan** | 标准版 — scan/agent/gogo/spray/zombie/neutron/proton/arsenal | -| **aiscan-full** | 完整版 — 额外包含 playwright、passive、katana，以及受支持 Windows/Linux 系统上的原生录屏 | +| **aiscan-full** | 完整版 — 额外包含 Web、playwright、passive 和 katana | | 系统 | 架构 | 标准版 | 完整版 | | --- | --- | --- | --- | @@ -108,8 +108,8 @@ make full # 前端 + 完整版 独立 agent 可执行文件不再作为维护或发布目标。参考 wiring 已迁移到 `examples/agent`，需要时可手动运行 `go run ./examples/agent --help`。执行 `make full` 需要 Node.js/npm 和可用的 CGO 工具链；它会先构建前端，再将最新的 -`web/static` 嵌入 full 二进制。在受支持的 Windows/Linux 目标上，它还会下载并 -校验固定版本的录屏 SDK；使用 `make record-native-source` 可从固定源码构建该 SDK。 +`web/static` 嵌入 full 二进制。默认 full 构建不包含原生 `record` 工具；SDK 和工具 +开发者可通过 `make record` 显式构建，详见 [record 文档](docs/record.md)。 ```bash make web WEB_ADDR=127.0.0.1:18081 WEB_TOKEN=local-dev # Full 构建并启动 Web UI @@ -165,6 +165,8 @@ libstdc++、libgcc 或 winpthread DLL。 - playwright — headless Chromium 会话、截图、网络捕获 - katana — Web 爬虫，支持 standard/headless/hybrid 引擎 - passive — 网络空间搜索（FOFA、Hunter、Shodan） + +**可选 SDK 工具** - record — 原生桌面/窗口截图和 H.264/MP4 录屏（Windows 与 Linux X11） **辅助工具** diff --git a/agent/defaults.go b/agent/defaults.go index 1b6a392e..f1919539 100644 --- a/agent/defaults.go +++ b/agent/defaults.go @@ -11,6 +11,6 @@ const ( DefaultKeepRecentTokens = 20000 DefaultTokenBudgetWarningPct = 80 DefaultInboxCapacity = 64 - SubInboxCapacity = 16 + SubInboxCapacity = 64 DefaultMaxParallelTools = 16 ) diff --git a/agent/inbox/inbox.go b/agent/inbox/inbox.go index 06608265..582f1be8 100644 --- a/agent/inbox/inbox.go +++ b/agent/inbox/inbox.go @@ -61,7 +61,20 @@ func (b *Buffered) Push(msg Message) error { return ErrInboxClosed } if len(b.buf) >= b.capacity { - return ErrInboxFull + victim := -1 + for i := range b.buf { + if b.buf[i].Priority >= msg.Priority { + continue + } + if victim < 0 || b.buf[i].Priority < b.buf[victim].Priority { + victim = i + } + } + if victim < 0 { + return ErrInboxFull + } + copy(b.buf[victim:], b.buf[victim+1:]) + b.buf = b.buf[:len(b.buf)-1] } wasEmpty := len(b.buf) == 0 b.buf = append(b.buf, msg) diff --git a/agent/inbox/inbox_test.go b/agent/inbox/inbox_test.go index b3de3b52..407af2bb 100644 --- a/agent/inbox/inbox_test.go +++ b/agent/inbox/inbox_test.go @@ -37,6 +37,27 @@ func TestBufferedCapacity(t *testing.T) { } } +func TestHigherPriorityMessageEvictsLowerPriorityWhenFull(t *testing.T) { + b := NewBuffered(2) + if err := b.Push(NewUserMessage("low-1").WithPriority(PriorityLow)); err != nil { + t.Fatal(err) + } + if err := b.Push(NewUserMessage("low-2").WithPriority(PriorityLow)); err != nil { + t.Fatal(err) + } + if err := b.Push(NewUserMessage("completion").WithPriority(PriorityHigh)); err != nil { + t.Fatalf("high-priority push = %v", err) + } + + msgs := b.Drain() + if len(msgs) != 2 { + t.Fatalf("messages = %d, want 2", len(msgs)) + } + if msgs[0].Priority != PriorityHigh || messageText(msgs[0].Message) != "completion" { + t.Fatalf("first message = %+v", msgs[0]) + } +} + func TestBufferedClose(t *testing.T) { b := NewBuffered(4) b.Push(NewUserMessage("a")) diff --git a/agent/tmux/manager_test.go b/agent/tmux/manager_test.go index 1be312f2..5c6abc2c 100644 --- a/agent/tmux/manager_test.go +++ b/agent/tmux/manager_test.go @@ -185,7 +185,7 @@ func TestPeekReturnsTail(t *testing.T) { } mgr := NewManager() dir := t.TempDir() - info, err := mgr.Create(dir, "for i in 1 2 3 4 5; do echo line$i; done", "peek-test", 5*time.Second, nil, "") + info, err := mgr.Create(dir, "for i in 1 2 3 4 5; do echo line$i; done; sleep 0.05", "peek-test", 5*time.Second, nil, "") if err != nil { t.Fatalf("Create: %v", err) } @@ -305,7 +305,7 @@ func TestCreateCmd(t *testing.T) { mgr := NewManager() dir := t.TempDir() - info, err := mgr.CreateCmd(dir, "/bin/sh", []string{"-c", "echo from-createcmd"}, "cmd-test", 10*time.Second, nil, "") + info, err := mgr.CreateCmd(dir, "/bin/sh", []string{"-c", "echo from-createcmd; sleep 0.05"}, "cmd-test", 10*time.Second, nil, "") if err != nil { t.Fatalf("CreateCmd: %v", err) } @@ -324,7 +324,7 @@ func TestCreateCmdWithEnv(t *testing.T) { mgr := NewManager() dir := t.TempDir() - info, err := mgr.CreateCmd(dir, "/bin/sh", []string{"-c", "echo $TEST_MAGIC"}, "env-test", 10*time.Second, []string{"TEST_MAGIC=pty_works"}, "") + info, err := mgr.CreateCmd(dir, "/bin/sh", []string{"-c", "echo $TEST_MAGIC; sleep 0.05"}, "env-test", 10*time.Second, []string{"TEST_MAGIC=pty_works"}, "") if err != nil { t.Fatalf("CreateCmd: %v", err) } @@ -370,7 +370,7 @@ func TestPeekNew(t *testing.T) { dir := t.TempDir() payload := strings.Repeat("x", 100) - info, err := mgr.Create(dir, "printf '"+payload+"'", "peeknew-test", 10*time.Second, nil, "") + info, err := mgr.Create(dir, "printf '"+payload+"'; sleep 0.05", "peeknew-test", 10*time.Second, nil, "") if err != nil { t.Fatalf("Create: %v", err) } @@ -473,7 +473,7 @@ func TestExecCommandDirect(t *testing.T) { mgr := NewManager() dir := t.TempDir() - info, err := mgr.CreateCmd(dir, "/bin/sh", []string{"-c", "echo direct"}, "", 5*time.Second, nil, "") + info, err := mgr.CreateCmd(dir, "/bin/sh", []string{"-c", "echo direct; sleep 0.05"}, "", 5*time.Second, nil, "") if err != nil { t.Fatalf("CreateCmd: %v", err) } @@ -530,7 +530,7 @@ func TestPeekBytes(t *testing.T) { t.Skip("unix-only test") } - info, err := mgr.Create(dir, "printf '0123456789'", "peekbytes-test", 5*time.Second, nil, "") + info, err := mgr.Create(dir, "printf '0123456789'; sleep 0.05", "peekbytes-test", 5*time.Second, nil, "") if err != nil { t.Fatal(err) } diff --git a/aop/file/protocol.pb.go b/aop/file/protocol.pb.go index 69fefa4b..48a1ffc9 100644 --- a/aop/file/protocol.pb.go +++ b/aop/file/protocol.pb.go @@ -9,6 +9,7 @@ package file import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" unsafe "unsafe" @@ -21,6 +22,125 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// AccessOp is what happened to the path. EDIT is a targeted patch and WRITE a +// full-content overwrite; both are distinguished from CREATE, which says the +// path did not exist beforehand. +type AccessOp int32 + +const ( + AccessOp_ACCESS_OP_UNSPECIFIED AccessOp = 0 + AccessOp_ACCESS_OP_READ AccessOp = 1 + AccessOp_ACCESS_OP_WRITE AccessOp = 2 + AccessOp_ACCESS_OP_EDIT AccessOp = 3 + AccessOp_ACCESS_OP_CREATE AccessOp = 4 + AccessOp_ACCESS_OP_DELETE AccessOp = 5 +) + +// Enum value maps for AccessOp. +var ( + AccessOp_name = map[int32]string{ + 0: "ACCESS_OP_UNSPECIFIED", + 1: "ACCESS_OP_READ", + 2: "ACCESS_OP_WRITE", + 3: "ACCESS_OP_EDIT", + 4: "ACCESS_OP_CREATE", + 5: "ACCESS_OP_DELETE", + } + AccessOp_value = map[string]int32{ + "ACCESS_OP_UNSPECIFIED": 0, + "ACCESS_OP_READ": 1, + "ACCESS_OP_WRITE": 2, + "ACCESS_OP_EDIT": 3, + "ACCESS_OP_CREATE": 4, + "ACCESS_OP_DELETE": 5, + } +) + +func (x AccessOp) Enum() *AccessOp { + p := new(AccessOp) + *p = x + return p +} + +func (x AccessOp) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AccessOp) Descriptor() protoreflect.EnumDescriptor { + return file_aop_file_protocol_proto_enumTypes[0].Descriptor() +} + +func (AccessOp) Type() protoreflect.EnumType { + return &file_aop_file_protocol_proto_enumTypes[0] +} + +func (x AccessOp) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AccessOp.Descriptor instead. +func (AccessOp) EnumDescriptor() ([]byte, []int) { + return file_aop_file_protocol_proto_rawDescGZIP(), []int{0} +} + +// AccessSource is how the access was observed, which is also how far it can be +// trusted. TOOL is an exact record taken inside the tool that performed it. +// SNAPSHOT is derived by diffing the work dir around a shell execution: the +// path and the operation are real, but attribution to that execution is an +// inference, and reads are invisible to it entirely. CONTROL is a file request +// this node served for a peer rather than anything the agent did. +type AccessSource int32 + +const ( + AccessSource_ACCESS_SOURCE_UNSPECIFIED AccessSource = 0 + AccessSource_ACCESS_SOURCE_TOOL AccessSource = 1 + AccessSource_ACCESS_SOURCE_SNAPSHOT AccessSource = 2 + AccessSource_ACCESS_SOURCE_CONTROL AccessSource = 3 +) + +// Enum value maps for AccessSource. +var ( + AccessSource_name = map[int32]string{ + 0: "ACCESS_SOURCE_UNSPECIFIED", + 1: "ACCESS_SOURCE_TOOL", + 2: "ACCESS_SOURCE_SNAPSHOT", + 3: "ACCESS_SOURCE_CONTROL", + } + AccessSource_value = map[string]int32{ + "ACCESS_SOURCE_UNSPECIFIED": 0, + "ACCESS_SOURCE_TOOL": 1, + "ACCESS_SOURCE_SNAPSHOT": 2, + "ACCESS_SOURCE_CONTROL": 3, + } +) + +func (x AccessSource) Enum() *AccessSource { + p := new(AccessSource) + *p = x + return p +} + +func (x AccessSource) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AccessSource) Descriptor() protoreflect.EnumDescriptor { + return file_aop_file_protocol_proto_enumTypes[1].Descriptor() +} + +func (AccessSource) Type() protoreflect.EnumType { + return &file_aop_file_protocol_proto_enumTypes[1] +} + +func (x AccessSource) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AccessSource.Descriptor instead. +func (AccessSource) EnumDescriptor() ([]byte, []int) { + return file_aop_file_protocol_proto_rawDescGZIP(), []int{1} +} + type ReadRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -453,6 +573,306 @@ func (x *Result) GetEof() bool { return false } +// Access is one observed file access. +type Access struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // tool_id is the AOP tool-call id whose execution produced this access, empty + // when it happened outside one (a control request, a detached session). + ToolId string `protobuf:"bytes,2,opt,name=tool_id,json=toolId,proto3" json:"tool_id,omitempty"` + Op AccessOp `protobuf:"varint,3,opt,name=op,proto3,enum=aop.file.AccessOp" json:"op,omitempty"` + Source AccessSource `protobuf:"varint,4,opt,name=source,proto3,enum=aop.file.AccessSource" json:"source,omitempty"` + // path is absolute; work_dir is the execution's working directory, carried so + // a consumer can present the path relative to it without guessing. + Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"` + WorkDir string `protobuf:"bytes,6,opt,name=work_dir,json=workDir,proto3" json:"work_dir,omitempty"` + Size int64 `protobuf:"varint,7,opt,name=size,proto3" json:"size,omitempty"` // file size after the access + Bytes int64 `protobuf:"varint,8,opt,name=bytes,proto3" json:"bytes,omitempty"` // bytes read or written by this access, 0 when unknown + Edits uint32 `protobuf:"varint,9,opt,name=edits,proto3" json:"edits,omitempty"` // patch count for EDIT + Digest string `protobuf:"bytes,10,opt,name=digest,proto3" json:"digest,omitempty"` // sha256 of the content after a write, when computed + Error string `protobuf:"bytes,11,opt,name=error,proto3" json:"error,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Access) Reset() { + *x = Access{} + mi := &file_aop_file_protocol_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Access) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Access) ProtoMessage() {} + +func (x *Access) ProtoReflect() protoreflect.Message { + mi := &file_aop_file_protocol_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Access.ProtoReflect.Descriptor instead. +func (*Access) Descriptor() ([]byte, []int) { + return file_aop_file_protocol_proto_rawDescGZIP(), []int{7} +} + +func (x *Access) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Access) GetToolId() string { + if x != nil { + return x.ToolId + } + return "" +} + +func (x *Access) GetOp() AccessOp { + if x != nil { + return x.Op + } + return AccessOp_ACCESS_OP_UNSPECIFIED +} + +func (x *Access) GetSource() AccessSource { + if x != nil { + return x.Source + } + return AccessSource_ACCESS_SOURCE_UNSPECIFIED +} + +func (x *Access) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *Access) GetWorkDir() string { + if x != nil { + return x.WorkDir + } + return "" +} + +func (x *Access) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *Access) GetBytes() int64 { + if x != nil { + return x.Bytes + } + return 0 +} + +func (x *Access) GetEdits() uint32 { + if x != nil { + return x.Edits + } + return 0 +} + +func (x *Access) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *Access) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *Access) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +// WatchConfig steers observation. Disabling it stops the node reporting, which +// is the only way a peer can opt out of the stream it would otherwise receive. +type WatchConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` + // ignore holds path substrings excluded from snapshot diffing; empty leaves + // the node's own defaults (VCS metadata, dependency trees) in place. + Ignore []string `protobuf:"bytes,2,rep,name=ignore,proto3" json:"ignore,omitempty"` + // max_entries bounds one snapshot. A work dir over it is not diffed at all, + // and the node says so through an Access carrying error rather than + // reporting a silently partial diff. + MaxEntries uint32 `protobuf:"varint,3,opt,name=max_entries,json=maxEntries,proto3" json:"max_entries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchConfig) Reset() { + *x = WatchConfig{} + mi := &file_aop_file_protocol_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchConfig) ProtoMessage() {} + +func (x *WatchConfig) ProtoReflect() protoreflect.Message { + mi := &file_aop_file_protocol_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchConfig.ProtoReflect.Descriptor instead. +func (*WatchConfig) Descriptor() ([]byte, []int) { + return file_aop_file_protocol_proto_rawDescGZIP(), []int{8} +} + +func (x *WatchConfig) GetEnabled() bool { + if x != nil { + return x.Enabled + } + return false +} + +func (x *WatchConfig) GetIgnore() []string { + if x != nil { + return x.Ignore + } + return nil +} + +func (x *WatchConfig) GetMaxEntries() uint32 { + if x != nil { + return x.MaxEntries + } + return 0 +} + +type Configure struct { + state protoimpl.MessageState `protogen:"open.v1"` + Watch *WatchConfig `protobuf:"bytes,1,opt,name=watch,proto3" json:"watch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Configure) Reset() { + *x = Configure{} + mi := &file_aop_file_protocol_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Configure) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Configure) ProtoMessage() {} + +func (x *Configure) ProtoReflect() protoreflect.Message { + mi := &file_aop_file_protocol_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Configure.ProtoReflect.Descriptor instead. +func (*Configure) Descriptor() ([]byte, []int) { + return file_aop_file_protocol_proto_rawDescGZIP(), []int{9} +} + +func (x *Configure) GetWatch() *WatchConfig { + if x != nil { + return x.Watch + } + return nil +} + +type WatchState struct { + state protoimpl.MessageState `protogen:"open.v1"` + Watching bool `protobuf:"varint,1,opt,name=watching,proto3" json:"watching,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchState) Reset() { + *x = WatchState{} + mi := &file_aop_file_protocol_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchState) ProtoMessage() {} + +func (x *WatchState) ProtoReflect() protoreflect.Message { + mi := &file_aop_file_protocol_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchState.ProtoReflect.Descriptor instead. +func (*WatchState) Descriptor() ([]byte, []int) { + return file_aop_file_protocol_proto_rawDescGZIP(), []int{10} +} + +func (x *WatchState) GetWatching() bool { + if x != nil { + return x.Watching + } + return false +} + +func (x *WatchState) GetError() string { + if x != nil { + return x.Error + } + return "" +} + type ProtocolMessage struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Message: @@ -463,6 +883,9 @@ type ProtocolMessage struct { // *ProtocolMessage_MkdirRequest // *ProtocolMessage_UploadRequest // *ProtocolMessage_Result + // *ProtocolMessage_Configure + // *ProtocolMessage_State + // *ProtocolMessage_Access Message isProtocolMessage_Message `protobuf_oneof:"message"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -470,7 +893,7 @@ type ProtocolMessage struct { func (x *ProtocolMessage) Reset() { *x = ProtocolMessage{} - mi := &file_aop_file_protocol_proto_msgTypes[7] + mi := &file_aop_file_protocol_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -482,7 +905,7 @@ func (x *ProtocolMessage) String() string { func (*ProtocolMessage) ProtoMessage() {} func (x *ProtocolMessage) ProtoReflect() protoreflect.Message { - mi := &file_aop_file_protocol_proto_msgTypes[7] + mi := &file_aop_file_protocol_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -495,7 +918,7 @@ func (x *ProtocolMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtocolMessage.ProtoReflect.Descriptor instead. func (*ProtocolMessage) Descriptor() ([]byte, []int) { - return file_aop_file_protocol_proto_rawDescGZIP(), []int{7} + return file_aop_file_protocol_proto_rawDescGZIP(), []int{11} } func (x *ProtocolMessage) GetMessage() isProtocolMessage_Message { @@ -559,6 +982,33 @@ func (x *ProtocolMessage) GetResult() *Result { return nil } +func (x *ProtocolMessage) GetConfigure() *Configure { + if x != nil { + if x, ok := x.Message.(*ProtocolMessage_Configure); ok { + return x.Configure + } + } + return nil +} + +func (x *ProtocolMessage) GetState() *WatchState { + if x != nil { + if x, ok := x.Message.(*ProtocolMessage_State); ok { + return x.State + } + } + return nil +} + +func (x *ProtocolMessage) GetAccess() *Access { + if x != nil { + if x, ok := x.Message.(*ProtocolMessage_Access); ok { + return x.Access + } + } + return nil +} + type isProtocolMessage_Message interface { isProtocolMessage_Message() } @@ -587,6 +1037,18 @@ type ProtocolMessage_Result struct { Result *Result `protobuf:"bytes,20,opt,name=result,proto3,oneof"` } +type ProtocolMessage_Configure struct { + Configure *Configure `protobuf:"bytes,21,opt,name=configure,proto3,oneof"` +} + +type ProtocolMessage_State struct { + State *WatchState `protobuf:"bytes,22,opt,name=state,proto3,oneof"` +} + +type ProtocolMessage_Access struct { + Access *Access `protobuf:"bytes,23,opt,name=access,proto3,oneof"` +} + func (*ProtocolMessage_ReadRequest) isProtocolMessage_Message() {} func (*ProtocolMessage_WriteRequest) isProtocolMessage_Message() {} @@ -599,11 +1061,17 @@ func (*ProtocolMessage_UploadRequest) isProtocolMessage_Message() {} func (*ProtocolMessage_Result) isProtocolMessage_Message() {} +func (*ProtocolMessage_Configure) isProtocolMessage_Message() {} + +func (*ProtocolMessage_State) isProtocolMessage_Message() {} + +func (*ProtocolMessage_Access) isProtocolMessage_Message() {} + var File_aop_file_protocol_proto protoreflect.FileDescriptor const file_aop_file_protocol_proto_rawDesc = "" + "\n" + - "\x17aop/file/protocol.proto\x12\baop.file\"O\n" + + "\x17aop/file/protocol.proto\x12\baop.file\x1a\x1fgoogle/protobuf/timestamp.proto\"O\n" + "\vReadRequest\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x16\n" + "\x06offset\x18\x02 \x01(\x03R\x06offset\x12\x14\n" + @@ -635,7 +1103,32 @@ const file_aop_file_protocol_proto_rawDesc = "" + "\n" + "media_type\x18\x06 \x01(\tR\tmediaType\x12\x16\n" + "\x06offset\x18\a \x01(\x03R\x06offset\x12\x10\n" + - "\x03eof\x18\b \x01(\bR\x03eof\"\x80\x03\n" + + "\x03eof\x18\b \x01(\bR\x03eof\"\xdc\x02\n" + + "\x06Access\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x17\n" + + "\atool_id\x18\x02 \x01(\tR\x06toolId\x12\"\n" + + "\x02op\x18\x03 \x01(\x0e2\x12.aop.file.AccessOpR\x02op\x12.\n" + + "\x06source\x18\x04 \x01(\x0e2\x16.aop.file.AccessSourceR\x06source\x12\x12\n" + + "\x04path\x18\x05 \x01(\tR\x04path\x12\x19\n" + + "\bwork_dir\x18\x06 \x01(\tR\aworkDir\x12\x12\n" + + "\x04size\x18\a \x01(\x03R\x04size\x12\x14\n" + + "\x05bytes\x18\b \x01(\x03R\x05bytes\x12\x14\n" + + "\x05edits\x18\t \x01(\rR\x05edits\x12\x16\n" + + "\x06digest\x18\n" + + " \x01(\tR\x06digest\x12\x14\n" + + "\x05error\x18\v \x01(\tR\x05error\x128\n" + + "\ttimestamp\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\"`\n" + + "\vWatchConfig\x12\x18\n" + + "\aenabled\x18\x01 \x01(\bR\aenabled\x12\x16\n" + + "\x06ignore\x18\x02 \x03(\tR\x06ignore\x12\x1f\n" + + "\vmax_entries\x18\x03 \x01(\rR\n" + + "maxEntries\"8\n" + + "\tConfigure\x12+\n" + + "\x05watch\x18\x01 \x01(\v2\x15.aop.file.WatchConfigR\x05watch\">\n" + + "\n" + + "WatchState\x12\x1a\n" + + "\bwatching\x18\x01 \x01(\bR\bwatching\x12\x14\n" + + "\x05error\x18\x02 \x01(\tR\x05error\"\x8f\x04\n" + "\x0fProtocolMessage\x12:\n" + "\fread_request\x18\n" + " \x01(\v2\x15.aop.file.ReadRequestH\x00R\vreadRequest\x12=\n" + @@ -643,8 +1136,23 @@ const file_aop_file_protocol_proto_rawDesc = "" + "\flist_request\x18\f \x01(\v2\x15.aop.file.ListRequestH\x00R\vlistRequest\x12=\n" + "\rmkdir_request\x18\r \x01(\v2\x16.aop.file.MkdirRequestH\x00R\fmkdirRequest\x12@\n" + "\x0eupload_request\x18\x0e \x01(\v2\x17.aop.file.UploadRequestH\x00R\ruploadRequest\x12*\n" + - "\x06result\x18\x14 \x01(\v2\x10.aop.file.ResultH\x00R\x06resultB\t\n" + - "\amessageB/Z-github.com/chainreactors/aiscan/aop/file;fileb\x06proto3" + "\x06result\x18\x14 \x01(\v2\x10.aop.file.ResultH\x00R\x06result\x123\n" + + "\tconfigure\x18\x15 \x01(\v2\x13.aop.file.ConfigureH\x00R\tconfigure\x12,\n" + + "\x05state\x18\x16 \x01(\v2\x14.aop.file.WatchStateH\x00R\x05state\x12*\n" + + "\x06access\x18\x17 \x01(\v2\x10.aop.file.AccessH\x00R\x06accessB\t\n" + + "\amessage*\x8e\x01\n" + + "\bAccessOp\x12\x19\n" + + "\x15ACCESS_OP_UNSPECIFIED\x10\x00\x12\x12\n" + + "\x0eACCESS_OP_READ\x10\x01\x12\x13\n" + + "\x0fACCESS_OP_WRITE\x10\x02\x12\x12\n" + + "\x0eACCESS_OP_EDIT\x10\x03\x12\x14\n" + + "\x10ACCESS_OP_CREATE\x10\x04\x12\x14\n" + + "\x10ACCESS_OP_DELETE\x10\x05*|\n" + + "\fAccessSource\x12\x1d\n" + + "\x19ACCESS_SOURCE_UNSPECIFIED\x10\x00\x12\x16\n" + + "\x12ACCESS_SOURCE_TOOL\x10\x01\x12\x1a\n" + + "\x16ACCESS_SOURCE_SNAPSHOT\x10\x02\x12\x19\n" + + "\x15ACCESS_SOURCE_CONTROL\x10\x03B/Z-github.com/chainreactors/aiscan/aop/file;fileb\x06proto3" var ( file_aop_file_protocol_proto_rawDescOnce sync.Once @@ -658,30 +1166,45 @@ func file_aop_file_protocol_proto_rawDescGZIP() []byte { return file_aop_file_protocol_proto_rawDescData } -var file_aop_file_protocol_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_aop_file_protocol_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_aop_file_protocol_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_aop_file_protocol_proto_goTypes = []any{ - (*ReadRequest)(nil), // 0: aop.file.ReadRequest - (*WriteRequest)(nil), // 1: aop.file.WriteRequest - (*ListRequest)(nil), // 2: aop.file.ListRequest - (*MkdirRequest)(nil), // 3: aop.file.MkdirRequest - (*UploadRequest)(nil), // 4: aop.file.UploadRequest - (*Entry)(nil), // 5: aop.file.Entry - (*Result)(nil), // 6: aop.file.Result - (*ProtocolMessage)(nil), // 7: aop.file.ProtocolMessage + (AccessOp)(0), // 0: aop.file.AccessOp + (AccessSource)(0), // 1: aop.file.AccessSource + (*ReadRequest)(nil), // 2: aop.file.ReadRequest + (*WriteRequest)(nil), // 3: aop.file.WriteRequest + (*ListRequest)(nil), // 4: aop.file.ListRequest + (*MkdirRequest)(nil), // 5: aop.file.MkdirRequest + (*UploadRequest)(nil), // 6: aop.file.UploadRequest + (*Entry)(nil), // 7: aop.file.Entry + (*Result)(nil), // 8: aop.file.Result + (*Access)(nil), // 9: aop.file.Access + (*WatchConfig)(nil), // 10: aop.file.WatchConfig + (*Configure)(nil), // 11: aop.file.Configure + (*WatchState)(nil), // 12: aop.file.WatchState + (*ProtocolMessage)(nil), // 13: aop.file.ProtocolMessage + (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp } var file_aop_file_protocol_proto_depIdxs = []int32{ - 5, // 0: aop.file.Result.entries:type_name -> aop.file.Entry - 0, // 1: aop.file.ProtocolMessage.read_request:type_name -> aop.file.ReadRequest - 1, // 2: aop.file.ProtocolMessage.write_request:type_name -> aop.file.WriteRequest - 2, // 3: aop.file.ProtocolMessage.list_request:type_name -> aop.file.ListRequest - 3, // 4: aop.file.ProtocolMessage.mkdir_request:type_name -> aop.file.MkdirRequest - 4, // 5: aop.file.ProtocolMessage.upload_request:type_name -> aop.file.UploadRequest - 6, // 6: aop.file.ProtocolMessage.result:type_name -> aop.file.Result - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 7, // 0: aop.file.Result.entries:type_name -> aop.file.Entry + 0, // 1: aop.file.Access.op:type_name -> aop.file.AccessOp + 1, // 2: aop.file.Access.source:type_name -> aop.file.AccessSource + 14, // 3: aop.file.Access.timestamp:type_name -> google.protobuf.Timestamp + 10, // 4: aop.file.Configure.watch:type_name -> aop.file.WatchConfig + 2, // 5: aop.file.ProtocolMessage.read_request:type_name -> aop.file.ReadRequest + 3, // 6: aop.file.ProtocolMessage.write_request:type_name -> aop.file.WriteRequest + 4, // 7: aop.file.ProtocolMessage.list_request:type_name -> aop.file.ListRequest + 5, // 8: aop.file.ProtocolMessage.mkdir_request:type_name -> aop.file.MkdirRequest + 6, // 9: aop.file.ProtocolMessage.upload_request:type_name -> aop.file.UploadRequest + 8, // 10: aop.file.ProtocolMessage.result:type_name -> aop.file.Result + 11, // 11: aop.file.ProtocolMessage.configure:type_name -> aop.file.Configure + 12, // 12: aop.file.ProtocolMessage.state:type_name -> aop.file.WatchState + 9, // 13: aop.file.ProtocolMessage.access:type_name -> aop.file.Access + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_aop_file_protocol_proto_init() } @@ -689,26 +1212,30 @@ func file_aop_file_protocol_proto_init() { if File_aop_file_protocol_proto != nil { return } - file_aop_file_protocol_proto_msgTypes[7].OneofWrappers = []any{ + file_aop_file_protocol_proto_msgTypes[11].OneofWrappers = []any{ (*ProtocolMessage_ReadRequest)(nil), (*ProtocolMessage_WriteRequest)(nil), (*ProtocolMessage_ListRequest)(nil), (*ProtocolMessage_MkdirRequest)(nil), (*ProtocolMessage_UploadRequest)(nil), (*ProtocolMessage_Result)(nil), + (*ProtocolMessage_Configure)(nil), + (*ProtocolMessage_State)(nil), + (*ProtocolMessage_Access)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_aop_file_protocol_proto_rawDesc), len(file_aop_file_protocol_proto_rawDesc)), - NumEnums: 0, - NumMessages: 8, + NumEnums: 2, + NumMessages: 12, NumExtensions: 0, NumServices: 0, }, GoTypes: file_aop_file_protocol_proto_goTypes, DependencyIndexes: file_aop_file_protocol_proto_depIdxs, + EnumInfos: file_aop_file_protocol_proto_enumTypes, MessageInfos: file_aop_file_protocol_proto_msgTypes, }.Build() File_aop_file_protocol_proto = out.File diff --git a/aop/traffic/protocol.pb.go b/aop/traffic/protocol.pb.go new file mode 100644 index 00000000..ea812ec2 --- /dev/null +++ b/aop/traffic/protocol.pb.go @@ -0,0 +1,1144 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v7.35.1 +// source: aop/traffic/protocol.proto + +package traffic + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// CaptureMode selects what the hub does with traffic it routes. RELAY forwards +// undecrypted and records nothing; RECORD intercepts (MITM) and stores flows. +type CaptureMode int32 + +const ( + CaptureMode_CAPTURE_MODE_UNSPECIFIED CaptureMode = 0 // leave capture unchanged (Configure) + CaptureMode_CAPTURE_MODE_RELAY CaptureMode = 1 // route only: no interception, no record + CaptureMode_CAPTURE_MODE_RECORD CaptureMode = 2 // intercept + record +) + +// Enum value maps for CaptureMode. +var ( + CaptureMode_name = map[int32]string{ + 0: "CAPTURE_MODE_UNSPECIFIED", + 1: "CAPTURE_MODE_RELAY", + 2: "CAPTURE_MODE_RECORD", + } + CaptureMode_value = map[string]int32{ + "CAPTURE_MODE_UNSPECIFIED": 0, + "CAPTURE_MODE_RELAY": 1, + "CAPTURE_MODE_RECORD": 2, + } +) + +func (x CaptureMode) Enum() *CaptureMode { + p := new(CaptureMode) + *p = x + return p +} + +func (x CaptureMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CaptureMode) Descriptor() protoreflect.EnumDescriptor { + return file_aop_traffic_protocol_proto_enumTypes[0].Descriptor() +} + +func (CaptureMode) Type() protoreflect.EnumType { + return &file_aop_traffic_protocol_proto_enumTypes[0] +} + +func (x CaptureMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CaptureMode.Descriptor instead. +func (CaptureMode) EnumDescriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{0} +} + +// RoutingMode selects how the egress chain is set. UNSPECIFIED leaves routing +// unchanged so a Configure can steer capture without touching the proxy. +type RoutingMode int32 + +const ( + RoutingMode_ROUTING_MODE_UNSPECIFIED RoutingMode = 0 + RoutingMode_ROUTING_MODE_DIRECT RoutingMode = 1 // revert to the original/direct egress + RoutingMode_ROUTING_MODE_PROXY RoutingMode = 2 // single proxy URL (url) + RoutingMode_ROUTING_MODE_SUBSCRIBE RoutingMode = 3 // load a clash subscription (url), no switch + RoutingMode_ROUTING_MODE_AUTO RoutingMode = 4 // subscription + adaptive load balancing + RoutingMode_ROUTING_MODE_SWITCH RoutingMode = 5 // switch active node within a loaded subscription + RoutingMode_ROUTING_MODE_CLEAR RoutingMode = 6 // clear subscription, revert to original +) + +// Enum value maps for RoutingMode. +var ( + RoutingMode_name = map[int32]string{ + 0: "ROUTING_MODE_UNSPECIFIED", + 1: "ROUTING_MODE_DIRECT", + 2: "ROUTING_MODE_PROXY", + 3: "ROUTING_MODE_SUBSCRIBE", + 4: "ROUTING_MODE_AUTO", + 5: "ROUTING_MODE_SWITCH", + 6: "ROUTING_MODE_CLEAR", + } + RoutingMode_value = map[string]int32{ + "ROUTING_MODE_UNSPECIFIED": 0, + "ROUTING_MODE_DIRECT": 1, + "ROUTING_MODE_PROXY": 2, + "ROUTING_MODE_SUBSCRIBE": 3, + "ROUTING_MODE_AUTO": 4, + "ROUTING_MODE_SWITCH": 5, + "ROUTING_MODE_CLEAR": 6, + } +) + +func (x RoutingMode) Enum() *RoutingMode { + p := new(RoutingMode) + *p = x + return p +} + +func (x RoutingMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RoutingMode) Descriptor() protoreflect.EnumDescriptor { + return file_aop_traffic_protocol_proto_enumTypes[1].Descriptor() +} + +func (RoutingMode) Type() protoreflect.EnumType { + return &file_aop_traffic_protocol_proto_enumTypes[1] +} + +func (x RoutingMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RoutingMode.Descriptor instead. +func (RoutingMode) EnumDescriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{1} +} + +// RoutingConfig steers the egress chain (State in tools/proxy). Fields beyond +// mode/url/selector are the auto-mode subscription filters. +type RoutingConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode RoutingMode `protobuf:"varint,1,opt,name=mode,proto3,enum=aop.traffic.RoutingMode" json:"mode,omitempty"` + Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` // proxy URL (PROXY) or subscription URL (SUBSCRIBE/AUTO) + Selector string `protobuf:"bytes,3,opt,name=selector,proto3" json:"selector,omitempty"` // node name or 1-based index (SWITCH) + Type string `protobuf:"bytes,4,opt,name=type,proto3" json:"type,omitempty"` // protocol filter, e.g. "trojan,vless" (AUTO) + Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` // node name keyword (AUTO) + Country string `protobuf:"bytes,6,opt,name=country,proto3" json:"country,omitempty"` // ISO 3166-1 alpha-2 filter, e.g. "HK,JP" (AUTO) + Strategy string `protobuf:"bytes,7,opt,name=strategy,proto3" json:"strategy,omitempty"` // adaptive|url-test|round-robin|random (AUTO) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RoutingConfig) Reset() { + *x = RoutingConfig{} + mi := &file_aop_traffic_protocol_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RoutingConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RoutingConfig) ProtoMessage() {} + +func (x *RoutingConfig) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RoutingConfig.ProtoReflect.Descriptor instead. +func (*RoutingConfig) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{0} +} + +func (x *RoutingConfig) GetMode() RoutingMode { + if x != nil { + return x.Mode + } + return RoutingMode_ROUTING_MODE_UNSPECIFIED +} + +func (x *RoutingConfig) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *RoutingConfig) GetSelector() string { + if x != nil { + return x.Selector + } + return "" +} + +func (x *RoutingConfig) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *RoutingConfig) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *RoutingConfig) GetCountry() string { + if x != nil { + return x.Country + } + return "" +} + +func (x *RoutingConfig) GetStrategy() string { + if x != nil { + return x.Strategy + } + return "" +} + +// FlowFilter bounds which flows are recorded (CaptureConfig) or returned (Query). +type FlowFilter struct { + state protoimpl.MessageState `protogen:"open.v1"` + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` // host substring + Status string `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` // status class or code, e.g. "2xx", "404", "5xx" + Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"` // Content-Type substring + Last uint32 `protobuf:"varint,4,opt,name=last,proto3" json:"last,omitempty"` // return only the last N flows (Query) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FlowFilter) Reset() { + *x = FlowFilter{} + mi := &file_aop_traffic_protocol_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FlowFilter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FlowFilter) ProtoMessage() {} + +func (x *FlowFilter) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FlowFilter.ProtoReflect.Descriptor instead. +func (*FlowFilter) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{1} +} + +func (x *FlowFilter) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +func (x *FlowFilter) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *FlowFilter) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *FlowFilter) GetLast() uint32 { + if x != nil { + return x.Last + } + return 0 +} + +// CaptureConfig sets the hub's capture behaviour. It flips the runtime record +// flag; the listener address never changes so in-flight children are unaffected. +type CaptureConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode CaptureMode `protobuf:"varint,1,opt,name=mode,proto3,enum=aop.traffic.CaptureMode" json:"mode,omitempty"` + DecryptHttps bool `protobuf:"varint,2,opt,name=decrypt_https,json=decryptHttps,proto3" json:"decrypt_https,omitempty"` // intercept CONNECT to MITM-decrypt HTTPS + Filter *FlowFilter `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"` // record only matching flows + Stream bool `protobuf:"varint,4,opt,name=stream,proto3" json:"stream,omitempty"` // push Flow messages as they are captured + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CaptureConfig) Reset() { + *x = CaptureConfig{} + mi := &file_aop_traffic_protocol_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CaptureConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CaptureConfig) ProtoMessage() {} + +func (x *CaptureConfig) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CaptureConfig.ProtoReflect.Descriptor instead. +func (*CaptureConfig) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{2} +} + +func (x *CaptureConfig) GetMode() CaptureMode { + if x != nil { + return x.Mode + } + return CaptureMode_CAPTURE_MODE_UNSPECIFIED +} + +func (x *CaptureConfig) GetDecryptHttps() bool { + if x != nil { + return x.DecryptHttps + } + return false +} + +func (x *CaptureConfig) GetFilter() *FlowFilter { + if x != nil { + return x.Filter + } + return nil +} + +func (x *CaptureConfig) GetStream() bool { + if x != nil { + return x.Stream + } + return false +} + +// Configure declares desired routing and/or capture state. An absent sub-message +// leaves that facet unchanged; the handler replies with the resulting State. +type Configure struct { + state protoimpl.MessageState `protogen:"open.v1"` + Routing *RoutingConfig `protobuf:"bytes,1,opt,name=routing,proto3" json:"routing,omitempty"` + Capture *CaptureConfig `protobuf:"bytes,2,opt,name=capture,proto3" json:"capture,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Configure) Reset() { + *x = Configure{} + mi := &file_aop_traffic_protocol_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Configure) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Configure) ProtoMessage() {} + +func (x *Configure) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Configure.ProtoReflect.Descriptor instead. +func (*Configure) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{3} +} + +func (x *Configure) GetRouting() *RoutingConfig { + if x != nil { + return x.Routing + } + return nil +} + +func (x *Configure) GetCapture() *CaptureConfig { + if x != nil { + return x.Capture + } + return nil +} + +// Query requests a snapshot: the current State and/or the recorded flows. +type Query struct { + state protoimpl.MessageState `protogen:"open.v1"` + State bool `protobuf:"varint,1,opt,name=state,proto3" json:"state,omitempty"` // request current State + Flows bool `protobuf:"varint,2,opt,name=flows,proto3" json:"flows,omitempty"` // request recorded flows (batched Flow replies) + Filter *FlowFilter `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"` // filter for flows = true + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Query) Reset() { + *x = Query{} + mi := &file_aop_traffic_protocol_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Query) ProtoMessage() {} + +func (x *Query) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Query.ProtoReflect.Descriptor instead. +func (*Query) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{4} +} + +func (x *Query) GetState() bool { + if x != nil { + return x.State + } + return false +} + +func (x *Query) GetFlows() bool { + if x != nil { + return x.Flows + } + return false +} + +func (x *Query) GetFilter() *FlowFilter { + if x != nil { + return x.Filter + } + return nil +} + +type RoutingState struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActiveNode string `protobuf:"bytes,1,opt,name=active_node,json=activeNode,proto3" json:"active_node,omitempty"` + EgressUrl string `protobuf:"bytes,2,opt,name=egress_url,json=egressUrl,proto3" json:"egress_url,omitempty"` + Auto bool `protobuf:"varint,3,opt,name=auto,proto3" json:"auto,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RoutingState) Reset() { + *x = RoutingState{} + mi := &file_aop_traffic_protocol_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RoutingState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RoutingState) ProtoMessage() {} + +func (x *RoutingState) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RoutingState.ProtoReflect.Descriptor instead. +func (*RoutingState) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{5} +} + +func (x *RoutingState) GetActiveNode() string { + if x != nil { + return x.ActiveNode + } + return "" +} + +func (x *RoutingState) GetEgressUrl() string { + if x != nil { + return x.EgressUrl + } + return "" +} + +func (x *RoutingState) GetAuto() bool { + if x != nil { + return x.Auto + } + return false +} + +type CaptureState struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode CaptureMode `protobuf:"varint,1,opt,name=mode,proto3,enum=aop.traffic.CaptureMode" json:"mode,omitempty"` + Capturing bool `protobuf:"varint,2,opt,name=capturing,proto3" json:"capturing,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CaptureState) Reset() { + *x = CaptureState{} + mi := &file_aop_traffic_protocol_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CaptureState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CaptureState) ProtoMessage() {} + +func (x *CaptureState) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CaptureState.ProtoReflect.Descriptor instead. +func (*CaptureState) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{6} +} + +func (x *CaptureState) GetMode() CaptureMode { + if x != nil { + return x.Mode + } + return CaptureMode_CAPTURE_MODE_UNSPECIFIED +} + +func (x *CaptureState) GetCapturing() bool { + if x != nil { + return x.Capturing + } + return false +} + +// State is the runner's reply to Configure/Query. +type State struct { + state protoimpl.MessageState `protogen:"open.v1"` + Routing *RoutingState `protobuf:"bytes,1,opt,name=routing,proto3" json:"routing,omitempty"` + Capture *CaptureState `protobuf:"bytes,2,opt,name=capture,proto3" json:"capture,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *State) Reset() { + *x = State{} + mi := &file_aop_traffic_protocol_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *State) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*State) ProtoMessage() {} + +func (x *State) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use State.ProtoReflect.Descriptor instead. +func (*State) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{7} +} + +func (x *State) GetRouting() *RoutingState { + if x != nil { + return x.Routing + } + return nil +} + +func (x *State) GetCapture() *CaptureState { + if x != nil { + return x.Capture + } + return nil +} + +func (x *State) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +type Header struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Header) Reset() { + *x = Header{} + mi := &file_aop_traffic_protocol_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Header) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Header) ProtoMessage() {} + +func (x *Header) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Header.ProtoReflect.Descriptor instead. +func (*Header) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{8} +} + +func (x *Header) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Header) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +// Flow is one captured request/response pair. Its fields mirror the consumer's +// http.exchange shape so a consumer can map it directly; tool_id is the AOP +// tool-call id whose egress produced this flow. +type Flow struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + ToolId string `protobuf:"bytes,2,opt,name=tool_id,json=toolId,proto3" json:"tool_id,omitempty"` + Method string `protobuf:"bytes,3,opt,name=method,proto3" json:"method,omitempty"` + Url string `protobuf:"bytes,4,opt,name=url,proto3" json:"url,omitempty"` + Protocol string `protobuf:"bytes,5,opt,name=protocol,proto3" json:"protocol,omitempty"` + StatusCode int32 `protobuf:"varint,6,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` + ReasonPhrase string `protobuf:"bytes,7,opt,name=reason_phrase,json=reasonPhrase,proto3" json:"reason_phrase,omitempty"` + RequestHeaders []*Header `protobuf:"bytes,8,rep,name=request_headers,json=requestHeaders,proto3" json:"request_headers,omitempty"` + ResponseHeaders []*Header `protobuf:"bytes,9,rep,name=response_headers,json=responseHeaders,proto3" json:"response_headers,omitempty"` + RequestBody []byte `protobuf:"bytes,10,opt,name=request_body,json=requestBody,proto3" json:"request_body,omitempty"` + ResponseBody []byte `protobuf:"bytes,11,opt,name=response_body,json=responseBody,proto3" json:"response_body,omitempty"` + Error string `protobuf:"bytes,12,opt,name=error,proto3" json:"error,omitempty"` + Complete bool `protobuf:"varint,13,opt,name=complete,proto3" json:"complete,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,14,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Flow) Reset() { + *x = Flow{} + mi := &file_aop_traffic_protocol_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Flow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Flow) ProtoMessage() {} + +func (x *Flow) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Flow.ProtoReflect.Descriptor instead. +func (*Flow) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{9} +} + +func (x *Flow) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Flow) GetToolId() string { + if x != nil { + return x.ToolId + } + return "" +} + +func (x *Flow) GetMethod() string { + if x != nil { + return x.Method + } + return "" +} + +func (x *Flow) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *Flow) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + +func (x *Flow) GetStatusCode() int32 { + if x != nil { + return x.StatusCode + } + return 0 +} + +func (x *Flow) GetReasonPhrase() string { + if x != nil { + return x.ReasonPhrase + } + return "" +} + +func (x *Flow) GetRequestHeaders() []*Header { + if x != nil { + return x.RequestHeaders + } + return nil +} + +func (x *Flow) GetResponseHeaders() []*Header { + if x != nil { + return x.ResponseHeaders + } + return nil +} + +func (x *Flow) GetRequestBody() []byte { + if x != nil { + return x.RequestBody + } + return nil +} + +func (x *Flow) GetResponseBody() []byte { + if x != nil { + return x.ResponseBody + } + return nil +} + +func (x *Flow) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *Flow) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + +func (x *Flow) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +type ProtocolMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Message: + // + // *ProtocolMessage_Configure + // *ProtocolMessage_Query + // *ProtocolMessage_State + // *ProtocolMessage_Flow + Message isProtocolMessage_Message `protobuf_oneof:"message"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProtocolMessage) Reset() { + *x = ProtocolMessage{} + mi := &file_aop_traffic_protocol_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProtocolMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProtocolMessage) ProtoMessage() {} + +func (x *ProtocolMessage) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProtocolMessage.ProtoReflect.Descriptor instead. +func (*ProtocolMessage) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{10} +} + +func (x *ProtocolMessage) GetMessage() isProtocolMessage_Message { + if x != nil { + return x.Message + } + return nil +} + +func (x *ProtocolMessage) GetConfigure() *Configure { + if x != nil { + if x, ok := x.Message.(*ProtocolMessage_Configure); ok { + return x.Configure + } + } + return nil +} + +func (x *ProtocolMessage) GetQuery() *Query { + if x != nil { + if x, ok := x.Message.(*ProtocolMessage_Query); ok { + return x.Query + } + } + return nil +} + +func (x *ProtocolMessage) GetState() *State { + if x != nil { + if x, ok := x.Message.(*ProtocolMessage_State); ok { + return x.State + } + } + return nil +} + +func (x *ProtocolMessage) GetFlow() *Flow { + if x != nil { + if x, ok := x.Message.(*ProtocolMessage_Flow); ok { + return x.Flow + } + } + return nil +} + +type isProtocolMessage_Message interface { + isProtocolMessage_Message() +} + +type ProtocolMessage_Configure struct { + Configure *Configure `protobuf:"bytes,10,opt,name=configure,proto3,oneof"` +} + +type ProtocolMessage_Query struct { + Query *Query `protobuf:"bytes,11,opt,name=query,proto3,oneof"` +} + +type ProtocolMessage_State struct { + State *State `protobuf:"bytes,12,opt,name=state,proto3,oneof"` +} + +type ProtocolMessage_Flow struct { + Flow *Flow `protobuf:"bytes,13,opt,name=flow,proto3,oneof"` +} + +func (*ProtocolMessage_Configure) isProtocolMessage_Message() {} + +func (*ProtocolMessage_Query) isProtocolMessage_Message() {} + +func (*ProtocolMessage_State) isProtocolMessage_Message() {} + +func (*ProtocolMessage_Flow) isProtocolMessage_Message() {} + +var File_aop_traffic_protocol_proto protoreflect.FileDescriptor + +const file_aop_traffic_protocol_proto_rawDesc = "" + + "\n" + + "\x1aaop/traffic/protocol.proto\x12\vaop.traffic\x1a\x1fgoogle/protobuf/timestamp.proto\"\xc9\x01\n" + + "\rRoutingConfig\x12,\n" + + "\x04mode\x18\x01 \x01(\x0e2\x18.aop.traffic.RoutingModeR\x04mode\x12\x10\n" + + "\x03url\x18\x02 \x01(\tR\x03url\x12\x1a\n" + + "\bselector\x18\x03 \x01(\tR\bselector\x12\x12\n" + + "\x04type\x18\x04 \x01(\tR\x04type\x12\x12\n" + + "\x04name\x18\x05 \x01(\tR\x04name\x12\x18\n" + + "\acountry\x18\x06 \x01(\tR\acountry\x12\x1a\n" + + "\bstrategy\x18\a \x01(\tR\bstrategy\"`\n" + + "\n" + + "FlowFilter\x12\x12\n" + + "\x04host\x18\x01 \x01(\tR\x04host\x12\x16\n" + + "\x06status\x18\x02 \x01(\tR\x06status\x12\x12\n" + + "\x04type\x18\x03 \x01(\tR\x04type\x12\x12\n" + + "\x04last\x18\x04 \x01(\rR\x04last\"\xab\x01\n" + + "\rCaptureConfig\x12,\n" + + "\x04mode\x18\x01 \x01(\x0e2\x18.aop.traffic.CaptureModeR\x04mode\x12#\n" + + "\rdecrypt_https\x18\x02 \x01(\bR\fdecryptHttps\x12/\n" + + "\x06filter\x18\x03 \x01(\v2\x17.aop.traffic.FlowFilterR\x06filter\x12\x16\n" + + "\x06stream\x18\x04 \x01(\bR\x06stream\"w\n" + + "\tConfigure\x124\n" + + "\arouting\x18\x01 \x01(\v2\x1a.aop.traffic.RoutingConfigR\arouting\x124\n" + + "\acapture\x18\x02 \x01(\v2\x1a.aop.traffic.CaptureConfigR\acapture\"d\n" + + "\x05Query\x12\x14\n" + + "\x05state\x18\x01 \x01(\bR\x05state\x12\x14\n" + + "\x05flows\x18\x02 \x01(\bR\x05flows\x12/\n" + + "\x06filter\x18\x03 \x01(\v2\x17.aop.traffic.FlowFilterR\x06filter\"b\n" + + "\fRoutingState\x12\x1f\n" + + "\vactive_node\x18\x01 \x01(\tR\n" + + "activeNode\x12\x1d\n" + + "\n" + + "egress_url\x18\x02 \x01(\tR\tegressUrl\x12\x12\n" + + "\x04auto\x18\x03 \x01(\bR\x04auto\"Z\n" + + "\fCaptureState\x12,\n" + + "\x04mode\x18\x01 \x01(\x0e2\x18.aop.traffic.CaptureModeR\x04mode\x12\x1c\n" + + "\tcapturing\x18\x02 \x01(\bR\tcapturing\"\x87\x01\n" + + "\x05State\x123\n" + + "\arouting\x18\x01 \x01(\v2\x19.aop.traffic.RoutingStateR\arouting\x123\n" + + "\acapture\x18\x02 \x01(\v2\x19.aop.traffic.CaptureStateR\acapture\x12\x14\n" + + "\x05error\x18\x03 \x01(\tR\x05error\"2\n" + + "\x06Header\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"\xed\x03\n" + + "\x04Flow\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x17\n" + + "\atool_id\x18\x02 \x01(\tR\x06toolId\x12\x16\n" + + "\x06method\x18\x03 \x01(\tR\x06method\x12\x10\n" + + "\x03url\x18\x04 \x01(\tR\x03url\x12\x1a\n" + + "\bprotocol\x18\x05 \x01(\tR\bprotocol\x12\x1f\n" + + "\vstatus_code\x18\x06 \x01(\x05R\n" + + "statusCode\x12#\n" + + "\rreason_phrase\x18\a \x01(\tR\freasonPhrase\x12<\n" + + "\x0frequest_headers\x18\b \x03(\v2\x13.aop.traffic.HeaderR\x0erequestHeaders\x12>\n" + + "\x10response_headers\x18\t \x03(\v2\x13.aop.traffic.HeaderR\x0fresponseHeaders\x12!\n" + + "\frequest_body\x18\n" + + " \x01(\fR\vrequestBody\x12#\n" + + "\rresponse_body\x18\v \x01(\fR\fresponseBody\x12\x14\n" + + "\x05error\x18\f \x01(\tR\x05error\x12\x1a\n" + + "\bcomplete\x18\r \x01(\bR\bcomplete\x128\n" + + "\ttimestamp\x18\x0e \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\"\xd5\x01\n" + + "\x0fProtocolMessage\x126\n" + + "\tconfigure\x18\n" + + " \x01(\v2\x16.aop.traffic.ConfigureH\x00R\tconfigure\x12*\n" + + "\x05query\x18\v \x01(\v2\x12.aop.traffic.QueryH\x00R\x05query\x12*\n" + + "\x05state\x18\f \x01(\v2\x12.aop.traffic.StateH\x00R\x05state\x12'\n" + + "\x04flow\x18\r \x01(\v2\x11.aop.traffic.FlowH\x00R\x04flowB\t\n" + + "\amessage*\\\n" + + "\vCaptureMode\x12\x1c\n" + + "\x18CAPTURE_MODE_UNSPECIFIED\x10\x00\x12\x16\n" + + "\x12CAPTURE_MODE_RELAY\x10\x01\x12\x17\n" + + "\x13CAPTURE_MODE_RECORD\x10\x02*\xc0\x01\n" + + "\vRoutingMode\x12\x1c\n" + + "\x18ROUTING_MODE_UNSPECIFIED\x10\x00\x12\x17\n" + + "\x13ROUTING_MODE_DIRECT\x10\x01\x12\x16\n" + + "\x12ROUTING_MODE_PROXY\x10\x02\x12\x1a\n" + + "\x16ROUTING_MODE_SUBSCRIBE\x10\x03\x12\x15\n" + + "\x11ROUTING_MODE_AUTO\x10\x04\x12\x17\n" + + "\x13ROUTING_MODE_SWITCH\x10\x05\x12\x16\n" + + "\x12ROUTING_MODE_CLEAR\x10\x06B5Z3github.com/chainreactors/aiscan/aop/traffic;trafficb\x06proto3" + +var ( + file_aop_traffic_protocol_proto_rawDescOnce sync.Once + file_aop_traffic_protocol_proto_rawDescData []byte +) + +func file_aop_traffic_protocol_proto_rawDescGZIP() []byte { + file_aop_traffic_protocol_proto_rawDescOnce.Do(func() { + file_aop_traffic_protocol_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_aop_traffic_protocol_proto_rawDesc), len(file_aop_traffic_protocol_proto_rawDesc))) + }) + return file_aop_traffic_protocol_proto_rawDescData +} + +var file_aop_traffic_protocol_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_aop_traffic_protocol_proto_msgTypes = make([]protoimpl.MessageInfo, 11) +var file_aop_traffic_protocol_proto_goTypes = []any{ + (CaptureMode)(0), // 0: aop.traffic.CaptureMode + (RoutingMode)(0), // 1: aop.traffic.RoutingMode + (*RoutingConfig)(nil), // 2: aop.traffic.RoutingConfig + (*FlowFilter)(nil), // 3: aop.traffic.FlowFilter + (*CaptureConfig)(nil), // 4: aop.traffic.CaptureConfig + (*Configure)(nil), // 5: aop.traffic.Configure + (*Query)(nil), // 6: aop.traffic.Query + (*RoutingState)(nil), // 7: aop.traffic.RoutingState + (*CaptureState)(nil), // 8: aop.traffic.CaptureState + (*State)(nil), // 9: aop.traffic.State + (*Header)(nil), // 10: aop.traffic.Header + (*Flow)(nil), // 11: aop.traffic.Flow + (*ProtocolMessage)(nil), // 12: aop.traffic.ProtocolMessage + (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp +} +var file_aop_traffic_protocol_proto_depIdxs = []int32{ + 1, // 0: aop.traffic.RoutingConfig.mode:type_name -> aop.traffic.RoutingMode + 0, // 1: aop.traffic.CaptureConfig.mode:type_name -> aop.traffic.CaptureMode + 3, // 2: aop.traffic.CaptureConfig.filter:type_name -> aop.traffic.FlowFilter + 2, // 3: aop.traffic.Configure.routing:type_name -> aop.traffic.RoutingConfig + 4, // 4: aop.traffic.Configure.capture:type_name -> aop.traffic.CaptureConfig + 3, // 5: aop.traffic.Query.filter:type_name -> aop.traffic.FlowFilter + 0, // 6: aop.traffic.CaptureState.mode:type_name -> aop.traffic.CaptureMode + 7, // 7: aop.traffic.State.routing:type_name -> aop.traffic.RoutingState + 8, // 8: aop.traffic.State.capture:type_name -> aop.traffic.CaptureState + 10, // 9: aop.traffic.Flow.request_headers:type_name -> aop.traffic.Header + 10, // 10: aop.traffic.Flow.response_headers:type_name -> aop.traffic.Header + 13, // 11: aop.traffic.Flow.timestamp:type_name -> google.protobuf.Timestamp + 5, // 12: aop.traffic.ProtocolMessage.configure:type_name -> aop.traffic.Configure + 6, // 13: aop.traffic.ProtocolMessage.query:type_name -> aop.traffic.Query + 9, // 14: aop.traffic.ProtocolMessage.state:type_name -> aop.traffic.State + 11, // 15: aop.traffic.ProtocolMessage.flow:type_name -> aop.traffic.Flow + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name +} + +func init() { file_aop_traffic_protocol_proto_init() } +func file_aop_traffic_protocol_proto_init() { + if File_aop_traffic_protocol_proto != nil { + return + } + file_aop_traffic_protocol_proto_msgTypes[10].OneofWrappers = []any{ + (*ProtocolMessage_Configure)(nil), + (*ProtocolMessage_Query)(nil), + (*ProtocolMessage_State)(nil), + (*ProtocolMessage_Flow)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_aop_traffic_protocol_proto_rawDesc), len(file_aop_traffic_protocol_proto_rawDesc)), + NumEnums: 2, + NumMessages: 11, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_aop_traffic_protocol_proto_goTypes, + DependencyIndexes: file_aop_traffic_protocol_proto_depIdxs, + EnumInfos: file_aop_traffic_protocol_proto_enumTypes, + MessageInfos: file_aop_traffic_protocol_proto_msgTypes, + }.Build() + File_aop_traffic_protocol_proto = out.File + file_aop_traffic_protocol_proto_goTypes = nil + file_aop_traffic_protocol_proto_depIdxs = nil +} diff --git a/aop/wire_test.go b/aop/wire_test.go index 5b050768..76679e5c 100644 --- a/aop/wire_test.go +++ b/aop/wire_test.go @@ -92,3 +92,38 @@ func TestSessionNodeIDUsesStableFieldThree(t *testing.T) { t.Fatalf("node_id = %q", session.GetNodeId()) } } + +func FuzzEnvelopeBinaryRoundTrip(f *testing.F) { + wrapped, err := aop.Wrap("seed", "", &aop.Session{NodeId: "local-1"}) + if err != nil { + f.Fatal(err) + } + valid, err := proto.Marshal(wrapped) + if err != nil { + f.Fatal(err) + } + f.Add(valid) + f.Add([]byte{}) + f.Add([]byte{0x0a, 0x01, 'x'}) + + f.Fuzz(func(t *testing.T, data []byte) { + envelope := new(aop.Envelope) + if err := proto.Unmarshal(data, envelope); err != nil { + return + } + encoded, err := proto.MarshalOptions{Deterministic: true}.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + roundTrip := new(aop.Envelope) + if err := proto.Unmarshal(encoded, roundTrip); err != nil { + t.Fatal(err) + } + if !proto.Equal(envelope, roundTrip) { + t.Fatal("protobuf binary round trip changed the envelope") + } + if envelope.Payload != nil { + _, _ = aop.Unwrap(envelope) + } + }) +} diff --git a/archtest/architecture.go b/archtest/architecture.go deleted file mode 100644 index 592ca61b..00000000 --- a/archtest/architecture.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package archtest contains repository-wide architecture and quality guards. -package archtest diff --git a/build.sh b/build.sh index 67cf71e0..27a4ed06 100755 --- a/build.sh +++ b/build.sh @@ -106,7 +106,7 @@ aiscan 构建脚本 --output DIR 输出目录 (默认: dist) --embed 嵌入扫描资源（不加 emptytemplates/noembed tag） --ioa (已废弃, ioa serve 已集成到 aiscan 主二进制) - --profile PROFILE 构建配置: mini (默认, ~77MB), full (~123MB) + --profile PROFILE 构建配置: mini (默认), full LLM 覆盖（优先级高于 aiscan.yaml）: --llm-provider TYPE openai (OpenAI-compatible) or anthropic @@ -251,7 +251,7 @@ CGO_MODE=0 case "$PROFILE" in mini) ;; full) - EXTRA_TAGS="full,record_ffmpeg,re2_cgo,re2_static${EXTRA_TAGS:+,$EXTRA_TAGS}" + EXTRA_TAGS="full,re2_cgo,re2_static${EXTRA_TAGS:+,$EXTRA_TAGS}" BUILD_IOA=true AISCAN_BIN="aiscan-full" CGO_MODE=1 @@ -295,21 +295,6 @@ echo "cgo: $CGO_MODE" echo "output: $OUTPUT_DIR" echo "" -if [ "$PROFILE" = "full" ]; then - case "$HOST_OS" in - linux|windows) - if [ "${AISCAN_RECORD_BUILD_FROM_SOURCE:-0}" = "1" ]; then - bash ".github/native/sdk.sh" build "$HOST_OS" "$HOST_ARCH" - else - bash ".github/native/sdk.sh" fetch "$HOST_OS" "$HOST_ARCH" - fi - while IFS= read -r assignment; do - export "$assignment" - done < <(bash ".github/native/sdk.sh" env "$HOST_OS" "$HOST_ARCH") - ;; - esac -fi - # ─── 编译 ──────────────────────────────────────────────────────── mkdir -p "$OUTPUT_DIR" diff --git a/cmd/aiscan/setup.go b/cmd/aiscan/setup.go deleted file mode 100644 index 0e497a97..00000000 --- a/cmd/aiscan/setup.go +++ /dev/null @@ -1,258 +0,0 @@ -package main - -import ( - "context" - "fmt" - "net/url" - "os" - "strings" - - "github.com/chainreactors/aiscan/agent" - aop "github.com/chainreactors/aiscan/aop" - "github.com/chainreactors/aiscan/core/capability" - cfg "github.com/chainreactors/aiscan/core/config" - "github.com/chainreactors/aiscan/core/pidlock" - "github.com/chainreactors/aiscan/core/resources" - "github.com/chainreactors/aiscan/core/telemetry" - "github.com/chainreactors/aiscan/pkg/commands" - "github.com/chainreactors/aiscan/pkg/runner" - "github.com/chainreactors/aiscan/pkg/tui" - "github.com/chainreactors/aiscan/skills" - "github.com/chainreactors/aiscan/tools/scan" - "github.com/chainreactors/aiscan/tools/scan/engine" - ioaclient "github.com/chainreactors/ioa/client" - "github.com/chainreactors/ioa/protocols" - ioaserver "github.com/chainreactors/ioa/server" -) - -func init() { - runner.ScannerInitFunc = scannerInit - runner.ScannerWithAgentFunc = scannerWithAgent - runner.IOAServeFunc = ioaServe - runner.IOAClientCommandFunc = ioaClientCommand -} - -// --------------------------------------------------------------------------- -// Scanner engine initialization -// --------------------------------------------------------------------------- - -func scannerInit(ctx context.Context, a *runner.App, rc runner.ApplicationConfig, logger telemetry.Logger) { - es := initEngines(ctx, rc.Scanner, logger) - a.Engines = es - registerScannerCommands(a.Commands, es, rc.Scanner, rc.Tools, - a.Provider, a.ProviderConfig, a.Skills, a.Events, logger) -} - -func initEngines(ctx context.Context, sc runner.ScannerConfig, logger telemetry.Logger) *engine.Set { - engineSet, err := engine.InitWithOptions(ctx, resources.Options{ - CyberhubURL: sc.CyberhubURL, - APIKey: sc.CyberhubKey, - Mode: sc.CyberhubMode, - Proxy: sc.Proxy, - }, logger) - if err != nil { - logger.Warnf("scanner engines init error=%q action=continue_without_scanners", err) - return nil - } - recon := engine.ReconOptions{ - FofaKey: sc.FofaKey, - HunterAPIKey: sc.HunterAPIKey, - IngressProxy: sc.ReconProxy, - Limit: sc.ReconLimit, - Credentials: sc.UncoverCredentials, - } - engineSet.SetupUncover(recon, logger) - return engineSet -} - -func registerScannerCommands(cmdReg *commands.CommandRegistry, engineSet *engine.Set, scanCfg runner.ScannerConfig, toolCfg runner.ToolConfig, llmProvider agent.Provider, providerConfig agent.ProviderConfig, skillStore *skills.Store, agentEvents aop.EventEmitter, logger telemetry.Logger) { - var scanOpts []scan.Option - if scanCfg.AIEnabled && llmProvider != nil { - scannerParent := agent.NewAgent(agent.Config{ - Provider: llmProvider, - Tools: cmdReg, - Model: providerConfig.Model, - MaxTokens: providerConfig.MaxTokens, - ContextWindow: providerConfig.ContextWindow, - Logger: logger, - Bus: agentEvents, - }) - scanOpts = append(scanOpts, scan.WithParent(scannerParent)) - scanOpts = append(scanOpts, scan.WithDeepBrowserFunc(func(ctx context.Context, targetURL string) (string, error) { - return runner.CollectDeepBrowserArtifacts(ctx, cmdReg, targetURL, logger) - })) - if skillStore != nil { - scanOpts = append(scanOpts, scan.WithSkillReader(func(name string) string { - content, ok, err := skillStore.ReadVirtual("aiscan://skills/scan/" + name + ".md") - if !ok || err != nil { - return "" - } - return content - })) - } - } - scanOpts = append(scanOpts, scan.WithLogger(logger)) - - workDir, _ := os.Getwd() - deps := &commands.Deps{ - WorkDir: workDir, - BashTimeout: toolCfg.BashTimeout, - SkillStore: skillStore, - ScannerProxy: scanCfg.Proxy, - Logger: logger, - TavilyKeys: toolCfg.TavilyKeys, - PlaywrightSession: toolCfg.PlaywrightSession, - Events: agentEvents, - } - commands.Provide(deps, scan.OptsKey, scanOpts) - if engineSet != nil { - commands.Provide(deps, engine.SetKey, engineSet) - commands.Provide(deps, resources.SetKey, engineSet.Resources) - } - commands.BuildPlan(capability.Select(capability.Options{Groups: []string{"scanner", "proxy", "ioa"}}), deps, cmdReg) - logger.Infof("%s", telemetry.StartupOK("scanner", strings.Join(cmdReg.GroupNames("scanner"), ","))) -} - -// --------------------------------------------------------------------------- -// Scanner with agent -// --------------------------------------------------------------------------- - -func scannerWithAgent(ctx context.Context, option *cfg.Option, application *runner.App, scannerArgs []string, logger telemetry.Logger) error { - if application.Provider == nil { - return fmt.Errorf("--ai requires a configured LLM provider") - } - - pidLock, err := pidlock.Acquire(pidlock.AgentPIDFilePath(), logger) - if err != nil { - return err - } - defer pidLock.Release() - - command := scannerArgs[0] - intent, err := resolveScannerIntent(option, application.Skills, command) - if err != nil { - return err - } - - rt, err := runner.NewAgentRuntime(ctx, option, logger, &runner.RuntimeConfig{ - ExistingApp: application, - PromptConfig: &runner.PromptConfig{ - Tools: application.Commands, - ScannerDocs: application.Commands.UsageDocs(), - Skills: application.Skills.Skills, - ScannerAgentMode: true, - ScannerName: command, - }, - }) - if err != nil { - return err - } - defer rt.Close() - - prompt := scan.FormatAgentTaskPrompt(scannerArgs, intent) - agentOutput := tui.NewStaticAgentOutput(option) - unsubscribe := rt.Subscribe(agentOutput.HandleEvent) - defer unsubscribe() - agentOutput.Start("scanner", strings.Join(scannerArgs, " ")) - session, err := rt.OpenSession(ctx, runner.SessionOptions{ID: "scanner"}) - if err != nil { - return err - } - run, err := session.Run(ctx, runner.RunInput{Content: []*aop.Content{aop.Text(prompt)}}) - if err != nil { - return err - } - result, err := run.Wait() - if strings.TrimSpace(result.Output) != "" { - agentOutput.Final(result.Output) - } - _ = rt.CloseSession(context.Background(), "scanner", runner.SessionCloseCompleted) - return err -} - -func resolveScannerIntent(option *cfg.Option, store *skills.Store, command string) (string, error) { - var sections []string - if conceptURI := scan.ScannerConceptURI(command); conceptURI != "" && cfg.ScannerCommandAvailable(command) { - if body, ok, err := store.ReadVirtualBody(conceptURI); err == nil && ok && body != "" { - sections = append(sections, skills.FormatVirtualInvocation(command, conceptURI, body)) - } - } - - intent, err := cfg.ResolvePrompt(option.Prompt) - if err != nil { - return "", err - } - if intent == "" && option.TaskFile != "" { - data, err := os.ReadFile(option.TaskFile) - if err != nil { - return "", fmt.Errorf("read task file: %w", err) - } - intent = strings.TrimSpace(string(data)) - } - if intent == "" { - intent = "Process the scanner output according to the user's intent. If no specific intent is provided, briefly explain the important evidence in the output." - } - intent, err = cfg.ApplySelectedSkills(intent, scan.FilterAutoSkill(option.Skills, command), store) - if err != nil { - return "", err - } - sections = append(sections, intent) - return strings.Join(sections, "\n\n"), nil -} - -// --------------------------------------------------------------------------- -// IOA -// --------------------------------------------------------------------------- - -func ioaServe(ctx context.Context, option *cfg.Option, logger telemetry.Logger) error { - store := ioaserver.NewMemoryStore() - logger.Importantf("aiscan server store=memory") - defer func() { _ = store.Close() }() - - accessKey := option.IOAToken - if accessKey == "" { - accessKey = protocols.NewToken() - } - listenURL := option.IOAURL - if listenURL == "" { - listenURL = "http://127.0.0.1:8765" - } - if u, err := url.Parse(listenURL); err == nil { - logger.Infof(" agent IOA connect: aiscan agent --transport local --ioa-url http://%s@%s", accessKey, u.Host) - } - - return ioaserver.RunServer(ctx, ioaserver.ServerOptions{ - URL: listenURL, - AccessKey: accessKey, - Store: store, - }) -} - -func ioaClientCommand(ctx context.Context, mode cfg.RunMode, option *cfg.Option, args cfg.IOAClientArgs, logger telemetry.Logger) error { - ioaURL := option.IOAURL - if ioaURL == "" { - ioaURL = "http://127.0.0.1:8765" - } - client, err := ioaclient.NewClient(ioaURL, "") - if err != nil { - return fmt.Errorf("connect to server: %w", err) - } - if client.AccessKey() != "" { - if err := client.EnsureRegistered(ctx, "aiscan-cli", "", nil); err != nil { - return fmt.Errorf("server auth register: %w", err) - } - } - - switch mode { - case cfg.RunModeIOASpaces: - return tui.RunIOASpaces(ctx, client, option, os.Stdout, os.Stderr) - case cfg.RunModeIOAMessages: - return tui.RunIOAMessages(ctx, client, option, args, os.Stdout, os.Stderr) - case cfg.RunModeIOAContext: - return tui.RunIOAContext(ctx, client, option, args, os.Stdout, os.Stderr) - case cfg.RunModeIOANodes: - return tui.RunIOANodes(ctx, client, option, args, os.Stdout, os.Stderr) - default: - return fmt.Errorf("unknown server mode: %s", mode) - } -} diff --git a/cmd/gen/main.go b/cmd/gen/main.go index d44b7bb2..a72ba981 100644 --- a/cmd/gen/main.go +++ b/cmd/gen/main.go @@ -32,6 +32,7 @@ var aopProtos = []string{ "aop/pty/protocol.proto", "aop/tool/protocol.proto", "aop/sco/protocol.proto", + "aop/traffic/protocol.proto", } var typeProtos = []string{ @@ -248,9 +249,12 @@ func findGoTool(root, envName, name string) (string, error) { } cmd := exec.Command("go", "tool", "-n", name) cmd.Dir = root - output, err := cmd.CombinedOutput() + output, err := cmd.Output() if err != nil { - return "", fmt.Errorf("go tool -n %s: %w: %s", name, err, strings.TrimSpace(string(output))) + if exitErr, ok := err.(*exec.ExitError); ok { + return "", fmt.Errorf("go tool -n %s: %w: %s", name, err, strings.TrimSpace(string(exitErr.Stderr))) + } + return "", fmt.Errorf("go tool -n %s: %w", name, err) } path := strings.Trim(strings.TrimSpace(string(output)), `"`) if path == "" { diff --git a/cmd/runner/main.go b/cmd/runner/main.go new file mode 100644 index 00000000..948fed10 --- /dev/null +++ b/cmd/runner/main.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + + aop "github.com/chainreactors/aiscan/aop" + cfg "github.com/chainreactors/aiscan/core/config" + "github.com/chainreactors/aiscan/core/telemetry" + node "github.com/chainreactors/aiscan/pkg/node" + "github.com/chainreactors/aiscan/pkg/runner" + _ "github.com/chainreactors/aiscan/tools" + _ "github.com/chainreactors/aiscan/tools/arsenal" + _ "github.com/chainreactors/aiscan/tools/gogo" + _ "github.com/chainreactors/aiscan/tools/ioa" + _ "github.com/chainreactors/aiscan/tools/neutron" + _ "github.com/chainreactors/aiscan/tools/proton" + _ "github.com/chainreactors/aiscan/tools/proxy" + _ "github.com/chainreactors/aiscan/tools/search" + _ "github.com/chainreactors/aiscan/tools/spray" + _ "github.com/chainreactors/aiscan/tools/zombie" +) + +type options struct { + server string + token string + id string + websocket string + configFile string + jsonFrames bool + version bool +} + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if err := run(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { + if errors.Is(err, flag.ErrHelp) { + return + } + fmt.Fprintln(os.Stderr, "runner:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { + options, err := parseOptions(args, stderr) + if err != nil { + return err + } + if options.version { + fmt.Fprintf(stdout, "runner v%s\n", cfg.Version) + return nil + } + option := new(cfg.Option) + option.ConfigFile = options.configFile + if _, err := runner.ResolveRuntimeConfig(option); err != nil { + return fmt.Errorf("load config: %w", err) + } + logger := telemetry.GlobalLogger(telemetry.LogConfig{ + Debug: option.Debug, Quiet: option.Quiet, Output: stderr, Color: !option.NoColor, + }) + application, err := newApplication(ctx, option, logger) + if err != nil { + return err + } + defer application.Close() + if err := application.WaitEngines(ctx); err != nil { + return err + } + logger.Infof("runner tools ready: %s", strings.Join(application.Commands.Names(), ", ")) + return node.RunToolNode(ctx, node.ToolNodeConfig{ + ServerURL: options.server, + WSPath: options.websocket, + ID: options.id, + Token: options.token, + Registry: application.Commands, + Events: application.EventBus, + Progress: application.Progress, + Logger: logger, + Version: cfg.Version, + JSONFrames: options.jsonFrames, + FileAudit: application.FileAudit, + ExtraNamespaces: []func(*aop.NamespaceMux) error{ + application.RegisterTrafficNamespace, + }, + }) +} + +func parseOptions(args []string, stderr io.Writer) (options, error) { + var result options + flags := flag.NewFlagSet("runner", flag.ContinueOnError) + flags.SetOutput(stderr) + flags.StringVar(&result.server, "server", "", "AOP server URL") + flags.StringVar(&result.token, "token", "", "server access token") + flags.StringVar(&result.id, "id", "", "stable runner ID (defaults to hostname)") + flags.StringVar(&result.websocket, "ws-path", node.DefaultWSPath, "AOP WebSocket path") + flags.StringVar(&result.configFile, "config", "", "path to aiscan.yaml") + flags.BoolVar(&result.jsonFrames, "json", false, "use ProtoJSON WebSocket frames") + flags.BoolVar(&result.version, "version", false, "print version") + if err := flags.Parse(args); err != nil { + return result, err + } + if strings.TrimSpace(result.server) == "" && !result.version { + return result, fmt.Errorf("--server is required") + } + return result, nil +} + +func newApplication(ctx context.Context, option *cfg.Option, logger telemetry.Logger) (*runner.App, error) { + config := runner.AppConfig(option, runner.RuntimeFeatures{ToolsEnabled: true}, logger) + config.Tools.RunnerMode = true + return runner.NewApp(ctx, config) +} diff --git a/cmd/runner/main_test.go b/cmd/runner/main_test.go new file mode 100644 index 00000000..911bfc3a --- /dev/null +++ b/cmd/runner/main_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + cfg "github.com/chainreactors/aiscan/core/config" + "github.com/chainreactors/aiscan/core/telemetry" +) + +func TestParseOptionsRequiresServer(t *testing.T) { + if _, err := parseOptions(nil, io.Discard); err == nil { + t.Fatal("missing server must be rejected") + } + options, err := parseOptions([]string{"--server", "http://127.0.0.1:8080"}, io.Discard) + if err != nil { + t.Fatal(err) + } + if options.server != "http://127.0.0.1:8080" { + t.Fatalf("server = %q", options.server) + } +} + +func TestRunPrintsVersionWithoutServer(t *testing.T) { + var stdout bytes.Buffer + if err := run(context.Background(), []string{"--version"}, &stdout, io.Discard); err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(stdout.String()), "runner v"+cfg.Version; got != want { + t.Fatalf("version = %q, want %q", got, want) + } +} + +func TestNewApplicationRegistersRunnerTools(t *testing.T) { + application, err := newApplication(context.Background(), new(cfg.Option), telemetry.NopLogger()) + if err != nil { + t.Fatal(err) + } + defer application.Close() + if err := application.WaitEngines(context.Background()); err != nil { + t.Fatal(err) + } + for _, name := range []string{"bash", "ls"} { + if _, ok := application.Commands.GetTool(name); !ok { + t.Fatalf("runner tool %q is not registered", name) + } + } +} diff --git a/core/config/options.go b/core/config/options.go index c414b632..9787b91d 100644 --- a/core/config/options.go +++ b/core/config/options.go @@ -70,6 +70,7 @@ type ScannerOptions struct { CyberhubKey string `long:"cyberhub-key" config:"key" description:"Cyberhub API key"` CyberhubMode string `long:"cyberhub-mode" config:"mode" description:"Cyberhub resource mode: merge or override"` Proxy string `long:"proxy" config:"proxy" description:"Proxy for scanner tools. Supports socks5://, trojan://, vless://, clash:// (subscription with load balancing)"` + Mitm *bool `long:"mitm" config:"mitm" description:"Record tool traffic through the MITM hub (default: enabled). Disable for pure proxy routing without interception/capture"` } type AgentOptions struct { diff --git a/docs/agent.md b/docs/agent.md index a474a232..3ccea2cb 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -351,9 +351,9 @@ playwright sessions # 列出活跃会话 `--record` 选项开启操作录制，可用于生成自动化测试模板。 -### record — 桌面/窗口截图与录屏（Windows、Linux X11 full 版） +### record — 桌面/窗口截图与录屏（Windows、Linux X11 可选工具） -`record` 是原生 Agent Tool，而不是 bash 伪命令。它支持桌面或指定窗口截图、固定时长录制，以及异步 `start` / `stop` / `status` 会话。窗口目标可以传 Windows HWND、X11 Window ID，或使用 PID 自动解析面积最大的可见主窗口。 +`record` 是面向 SDK 和工具开发者的可选原生 Agent Tool，不包含在默认 full 构建中。它支持桌面或指定窗口截图、固定时长录制，以及异步 `start` / `stop` / `status` 会话。窗口目标可以传 Windows HWND、X11 Window ID，或使用 PID 自动解析面积最大的可见主窗口。 默认输出 PNG 截图和 H.264/MP4 视频，不录制音频；Wayland、最小化窗口和不可见后台窗口不受支持。详细参数见 [record 文档](record.md)。 diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..94528410 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,244 @@ +# AIScan Agent 架构 + +AIScan 的核心是一个可嵌入、可扩展、可远程驱动的 Agent 内核。本文用 **PiAgent** 指代 `agent/` 中的 Agent kernel；代码中的公开类型仍为 `agent.Agent`。 + +PiAgent 的职责是：**维护上下文，调用 LLM，根据模型决策执行工具，并将执行过程输出为统一事件。** + +它不直接处理 Web、CLI 或多会话管理。外部任务由 Agent Host 写入 Inbox，PiAgent 从 Inbox 取出消息并执行，从而让所有入口复用同一个 Agent 内核。Agent Host 在代码中对应 `AgentRuntime`。 + +## 1. 总体架构 + +```mermaid +flowchart TB + subgraph ENTRY[外部入口] + CLI[CLI / REPL] + WEB[Web / Application Client] + STDIO[stdio Controller] + IOA[IOA Peer] + EMBED[Go Embedder] + end + + subgraph RUNTIME[Agent Host] + AOP[AOP Protocol] + RT[Session + Control] + INBOX[Session Inbox] + end + + subgraph PI[PiAgent Kernel] + LOOP[Agent Loop] + CONTEXT[Context / Transcript] + POLICY[Hooks / Budget / Compaction] + EXEC[Tool Executor] + end + + subgraph CAP[能力层] + LLM[LLM Provider] + TOOLS[Tools / Scanner Commands] + SKILLS[Skills] + SUB[Subagent] + end + + subgraph OUTPUT[输出层] + EVENTS[AOP EventBus] + VIEW[CLI / Web] + STORE[JSONL / Session Store] + end + + CLI --> RT + WEB --> AOP --> RT + STDIO --> AOP + IOA --> RT + EMBED --> RT + RT --> INBOX --> LOOP + + SKILLS --> RT + LOOP <--> CONTEXT + POLICY --> LOOP + LOOP <--> LLM + LOOP --> EXEC --> TOOLS + EXEC --> SUB + SUB --> INBOX + + LOOP --> EVENTS + EVENTS --> VIEW + EVENTS --> STORE +``` + +架构分为四层： + +| 层 | 核心职责 | +| --- | --- | +| 入口与协议 | 接收外部消息和生命周期控制，传递实时事件 | +| Agent Host | 将消息写入 Inbox，管理 Session、取消和恢复 | +| PiAgent | 完成模型决策、上下文管理和工具调用循环 | +| 能力与输出 | 提供 LLM、工具、Skill，并消费统一事件流 | + +Scan 是与 Agent 平级的确定性执行路径。两者复用相同的工具和事件体系，但只有 Agent 经过 LLM 决策循环。 + +## 2. PiAgent 核心设计 + +PiAgent 是一个小而稳定的内核。它不理解具体扫描器，也不绑定某个模型供应商，只依赖 Provider 和 Tool Executor 两个能力边界。 + +```mermaid +flowchart LR + INPUT[External Message] --> INBOX[Session Inbox] + INBOX --> CTX[Build Context] + CTX --> COMPACT[Transform / Compact] + COMPACT --> LLM[LLM Provider] + LLM --> DECISION{Model Decision} + + DECISION -->|tool calls| GUARD[Policy Hooks] + GUARD --> EXEC[Tool Executor] + EXEC --> RESULT[Tool Results] + RESULT --> CTX + + DECISION -->|final answer| DONE[Final Result] + + CTX --> EVENT[AOP Events] + LLM --> EVENT + EXEC --> EVENT + DONE --> EVENT +``` + +一次任务执行的关键过程是： + +1. 外部消息先写入 Session Inbox。 +2. PiAgent 从 Inbox 取出消息，将其合并到 Transcript 并构造本轮上下文。 +3. 调用 Provider，并接收模型文本或 tool call。 +4. 工具执行前经过策略检查，再由统一 Executor 调用工具。 +5. 工具结果写回上下文并继续决策，直到完成、达到预算或被取消。 + +这套循环保持三个关键约束： + +- **上下文一致**：每轮 Provider 请求使用稳定快照，异步结果只在轮次边界进入。 +- **副作用受控**：LLM 不能直接访问 shell、网络或 scanner，所有能力必须经过 Tool Executor。 +- **过程可观察**：message、tool、usage、error 和生命周期统一输出为 AOP Event。 + +### Context 与状态 + +`agent.Agent` 保存跨任务的消息历史，因此同一个 Agent 可以连续对话。每次执行开始时会取得 Config 快照，正在执行的任务不会受到中途切换 Provider 或配置的影响。 + +上下文接近模型窗口时会自动压缩；工具输出也会在送回模型前限制大小，避免一次扫描结果耗尽整个上下文。 + +### Tool 与 Skill + +- **Tool** 是 LLM 可以实际调用的能力，例如文件、搜索、shell、scanner 和 subagent。 +- **Command** 是 AIScan 的可执行命令，由 CommandRegistry 管理，并可以通过工具入口复用。 +- **Skill** 是提供给 Agent 的知识和工作方式，不直接执行代码。 + +PiAgent 只看到工具定义和工具结果，不在内核中按工具名称编写业务分支。新增能力应通过注册 Tool、Command 或 Skill 完成。 + +## 3. Session 与子 Agent + +Agent Host 负责管理多个 Session，并将不同入口的消息送入对应 Inbox。 + +```mermaid +flowchart TB + RT[Agent Host
shared capabilities and events] + + subgraph S1[Session A] + I1[Inbox] + A1[PiAgent
private context] + I1 --> A1 + end + + subgraph S2[Session B] + I2[Inbox] + A2[PiAgent
private context] + I2 --> A2 + end + + RT --> S1 + RT --> S2 + + A1 -->|delegate| CHILD[Subagent
fresh or forked context] + CHILD -->|completion| I1 + + SHARED[Shared Provider / Tools / Hooks] + SHARED --> A1 + SHARED --> A2 + SHARED --> CHILD +``` + +每个 Session 拥有独立的 Inbox、PiAgent 和上下文。Provider、Tools、Hooks、Skills 和 EventBus 由 Runtime/App 共享。 + +子 agent 从父 Agent 派生，共享基础能力但拥有独立状态。它可以同步返回，也可以在后台执行并通过父 Inbox 回传结果。父子关系会进入 AOP 事件，外部可以还原完整的 Agent 调用树。 + +Inbox 是所有动态消息进入 Agent 的统一入口，典型来源包括用户任务、follow-up、IOA peer、后台工具、定时任务和子 agent。它避免外部生产者直接调用 PiAgent 或修改 Transcript。 + +## 4. 外部如何介入 + +外部介入分为两类： + +- **跨进程消息与控制**：通过 AOP、stdio 或 IOA 提交消息或操作 Session 生命周期。 +- **进程内扩展**：通过 Provider、Tool、Skill、Hook 和 EventBus 扩展 PiAgent。 + +```mermaid +flowchart LR + CLIENT[Remote Client] -->|OpenSession / RunTurn / CancelTurn| AOP[AOP Protocol] + AOP --> RT[Agent Host
session + lifecycle control] + + PEER[IOA / Async Producer] -->|message| RT + HOST[Local Host / CLI] -->|message| RT + RT -->|normalized message| INBOX[Session Inbox] + INBOX --> PI[PiAgent] + + EXT[In-process Extension] -->|assembly-time configuration| PI + + PI -->|AOP Events| OBS[UI / Recorder / Store] +``` + +| 介入目标 | 正式入口 | 作用 | +| --- | --- | --- | +| 发起或继续任务 | `RunTurn` / `RunInput` | 转换为 Inbox 消息并唤醒 Agent | +| 追加异步信息 | Inbox / IOA | 写入 Inbox，在轮次边界加入上下文 | +| 停止任务 | `CancelTurn` / context cancel | 取消当前执行 | +| 调整模型行为 | Provider / Config / Skill | 改变模型、提示和知识 | +| 扩展执行能力 | Tool / Command registration | 增加 Agent 可调用能力 | +| 约束执行策略 | Hook Registry | 改写上下文、审批工具、处理结果 | +| 观察运行过程 | EventBus / WatchEvents | 消费事件，不直接修改执行 | + +### 跨进程入口 + +AOP 是远程接入的正式协议边界。`RunTurn` 中的输入经 Runtime 写入 Session Inbox，再由 PiAgent 消费；Web Hub 可以把请求路由到本地或远程 Agent Node，但最终遵守相同的入口顺序。 + +`OpenSession`、`CancelTurn` 和 `CloseSession` 属于生命周期控制，由 Runtime 直接处理，不进入对话 Inbox。消息面与控制面保持分离。 + +`RunTurnResponse` 只是任务已接收的回执，实际回答和工具过程通过事件流返回，`turn_ended` 是一轮结束的稳定信号。 + +IOA 不直接操作 Agent 内存，而是将 peer 消息写入 Session Inbox。这样外部协作与本地异步任务使用同一套消息语义。 + +### 进程内扩展 + +Hook 是 PiAgent 的策略扩展点，关键阶段包括： + +- 任务开始时调整 system prompt； +- Provider 调用前过滤或补充上下文； +- Tool 执行前审批或阻断； +- Tool 执行后改写、脱敏或终止； +- 任务和 Session 结束时进行审计与清理。 + +工具审批采用 fail-closed：策略 handler 失败时不会放行工具。EventBus 则只负责观察，不能替代执行前的 Hook。简单说，**Hook 控制未来动作，Inbox 增加新信息，Event 记录已经发生的事实。** + +## 5. 关键设计原则 + +1. **一个 Agent 内核**：CLI、Web、stdio、IOA 和 `--ai` 复用同一套 Agent Host/PiAgent。 +2. **Session 隔离**：上下文和 Inbox 属于 Session，Provider、Tools、Hooks 和 EventBus 可以共享。 +3. **控制与观察分离**：Hook、Inbox、Cancel 可以改变执行；EventBus 只描述执行事实。 +4. **能力通过工具扩展**：LLM 的所有副作用都经过 Tool Executor 和策略检查。 +5. **异步结果通过 Inbox 回流**：后台任务和子 agent 不直接修改上下文。 +6. **跨进程统一使用 AOP**：Application、Node 和 stdio 共享 protobuf Envelope 和事件语义。 + +## 6. 代码导航 + +| 关注点 | 实现位置 | +| --- | --- | +| PiAgent API 与状态 | `agent/agent.go`、`agent/types.go` | +| 核心循环 | `agent/loop.go` | +| Inbox、Hooks、Subagent | `agent/inbox/`、`agent/hooks/`、`agent/subagent.go` | +| Runtime 与 Session | `pkg/runner/runner.go`、`pkg/runner/runtime_session.go` | +| Tool 与 Command | `core/tool/`、`pkg/commands/` | +| AOP Runtime 接口 | `pkg/runner/runtime_protocol.go` | +| Web 与远程 Node | `pkg/web/`、`pkg/node/` | +| 协议设计 | [protocol-architecture.md](protocol-architecture.md) | +| 第三方接入 | [integration.md](integration.md) | diff --git a/docs/changelog.md b/docs/changelog.md index 9189ca14..dd516aa7 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,29 @@ # Changelog +## v1.0.0-rc2 — 流量与文件审计 + 单版本 Runner + 发布门禁 + +v1.0.0-rc2 聚焦远程 Runner 的原生运行时组装、可审计的流量与文件访问，以及发布链路的可重复验证。Runner 现在只有一个无 build tag 的实现和发行 profile，CI 与发布 wrapper 共用只读的 release-build workflow 完成构建、打包和 smoke test。 + +### New Features + +- AOP 新增常驻流量捕获 namespace，proxy 共享统一 capture hub，并能按任务查询捕获结果。 +- Runner 文件访问进入 task-scoped audit trail，并通过 AOP namespace 对控制面提供结构化记录。 +- 官方 Release 新增 Linux、macOS、Windows amd64/arm64 的单一 `runner` 产物。 + +### Improvements + +- Runner 与 AIScan 共用 `pkg/runner.App` 的原生组装路径，删除重复 setup 与全局 hook 状态。 +- `make runner` 直接、无 build tag 地构建 `./cmd/runner`；`make all` 自动包含 runner。 +- 原生 record 工具改为显式 opt-in，默认 full 与官方 Release 不再隐式下载或链接 recorder SDK。 +- macOS full 产物由 Linux 上的 Zig 与固定 SDK 交叉编译，工具链版本和校验和固定。 +- CI 恢复 `go vet`，预编译 integration-tag 回归，并以只读 release-build 验证正式平台矩阵。 +- Release notes 优先读取本文件中的对应版本章节，回退日志也会正确以上一个 prerelease 为基线。 + +### Bug Fixes + +- 修复 scanner-regression 因 integration test 遗留未使用 import 而持续无法编译的问题。 +- 合入 rc1 后的 Node WebSocket 稳定性、尾随 artifact 丢弃、UTF-8 边界清洗和 scanner artifact 体积预算修复。 + ## v1.0.0-rc1 — 原生录屏 + 浏览器自动化扩展 + 稳定接口候选 v1.0.0-rc1 是 AIScan 首个 v1 发布候选版本。它在 v0.4.0 Web 工作台、Agent 会话和 SCO 资产模型之上补齐原生桌面录制、可复用浏览器自动化、scanner-native Artifact/Loot 传输和跨平台 shell 命令组合，同时把 CLI、配置、AOP/Connect 协议、包边界与 standard/full 发布矩阵收敛为 v1 稳定基线。 @@ -8,13 +32,13 @@ v1.0.0-rc1 是 AIScan 首个 v1 发布候选版本。它在 v0.4.0 Web 工作台 **record — 原生桌面与窗口捕获** -Full 版新增原生 `record` Agent Tool，用于截取桌面或可见应用窗口，并生成 PNG 截图或 H.264/MP4 视频。它不依赖外部 ffmpeg 命令；官方 Windows amd64 与 Linux amd64/arm64 full 产物静态链接裁剪后的 FFmpeg/libx264 SDK。 +新增可选的原生 `record` Agent Tool，用于截取桌面或可见应用窗口，并生成 PNG 截图或 H.264/MP4 视频。它不依赖外部 ffmpeg 命令；SDK 和工具开发者可在 Windows amd64 与 Linux amd64/arm64 上显式链接裁剪后的 FFmpeg/libx264 SDK，官方 full 产物默认不编译该工具。 - 支持 `screenshot`、固定时长 `record`，以及异步 `start` / `stop` / `status` - 支持桌面、Windows HWND、X11 Window ID，或通过 PID 自动选择最大的可见窗口 - 默认捕获鼠标，视频使用 H.264/libx264 编码并封装为 MP4；最多可并行运行四个录制会话 - 截图通过 AOP media 返回有界预览；视频通过 task-relative `Resource.uri` 与分段 `aop.file` 请求传输 -- `make full` 自动下载、校验并缓存固定版本的 recorder SDK；维护者也可从固定源码重建 SDK +- `make record` 按需下载、校验并缓存固定版本的 recorder SDK，并构建独立的 record-enabled 产物；维护者也可从固定源码重建 SDK Wayland、macOS、Windows arm64、无图形会话的 headless 主机和 Windows session 0 暂不支持原生录制。完整限制与构建说明见 [record 文档](record.md)。 @@ -53,10 +77,10 @@ Unix 使用本地 socket，Windows 使用 named pipe；进程退出或异常中 **发布与原生构建链路** -- standard 发布 Linux、macOS、Windows 的 amd64/arm64；full 发布 Linux/macOS amd64/arm64 与 Windows amd64 -- CI、nightly 和正式 release 共用 `.github/workflows/go-release.yml`，构建标签、版本注入、压缩和平台矩阵不再漂移 -- recorder SDK 使用固定源码、组件 allowlist、SHA-256 和静态库体积预算；缺少预构建 SDK 时 CI 可回退到源码构建 -- full profile 恢复静态 RE2，并验证 Windows recorder/RE2 原生库没有变成运行时 DLL 依赖 +- standard 由 Linux runner 交叉编译 Linux、macOS、Windows 的 amd64/arm64；full 的 macOS amd64/arm64 也通过 Linux 上的 Zig 和固定 SDK 交叉编译 +- CI、定时回归和正式 release 共用同一套构建标签与发布约束，版本注入、压缩和平台矩阵不再漂移 +- recorder SDK 使用固定源码、组件 allowlist、SHA-256 和静态库体积预算，并通过独立 workflow 构建发布 +- full profile 恢复静态 RE2，并验证 Windows RE2 原生库没有变成运行时 DLL 依赖 - Windows 发布包经 UPX 压缩后会在干净 runner 中解压并真实执行 `--version`，避免“能打包但无法启动” - 本地 standard/full release profile 默认使用 `-s -w`；Windows full 从约 200 MiB 恢复到约 124 MiB，且架构测试阻止调试段再次进入发布构建 @@ -91,8 +115,8 @@ Unix 使用本地 socket，Windows 使用 named pipe；进程退出或异常中 | 产物 | Linux | macOS | Windows | | --- | --- | --- | --- | | `aiscan` | amd64、arm64 | amd64、arm64 | amd64、arm64 | -| `aiscan-full` | amd64、arm64 | amd64、arm64 | amd64 | -| 原生 `record` | X11 amd64/arm64 | 不支持 | amd64 | +| `aiscan-full` | amd64、arm64 | amd64、arm64（Linux 交叉编译） | amd64 | +| 可选原生 `record` SDK 构建 | X11 amd64/arm64 | 不支持 | amd64 | 迁移细节、兼容承诺和发布门禁见 [v1.0.0 发布与迁移](v1.0.0.md)。 diff --git a/docs/record.md b/docs/record.md index 25390447..f0d2656f 100644 --- a/docs/record.md +++ b/docs/record.md @@ -1,6 +1,6 @@ # record — desktop and window capture -`record` is a native full-build tool that captures PNG screenshots and H.264/MP4 recordings from the desktop or a visible application window. +`record` is an optional native tool for SDK and tool developers. It captures PNG screenshots and H.264/MP4 recordings from the desktop or a visible application window. Default full builds do not compile or register it. | Platform | Support | | --- | --- | @@ -44,19 +44,19 @@ Limitations: - macOS and Windows arm64 do not have a native recorder backend. - Capture requires an interactive graphical session; headless hosts and Windows session 0 are not supported. - The window must be visible and non-minimized. Capture size is fixed when recording starts; closing, minimizing, or shrinking the window can terminate the recording. -- The native backend is present in official Windows amd64 and Linux amd64/arm64 full builds. Custom builds require CGO and a supported C toolchain. `make full` downloads the pinned, prebuilt FFmpeg/x264 SDK automatically. +- The native backend is not present in official full builds. Custom builds require CGO, the `record_ffmpeg` build tag, and a supported C toolchain. -The full build statically links a feature-minimal FFmpeg and x264, so users do not install either runtime separately. This is single-file distribution, not literally zero runtime dependencies: Windows still uses system DLLs; Linux requires glibc, X11/XCB libraries, and an accessible `DISPLAY`. The SDK only enables the platform capture input, its raw/BMP decoder, libx264, the MP4 muxer, file output, and pixel conversion. It is not a general-purpose FFmpeg build. +Record-enabled builds statically link a feature-minimal FFmpeg and x264, so users do not install either runtime separately. This is single-file distribution, not literally zero runtime dependencies: Windows still uses system DLLs; Linux requires glibc, X11/XCB libraries, and an accessible `DISPLAY`. The SDK only enables the platform capture input, its raw/BMP decoder, libx264, the MP4 muxer, file output, and pixel conversion. It is not a general-purpose FFmpeg build. ## Two-stage native build -Normal users should use an official `aiscan-full` archive; recording works without installing FFmpeg or x264. Developers building from source use: +Build the record-enabled edition with the dedicated target: ```bash -make full +make record ``` -The `record-native` prerequisite downloads a versioned SDK into `.cache/record-native/-`, verifies its SHA-256 sidecar and manifest, then links it into the full binary. Supported SDK targets are Linux amd64/arm64 and Windows amd64. Linux source builds still need a C compiler, `pkg-config`, and XCB development packages; Windows source builds need MinGW-w64 and `pkgconf`. +`make record` builds the frontend, downloads a versioned SDK into `.cache/record-native/-`, verifies its SHA-256 sidecar and manifest, applies the native link environment, and compiles `bin/aiscan-record` with the `full` and `record_ffmpeg` tags. Supported SDK targets are Linux amd64/arm64 and Windows amd64. Linux source builds still need a C compiler, `pkg-config`, and XCB development packages; Windows source builds need MinGW-w64 and `pkgconf`. Maintainers build the SDK from the pinned commits separately: @@ -66,9 +66,9 @@ make record-native-source record-native-package Set `RECORD_ARCH=arm64` or `RECORD_NATIVE_OUTPUT=` when the defaults do not match the target. The Makefile is the supported build interface; `.github/native/sdk.sh` is the underlying maintainer/CI implementation. -The `recorder-native-sdk` GitHub Actions workflow performs that source-build/package phase for every supported target and publishes the archives under the release tag declared in `.github/native/versions.env`. Release builds, `make full`, and `build.sh -p full` consume those archives. Pull-request CI falls back to the pinned source builders while a new SDK release is not available yet. Set `AISCAN_RECORD_BUILD_FROM_SOURCE=1` when invoking `make full` or `build.sh -p full` to opt into the slow source-build path locally. `AISCAN_RECORD_PREFIX` changes the SDK cache/install directory, and `AISCAN_RECORD_NATIVE_URL` can point downloads at an internal mirror. +The `recorder-native-sdk` GitHub Actions workflow performs that source-build/package phase for every supported target and publishes the archives under the release tag declared in `.github/native/versions.env`. It is independent of the normal CI and release build paths. Set `AISCAN_RECORD_BUILD_FROM_SOURCE=1` when invoking `make record` or `make record-native` to opt into the slow source-build path locally. `AISCAN_RECORD_PREFIX` changes the SDK cache/install directory, and `AISCAN_RECORD_NATIVE_URL` can point downloads at an internal mirror. -The source build verifies an exact FFmpeg component allowlist, and packaging rejects static libraries above a 16 MiB budget unless `AISCAN_RECORD_MAX_LIB_BYTES` explicitly overrides it. This prevents an FFmpeg upgrade or configure change from silently restoring all default codecs and adding tens of megabytes to `aiscan-full`. +The source build verifies an exact FFmpeg component allowlist, and packaging rejects static libraries above a 16 MiB budget unless `AISCAN_RECORD_MAX_LIB_BYTES` explicitly overrides it. This prevents an FFmpeg upgrade or configure change from silently restoring all default codecs and adding tens of megabytes to record-enabled binaries. Native smoke tests are opt-in because they require an interactive desktop/X11 session: diff --git a/docs/reference.md b/docs/reference.md index 3579c405..7928b844 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -84,6 +84,8 @@ cyberhub: url: "" key: "" mode: "" # merge（默认）或 override + proxy: "" # scanner/工具出口代理：socks5://、trojan://、vless://、clash:// + mitm: true # 记录工具流量（默认开）；false = 纯代理路由，不拦截/不抓包 # IOA 协作 ioa: @@ -172,6 +174,7 @@ misc: | 参数 | 说明 | | --- | --- | | `--proxy` | Scanner 代理，支持 `socks5://`、`trojan://`、`vless://`、`clash://`（订阅自动负载均衡） | +| `--mitm` | 是否记录工具流量（默认开启）。关闭后为纯代理路由，不拦截/不抓包 | | `--cyberhub-url` | Cyberhub 资源服务 URL | | `--cyberhub-key` | Cyberhub API key | | `--cyberhub-mode` | 资源模式：`merge`（默认）或 `override` | @@ -256,6 +259,22 @@ aiscan scan -i http://target.example --proxy clash://https://subscribe.example/l Agent 模式下还可通过 `proxy` 工具在运行时动态管理代理，详见 [Agent 模式详解](agent.md)。 +### 流量捕获与多级代理（MITM Hub） + +运行期常驻一个本地 MITM Hub 作为**统一路由底座**:所有工具(内置 curl/scanner、以及 bash 里的 curl/wget 等外部命令)的流量都经它出站。它有两层解耦—— + +- **稳定前端**:Hub 监听固定本地地址,一次性注入到所有工具(env + 内置 client),地址不变。 +- **动态后端**:出口代理链由 `proxy` 命令驱动(节点/订阅/负载均衡),`proxy switch/auto` 只热切换 Hub 的上游,已在跑的子进程无感,存量连接也能换出口。 + +两个命令职责分明,均为命令行优先: + +- `proxy` —— 管理代理(订阅、切换、负载均衡、一次性 `proxy ` 直连)。 +- `mitm` —— 查看已捕获流量:`mitm flows [--host --status --type --last]`、`mitm flow `、`mitm analyze`、`mitm clear`。 + +捕获默认开启,可用 `--mitm=false` 或配置 `mitm: false` 关闭(转为纯路由,不拦截 HTTPS、不抓包、无需信任 CA)。HTTPS 捕获会为工具注入 Hub CA(`CURL_CA_BUNDLE`/`SSL_CERT_FILE` 等);对**裸 IP** 目标的 HTTPS 因证书无 IP SAN 可能被严格校验拒绝,使用主机名不受影响。 + +作为 Cairn Runner 运行时,每次工具执行的完整流量会作为 `http.exchange.v1` 证据进入流量表(敏感头在 Runner 侧脱敏),覆盖全部工具流量而非仅漏洞相关的零散记录。 + ### LLM API 代理 `--llm-proxy` 单独为 LLM API 请求设置 HTTP 代理： diff --git a/docs/v1.0.0.md b/docs/v1.0.0.md index 03839ee3..551bcff9 100644 --- a/docs/v1.0.0.md +++ b/docs/v1.0.0.md @@ -7,9 +7,10 @@ v1.0.0 是 AIScan 的首个稳定接口基线。此前版本用于快速迭代 | 版本 | 平台 | 内容 | | --- | --- | --- | | `aiscan` | Linux/macOS/Windows amd64、arm64 | standard：scan、agent、IOA 和纯 Go 工具集 | -| `aiscan-full` | Linux amd64/arm64、macOS amd64/arm64、Windows amd64 | standard + Web、Playwright、passive、katana；Linux/Windows 支持原生录屏 | +| `aiscan-full` | Linux/macOS amd64、arm64；Windows amd64 | standard + Web、Playwright、passive、katana | +| `runner` | Linux/macOS/Windows amd64、arm64 | 单一、无 build tag 的远程工具节点 | -macOS full 包含 Web、浏览器和被动测绘能力，但不包含原生录屏。Windows arm64 只发布 standard。 +macOS standard/full 均由 Linux runner 交叉编译，不使用 macOS 原生 runner；Windows arm64 只发布 standard。原生 `record` 不包含在官方 full 产物中，SDK 和工具开发者可在 Linux/Windows 支持的平台上显式启用。 ## 从 pre-v1 迁移 @@ -62,4 +63,4 @@ PTY/terminal 路由位于 `pkg/terminal`。`core` 仅保留 AIScan 的领域配 ## 发布门禁 -正式发布前需通过：Go 单元/竞态/架构测试、`go vet`、`golangci-lint`、protobuf 与资源生成一致性、standard/full 构建、Web 前端构建与 E2E、cyber-ui viewer 测试、Windows 产物启动验证。依赖漏洞报告单独跟踪，不作为本次 v1.0.0 发布门禁。 +正式发布前需通过：Go 单元/竞态/架构测试、`go vet`、`golangci-lint`、protobuf 与资源生成一致性、standard/full/runner 构建、Web 前端构建与 E2E、cyber-ui viewer 测试、Windows 产物启动验证。CI 与发布 wrapper 共用只读的 release-build workflow，执行同一套构建、打包和 smoke test；只有门禁通过后的发布 wrapper 能创建 Git tag 和 Release。依赖漏洞报告单独跟踪，不作为本次 v1.0.0 发布门禁。 diff --git a/go.mod b/go.mod index 47ffd7c7..30b686e8 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/chainreactors/tui/console v0.0.0-20260712082522-2ba36ad7841f github.com/chainreactors/tui/readline v0.0.0-20260723062039-ed89e758c21b github.com/chainreactors/utils v0.0.0-20260711153742-f3d210a5fa9d - github.com/chainreactors/utils/mitmproxy v0.0.0-20260722180147-5b1816060721 + github.com/chainreactors/utils/mitmproxy v0.0.0-20260818093021-b0af431aff73 github.com/chainreactors/utils/parsers v0.0.3 github.com/chainreactors/utils/pty v0.0.0-20260722180147-5b1816060721 github.com/chainreactors/zombie v1.3.1-0.20260809133033-0d0df6fa50f5 @@ -154,7 +154,7 @@ require ( github.com/buger/jsonparser v1.1.2 // indirect github.com/carapace-sh/carapace-shlex v1.1.1 // indirect github.com/censys/censys-sdk-go v0.19.1 // indirect - github.com/chainreactors/aiscan/aop v0.0.0 + github.com/chainreactors/aiscan/aop v0.0.0-20260818112202-76d90a72b2c5 github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0 // indirect github.com/chainreactors/neutron/operators/full v0.1.1-0.20260704194031-f57d0a560e32 // indirect github.com/chainreactors/parsers v0.0.0-20260608085142-3d2c51baa8fe // indirect diff --git a/go.sum b/go.sum index 490bd840..48468c79 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,8 @@ github.com/chainreactors/utils v0.0.0-20260711153742-f3d210a5fa9d h1:wlJ6oMbVLKr github.com/chainreactors/utils v0.0.0-20260711153742-f3d210a5fa9d/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= github.com/chainreactors/utils/cert v0.0.0-20260722180147-5b1816060721 h1:mtC+2UKpXO5Yel5JL2Ah6Z2r/X6wx4Fbii/36MmKcLI= github.com/chainreactors/utils/cert v0.0.0-20260722180147-5b1816060721/go.mod h1:xvvWMcU9Fcht6GR1cc9ceAZ3/Hl2HrkoRzpeyOzx1rQ= -github.com/chainreactors/utils/mitmproxy v0.0.0-20260722180147-5b1816060721 h1:BJh043izz46BCpNN3SJBGwEQDWW9SrKLGUFrbP5+/H0= -github.com/chainreactors/utils/mitmproxy v0.0.0-20260722180147-5b1816060721/go.mod h1:2O3/Vw66VnbzhwsHGFJ2Ge98RuSh6XzMMFGZmMmlZ9M= +github.com/chainreactors/utils/mitmproxy v0.0.0-20260818093021-b0af431aff73 h1:ij4Mt9XkLpO0IR6d7S7mYQpfrpOEL8ys5cYM5KZKNvo= +github.com/chainreactors/utils/mitmproxy v0.0.0-20260818093021-b0af431aff73/go.mod h1:2O3/Vw66VnbzhwsHGFJ2Ge98RuSh6XzMMFGZmMmlZ9M= github.com/chainreactors/utils/parsers v0.0.3 h1:3ld7xG5TSvzikVOCkQHjqjHO3otjODwSwHQDkMKbu5o= github.com/chainreactors/utils/parsers v0.0.3/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= github.com/chainreactors/utils/pty v0.0.0-20260722180147-5b1816060721 h1:gxkedbTvFEFTtel7XJEPMVh1iznfD+91woPkGBXZMNk= diff --git a/internal/repositorytest/architecture.go b/internal/repositorytest/architecture.go new file mode 100644 index 00000000..a829ad7f --- /dev/null +++ b/internal/repositorytest/architecture.go @@ -0,0 +1,2 @@ +// Package repositorytest contains repository-wide architecture and quality guards. +package repositorytest diff --git a/archtest/architecture_test.go b/internal/repositorytest/architecture_test.go similarity index 75% rename from archtest/architecture_test.go rename to internal/repositorytest/architecture_test.go index 7f2e0d1e..bb4b7abf 100644 --- a/archtest/architecture_test.go +++ b/internal/repositorytest/architecture_test.go @@ -1,4 +1,4 @@ -package archtest +package repositorytest import ( "bytes" @@ -52,6 +52,100 @@ func TestRunnerDoesNotDependOnWeb(t *testing.T) { assertNoImportPrefix(t, filepath.Join(root, "pkg", "runner"), modulePath+"/pkg/rpc") } +func TestRunnerIsSingleTagFreeImplementation(t *testing.T) { + root := repositoryRoot(t) + for _, rel := range []string{filepath.Join("pkg", "runner"), filepath.Join("cmd", "runner")} { + dir := filepath.Join(root, rel) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" { + continue + } + content, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err != nil { + t.Fatal(err) + } + if strings.HasPrefix(string(content), "//go:build ") { + t.Errorf("runner source must not use build tags: %s", filepath.Join(rel, entry.Name())) + } + } + } + + runnerDir := filepath.Join(root, "pkg", "runner") + entries, err := os.ReadDir(runnerDir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + source := strings.TrimSuffix(entry.Name(), "_test.go") + ".go" + if _, err := os.Stat(filepath.Join(runnerDir, source)); err != nil { + t.Errorf("runner test must map to exactly one source file: %s", entry.Name()) + } + } + + makefile := readRepositoryFile(t, root, "Makefile") + start := strings.Index(makefile, "runner: prepare\n") + if start < 0 { + t.Fatal("Makefile missing runner target") + } + block := makefile[start:] + if next := strings.Index(block, "\n\n"); next >= 0 { + block = block[:next] + } + if !strings.Contains(block, "./cmd/runner") { + t.Error("Makefile runner target must build ./cmd/runner") + } + if strings.Contains(block, "-tags") { + t.Error("Makefile runner target must not use build tags") + } + + releaseWorkflow := readRepositoryFile(t, root, filepath.Join(".github", "workflows", "release-build.yml")) + if count := strings.Count(releaseWorkflow, "main: ./cmd/runner"); count != 1 { + t.Fatalf("release workflow must contain exactly one runner build, got %d", count) + } + runnerStart := strings.Index(releaseWorkflow, " - id: runner\n") + if runnerStart < 0 { + t.Fatal("release workflow is missing the runner build") + } + runnerBuild := releaseWorkflow[runnerStart:] + if next := strings.Index(runnerBuild[1:], "\n - id:"); next >= 0 { + runnerBuild = runnerBuild[:next+1] + } + for _, required := range []string{ + "profile: runner", + "main: ./cmd/runner", + "binary: runner", + "tags: \"\"", + "linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64", + } { + if !strings.Contains(runnerBuild, required) { + t.Errorf("runner release build is missing %q", required) + } + } + + goreleaser := readRepositoryFile(t, root, ".goreleaser.yml") + if count := strings.Count(goreleaser, "main: ./cmd/runner"); count != 1 { + t.Fatalf("GoReleaser must contain exactly one runner build, got %d", count) + } + runnerStart = strings.Index(goreleaser, " - id: runner\n") + if runnerStart < 0 { + t.Fatal("GoReleaser is missing the runner build") + } + runnerBuild = goreleaser[runnerStart:] + if next := strings.Index(runnerBuild[1:], "\n - id:"); next >= 0 { + runnerBuild = runnerBuild[:next+1] + } + if strings.Contains(runnerBuild, "\n tags:") { + t.Error("GoReleaser runner build must not use build tags") + } +} + func TestGeneratedProtobufLivesInOwnedProtocolTrees(t *testing.T) { root := repositoryRoot(t) for _, rel := range trackedFiles(t, root) { @@ -169,7 +263,8 @@ func TestGoTestFilesFollowSourceFiles(t *testing.T) { continue } base := strings.TrimSuffix(filepath.Base(path), "_test.go") - matched := false + _, exactErr := os.Stat(filepath.Join(filepath.Dir(path), base+".go")) + matched := exactErr == nil for _, source := range sources[filepath.Dir(path)] { if base == source { matched = true @@ -279,9 +374,11 @@ func TestBuildProfilesUseExpectedCGOModes(t *testing.T) { for _, required := range []string{ "GO_LDFLAGS ?= -s -w", "standard: prepare\n\tCGO_ENABLED=0 $(GO) build $(BUILD_FLAGS) -ldflags \"$(GO_LDFLAGS)\"", - "full: frontend record-native prepare\n\t$(RECORD_BUILD_ENV) CGO_ENABLED=1 $(GO) build $(BUILD_FLAGS) -ldflags \"$(GO_LDFLAGS)\"", + "full: frontend prepare\n\tCGO_ENABLED=1 $(GO) build $(BUILD_FLAGS) -ldflags \"$(GO_LDFLAGS)\"", + "record: frontend record-native prepare\n\t$(RECORD_BUILD_ENV) CGO_ENABLED=1 $(GO) build $(BUILD_FLAGS) -ldflags \"$(GO_LDFLAGS)\"", "STANDARD_TAGS := forceposix emptytemplates noembed osusergo netgo", - "FULL_TAGS := forceposix emptytemplates noembed osusergo netgo full sqlite record_ffmpeg re2_cgo re2_static", + "FULL_TAGS := forceposix emptytemplates noembed osusergo netgo full sqlite re2_cgo re2_static", + "RECORD_TAGS := $(FULL_TAGS) record_ffmpeg", } { if !strings.Contains(makefile, required) { t.Errorf("Makefile missing build profile contract %q", required) @@ -291,7 +388,7 @@ func TestBuildProfilesUseExpectedCGOModes(t *testing.T) { for _, required := range []string{ "CGO_MODE=0", "CGO_MODE=1", - `EXTRA_TAGS="full,record_ffmpeg,re2_cgo,re2_static${EXTRA_TAGS:+,$EXTRA_TAGS}"`, + `EXTRA_TAGS="full,re2_cgo,re2_static${EXTRA_TAGS:+,$EXTRA_TAGS}"`, `CGO_ENABLED="$CGO_MODE"`, `OSARCH="${HOST_OS}/${HOST_ARCH}"`, } { @@ -299,6 +396,17 @@ func TestBuildProfilesUseExpectedCGOModes(t *testing.T) { t.Errorf("build.sh missing build profile contract %q", required) } } + for name, profile := range map[string]string{"Makefile full profile": makefile, "build.sh full profile": buildScript} { + if strings.Contains(profile, "full,record_ffmpeg") || strings.Contains(profile, "full: frontend record-native") { + t.Errorf("%s must not enable the optional recorder", name) + } + } + releaseWorkflow := readRepositoryFile(t, root, filepath.Join(".github", "workflows", "release-build.yml")) + for _, forbidden := range []string{"record_ffmpeg", "matrix.recorder"} { + if strings.Contains(releaseWorkflow, forbidden) { + t.Errorf("release workflow must not enable the optional recorder; found %q", forbidden) + } + } goreleaser := readRepositoryFile(t, root, ".goreleaser.yml") fullStart := strings.Index(goreleaser, " - id: aiscan-full\n") @@ -312,6 +420,9 @@ func TestBuildProfilesUseExpectedCGOModes(t *testing.T) { if !strings.Contains(fullConfig, "CGO_ENABLED=1") { t.Error(".goreleaser.yml aiscan-full must enable CGO") } + if !strings.Contains(fullConfig, " - darwin\n") { + t.Error(".goreleaser.yml aiscan-full must publish Darwin builds") + } for _, tag := range []string{"re2_cgo", "re2_static"} { if !strings.Contains(fullConfig, " - "+tag+"\n") { t.Errorf(".goreleaser.yml aiscan-full missing build tag %q", tag) @@ -319,6 +430,89 @@ func TestBuildProfilesUseExpectedCGOModes(t *testing.T) { } } +func TestGitHubActionsCrossCompileDarwinWithoutMacOSRunners(t *testing.T) { + root := repositoryRoot(t) + workflowDir := filepath.Join(root, ".github", "workflows") + macOSRunner := regexp.MustCompile(`(?i)^(?:runs-on|runner):\s*macos(?:-|\s|$)`) + err := filepath.WalkDir(workflowDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || (filepath.Ext(path) != ".yml" && filepath.Ext(path) != ".yaml") { + return nil + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + for lineNumber, line := range strings.Split(string(content), "\n") { + if macOSRunner.MatchString(strings.TrimSpace(line)) { + t.Errorf("macOS GitHub Actions runner in %s:%d", relative(root, path), lineNumber+1) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + releaseWorkflow := readRepositoryFile(t, root, filepath.Join(".github", "workflows", "release-build.yml")) + standardStart := strings.Index(releaseWorkflow, " - id: aiscan\n") + if standardStart < 0 { + t.Fatal("release workflow is missing the standard aiscan build") + } + standardConfig := releaseWorkflow[standardStart:] + if next := strings.Index(standardConfig[1:], "\n - id:"); next >= 0 { + standardConfig = standardConfig[:next+1] + } + for _, required := range []string{ + "runner: ubuntu-22.04", + "darwin/amd64", + "darwin/arm64", + `cgo: "0"`, + } { + if !strings.Contains(standardConfig, required) { + t.Errorf("standard release build must cross-compile Darwin on Linux; missing %q", required) + } + } + + fullDarwinStart := strings.Index(releaseWorkflow, " - id: aiscan-full-darwin\n") + if fullDarwinStart < 0 { + t.Fatal("release workflow is missing the full Darwin cross-build") + } + fullDarwinConfig := releaseWorkflow[fullDarwinStart:] + if next := strings.Index(fullDarwinConfig[1:], "\n - id:"); next >= 0 { + fullDarwinConfig = fullDarwinConfig[:next+1] + } + for _, required := range []string{ + "runner: ubuntu-22.04", + "darwin/amd64", + "darwin/arm64", + `cgo: "1"`, + "cross: darwin", + "re2_cgo", + "re2_static", + } { + if !strings.Contains(fullDarwinConfig, required) { + t.Errorf("full release build must cross-compile Darwin CGO binaries on Linux; missing %q", required) + } + } + if strings.Contains(fullDarwinConfig, "record_ffmpeg") { + t.Error("full Darwin cross-build must not enable the unsupported native recorder") + } + versions := readRepositoryFile(t, root, filepath.Join(".github", "native", "versions.env")) + for _, required := range []string{ + "MACOS_CROSS_ZIG_VERSION=", + "MACOS_CROSS_SDK_VERSION=", + "MACOS_CROSS_SDK_SHA256=", + "MACOS_CROSS_DEPLOYMENT_TARGET=", + } { + if !strings.Contains(versions, required) { + t.Errorf("native versions file is missing macOS cross-build pin %q", required) + } + } +} + func TestRecorderNativeBuildUsesSingleSDKScript(t *testing.T) { root := repositoryRoot(t) obsoleteScripts := []string{ @@ -348,6 +542,7 @@ func TestRecorderNativeBuildUsesSingleSDKScript(t *testing.T) { "build.sh", filepath.Join(".github", "workflows", "ci.yml"), filepath.Join(".github", "workflows", "go-release.yml"), + filepath.Join(".github", "workflows", "release-build.yml"), filepath.Join(".github", "workflows", "record-native.yml"), } { content := readRepositoryFile(t, root, rel) diff --git a/pkg/commands/bash.go b/pkg/commands/bash.go index 959e8362..9b866c9e 100644 --- a/pkg/commands/bash.go +++ b/pkg/commands/bash.go @@ -2,6 +2,8 @@ package commands import ( "context" + "encoding/json" + "errors" "fmt" "io" "os" @@ -18,30 +20,34 @@ import ( ) const ( - defaultTimeout = 300 - autoBackgroundThreshold = 15 * time.Second + defaultTimeout = 600 + unlimitedTimeout = time.Duration(1<<63 - 1) streamInterval = 100 * time.Millisecond monitorInterval = 10 * time.Second + completionRetryInterval = 10 * time.Millisecond ) // BashExecOptions controls one foreground execution without mutating the // BashTool defaults. Runner/WebAgent transports use this entry point while the -// agent-facing Execute method keeps its auto-background behavior. +// agent-facing Execute method applies the explicit wait/background contract. type BashExecOptions struct { - Name string - WorkDir string - Env map[string]string - Timeout time.Duration - OnOutput func([]byte) - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer + Name string + WorkDir string + Env map[string]string + Timeout time.Duration + TimeoutSet bool + OnOutput func([]byte) + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer } type BashTool struct { workDir string timeout int scannerProxy string + scannerProxyCA string + egressResolver func(callID string) (proxyURL, caPath string) tasks *tmux.Manager commandNames func() []string resolveCommand func(string) (Command, bool) @@ -49,6 +55,7 @@ type BashTool struct { adapterMu sync.Mutex shellAdapter *shellCommandAdapter closeOnce sync.Once + audit *FileAudit } func NewBashTool(workDir string, timeout int) *BashTool { @@ -58,8 +65,20 @@ func NewBashTool(workDir string, timeout int) *BashTool { return &BashTool{workDir: workDir, timeout: timeout, tasks: tmux.NewManager()} } -func (t *BashTool) Manager() *tmux.Manager { return t.tasks } -func (t *BashTool) SetScannerProxy(proxy string) { t.scannerProxy = proxy } +// WithAudit attaches the file-access audit trail. Shell commands are the one +// place the runtime cannot observe a file access directly, so what this buys is +// the work dir diff taken around every execution. +func (t *BashTool) WithAudit(audit *FileAudit) *BashTool { + t.audit = audit + return t +} + +func (t *BashTool) Manager() *tmux.Manager { return t.tasks } +func (t *BashTool) SetScannerProxy(proxy string) { t.scannerProxy = proxy } +func (t *BashTool) SetScannerProxyCA(caPath string) { t.scannerProxyCA = caPath } +func (t *BashTool) SetEgressResolver(fn func(callID string) (string, string)) { + t.egressResolver = fn +} func (t *BashTool) SetCommandNames(fn func() []string) { t.commandNames = fn } func (t *BashTool) SetCommandResolver(fn func(string) (Command, bool)) { t.resolveCommand = fn @@ -112,6 +131,16 @@ func (t *BashTool) WithScannerProxy(proxy string) *BashTool { return t } +func (t *BashTool) WithScannerProxyCA(caPath string) *BashTool { + t.scannerProxyCA = caPath + return t +} + +func (t *BashTool) WithEgressResolver(fn func(callID string) (string, string)) *BashTool { + t.egressResolver = fn + return t +} + func (t *BashTool) Description() string { desc := "Execute a shell command and return its output." if t.commandNames != nil { @@ -124,7 +153,45 @@ func (t *BashTool) Description() string { type BashArgs struct { Command string `json:"command" jsonschema:"description=The command to execute. For shell commands: any valid sh command. For pseudo-commands (scan, gogo, tmux, etc.): pass them directly here."` - Timeout int `json:"timeout,omitempty" jsonschema:"description=Optional timeout in seconds. The command is killed when it exceeds this. Omit to use the default (300s). Commands still running after 15s are moved to background and keep running until this timeout."` + Wait int `json:"wait,omitempty" jsonschema:"minimum=0,description=Foreground wait in seconds. 0 waits until completion. A positive value moves a still-running command to background after that many seconds without canceling it."` + Timeout int `json:"timeout,omitempty" jsonschema:"minimum=0,description=Maximum total command runtime in seconds. 0 means unlimited when explicitly provided. Omit to use the default (600s). The timeout continues to apply after a command moves to background."` + + timeoutSet bool +} + +// UnmarshalJSON preserves the distinction between an omitted timeout (use the +// tool default) and an explicit timeout of zero (no command deadline). +func (a *BashArgs) UnmarshalJSON(data []byte) error { + var raw struct { + Command string `json:"command"` + Wait int `json:"wait"` + Timeout *int `json:"timeout"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + a.Command = raw.Command + a.Wait = raw.Wait + a.Timeout = 0 + a.timeoutSet = raw.Timeout != nil + if raw.Timeout != nil { + a.Timeout = *raw.Timeout + } + return nil +} + +func (a BashArgs) TimeoutSpecified() bool { + return a.timeoutSet || a.Timeout != 0 +} + +func (a BashArgs) Validate() error { + if a.Wait < 0 { + return fmt.Errorf("wait must be greater than or equal to 0") + } + if a.Timeout < 0 { + return fmt.Errorf("timeout must be greater than or equal to 0") + } + return nil } func (t *BashTool) Definition() *coretool.Definition { @@ -136,6 +203,9 @@ func (t *BashTool) Execute(ctx context.Context, arguments string) (*coretool.Res if err != nil { return nil, err } + if err := args.Validate(); err != nil { + return nil, err + } command := strings.TrimSpace(args.Command) if command == "" { @@ -145,17 +215,24 @@ func (t *BashTool) Execute(ctx context.Context, arguments string) (*coretool.Res return coretool.TextResult("ok"), nil } - options := BashExecOptions{} - options.WorkDir = coretool.WorkDirFromContext(ctx, "") - if args.Timeout > 0 { + options := BashExecOptions{WorkDir: coretool.WorkDirFromContext(ctx, "")} + if args.TimeoutSpecified() { options.Timeout = time.Duration(args.Timeout) * time.Second + options.TimeoutSet = true } - execution, err := t.Start(ctx, command, options) + var result *coretool.Result + err = t.audit.Around(ctx, options.WorkDir, func() error { + execution, startErr := t.Start(ctx, command, options) + if startErr != nil { + return startErr + } + result = t.waitOrBackground(execution, ctx, inbox.FromContext(ctx), time.Duration(args.Wait)*time.Second) + return nil + }) if err != nil { return nil, err } - - return t.waitOrBackground(execution, ctx, inbox.FromContext(ctx)), nil + return result, nil } // RunForeground executes command through the same tmux/registered-command @@ -163,6 +240,20 @@ func (t *BashTool) Execute(ctx context.Context, arguments string) (*coretool.Res // final session state. Non-zero exits are represented by Info.ExitCode rather // than returned as transport errors. func (t *BashTool) RunForeground(ctx context.Context, command string, options BashExecOptions) (*Execution, error) { + workDir := options.WorkDir + if workDir == "" { + workDir = coretool.WorkDirFromContext(ctx, t.workDir) + } + var execution *Execution + err := t.audit.Around(ctx, workDir, func() error { + var runErr error + execution, runErr = t.runForeground(ctx, command, options) + return runErr + }) + return execution, err +} + +func (t *BashTool) runForeground(ctx context.Context, command string, options BashExecOptions) (*Execution, error) { command = strings.TrimSpace(command) if command == "" { return nil, fmt.Errorf("empty command") @@ -225,7 +316,7 @@ func (t *BashTool) RunForeground(ctx context.Context, command string, options Ba // RunForegroundTool executes a command in the foreground and returns the // collected ToolResult (bounded text and media), streaming raw -// output through options.OnOutput. Transports that must not auto-background +// output through options.OnOutput. Transports that must remain foreground // (AOP tool.call) use this instead of Execute. func (t *BashTool) RunForegroundTool(ctx context.Context, command string, options BashExecOptions) (*coretool.Result, error) { execution, err := t.RunForeground(ctx, command, options) @@ -246,9 +337,15 @@ func (t *BashTool) Start(ctx context.Context, command string, options BashExecOp ctx = context.Background() } timeout := options.Timeout - if timeout <= 0 { + if timeout < 0 { + return nil, fmt.Errorf("timeout must be greater than or equal to 0") + } + if timeout == 0 && !options.TimeoutSet { timeout = time.Duration(t.timeout) * time.Second } + if timeout == 0 { + timeout = unlimitedTimeout + } workDir := options.WorkDir if workDir == "" { workDir = t.workDir @@ -260,7 +357,7 @@ func (t *BashTool) Start(ctx context.Context, command string, options BashExecOp if tokens, err := SplitCommandLine(left); err == nil { if args, syntaxErr := stripShellSyntax(tokens[1:]); syntaxErr == nil { args = normalizeNoColor(cmd.Name, args) - return t.startBuiltin(ctx, cmd, args, timeout, workDir, t.runEnv(options.Env, nil, ""), options) + return t.startBuiltin(ctx, cmd, args, timeout, workDir, t.runEnv(ctx, options.Env, nil, ""), options) } } } @@ -271,7 +368,7 @@ func (t *BashTool) Start(ctx context.Context, command string, options BashExecOp } if adapter != nil { contextID := adapter.retainContext(ctx) - env := t.runEnv(options.Env, adapter, contextID) + env := t.runEnv(ctx, options.Env, adapter, contextID) execution := newExecution(t.tasks, command, nil, workDir, env) info, err := t.tasks.Create(workDir, command, options.Name, timeout, env, "") if err != nil { @@ -285,7 +382,7 @@ func (t *BashTool) Start(ctx context.Context, command string, options BashExecOp }() return execution, nil } - env := t.runEnv(options.Env, nil, "") + env := t.runEnv(ctx, options.Env, nil, "") if cmd, ok := t.resolve(leftToken); ok { tokens, err := SplitCommandLine(left) if err != nil { @@ -481,18 +578,29 @@ func configureProcess(cmd *exec.Cmd, workDir string, env []string) { } } -func (t *BashTool) waitOrBackground(execution *Execution, ctx context.Context, targetInbox inbox.Inbox) *coretool.Result { +func (t *BashTool) waitOrBackground(execution *Execution, ctx context.Context, targetInbox inbox.Inbox, wait time.Duration) *coretool.Result { done := t.tasks.Done(execution.ID) + var waitTimer *time.Timer + var waitDone <-chan time.Time + if wait > 0 { + waitTimer = time.NewTimer(wait) + waitDone = waitTimer.C + defer waitTimer.Stop() + } select { case <-done: execution.refresh() return t.collectResult(execution) - case <-time.After(autoBackgroundThreshold): - info, _ := t.tasks.Get(execution.ID) + case <-waitDone: + info, ok := t.tasks.Get(execution.ID) + if !ok { + execution.refresh() + return t.collectResult(execution) + } t.startMonitor(info, targetInbox) return coretool.TextResult(fmt.Sprintf( - "Command auto-backgrounded (exceeded %s).\nsession id=%s name=%s\nIncremental output will be delivered automatically. Use `tmux kill -t %s` to stop.", - autoBackgroundThreshold, info.ID, info.Name, info.ID)) + "Command moved to background after waiting %s. It is still running.\nsession id=%s name=%s\nCompletion will be delivered automatically. Use `tmux kill -t %s` to stop.", + wait, info.ID, info.Name, info.ID)) case <-ctx.Done(): _ = execution.Kill() <-done @@ -522,9 +630,9 @@ func (t *BashTool) collectResult(execution *Execution) *coretool.Result { return result } -func (t *BashTool) runEnv(overrides map[string]string, adapter *shellCommandAdapter, shellContextID string) []string { +func (t *BashTool) runEnv(ctx context.Context, overrides map[string]string, adapter *shellCommandAdapter, shellContextID string) []string { values := make(map[string]string) - for _, item := range t.proxyEnv() { + for _, item := range t.proxyEnv(ctx) { if key, value, ok := strings.Cut(item, "="); ok { values[key] = value } @@ -556,29 +664,58 @@ func (t *BashTool) runEnv(overrides map[string]string, adapter *shellCommandAdap return out } -func (t *BashTool) proxyEnv() []string { - if t.scannerProxy == "" { +func (t *BashTool) proxyEnv(ctx context.Context) []string { + proxy, ca := t.scannerProxy, t.scannerProxyCA + // When an egress resolver is wired, it supersedes the static values: it tags + // the proxy URL with this execution's tool-call id (so the hub attributes + // captured flows to it) and returns the CA path from live hub state — empty + // while the hub is not intercepting, so a relaying child is not handed a + // CA-only bundle that would reject the real server certificate. + if t.egressResolver != nil { + callID := coretool.InvocationFromContext(ctx).CallID + proxy, ca = t.egressResolver(callID) + } + if proxy == "" { return nil } - return []string{ - "ALL_PROXY=" + t.scannerProxy, "all_proxy=" + t.scannerProxy, - "HTTP_PROXY=" + t.scannerProxy, "http_proxy=" + t.scannerProxy, - "HTTPS_PROXY=" + t.scannerProxy, "https_proxy=" + t.scannerProxy, - } + env := []string{ + "ALL_PROXY=" + proxy, "all_proxy=" + proxy, + "HTTP_PROXY=" + proxy, "http_proxy=" + proxy, + "HTTPS_PROXY=" + proxy, "https_proxy=" + proxy, + } + // Point common HTTP clients at the MITM hub CA so intercepted HTTPS is + // trusted. Tools that use the system pool or pin certs ignore these and + // degrade to CONNECT-metadata capture, which is acceptable. + if ca != "" { + env = append(env, + "CURL_CA_BUNDLE="+ca, + "SSL_CERT_FILE="+ca, + "NODE_EXTRA_CA_CERTS="+ca, + "REQUESTS_CA_BUNDLE="+ca, + "GIT_SSL_CAINFO="+ca, + ) + } + return env } func (t *BashTool) startMonitor(info tmux.Info, targetInbox inbox.Inbox) { if targetInbox == nil { return } + producer := targetInbox.RegisterProducer("bash:" + info.ID) t.tasks.Monitor(info.ID, monitorInterval, func(output string) { msg := inbox.NewMessage(inbox.OriginSession, "user", fmt.Sprintf("\n%s\n", info.ID, info.Name, output)) msg.Priority = inbox.PriorityLow msg.Meta = map[string]any{"session_id": info.ID, "session_name": info.Name, "type": "incremental"} + // Incremental output is best-effort. Completion below is high priority + // and retried so a full inbox cannot make a background task disappear. _ = targetInbox.Push(msg) }) go func() { + if producer != nil { + defer producer.Done() + } <-t.tasks.Done(info.ID) final, ok := t.tasks.Get(info.ID) if !ok { @@ -586,15 +723,33 @@ func (t *BashTool) startMonitor(info tmux.Info, targetInbox inbox.Inbox) { } tail := t.tasks.PeekOrEmpty(info.ID, 20) msg := inbox.NewMessage(inbox.OriginSession, "user", tmux.FormatCompletion(final, tail)) + msg.Priority = inbox.PriorityHigh msg.Meta = map[string]any{ "session_id": final.ID, "session_name": final.Name, "exit_code": final.ExitCode, + "type": "completion", } - _ = targetInbox.Push(msg) + pushCompletion(targetInbox, msg) }() } +func pushCompletion(targetInbox inbox.Inbox, msg inbox.Message) { + for { + err := targetInbox.Push(msg) + switch { + case err == nil, errors.Is(err, inbox.ErrInboxClosed): + return + case !errors.Is(err, inbox.ErrInboxFull): + return + } + if targetInbox.Closed() { + return + } + time.Sleep(completionRetryInterval) + } +} + func isOnlyCommentsOrBlank(cmdLine string) bool { for _, line := range strings.Split(cmdLine, "\n") { trimmed := strings.TrimSpace(line) diff --git a/pkg/commands/bash_test.go b/pkg/commands/bash_test.go index be5481c8..dfe2a207 100644 --- a/pkg/commands/bash_test.go +++ b/pkg/commands/bash_test.go @@ -59,6 +59,12 @@ type stagedOutputCommand struct { value string } +type delayedCommand struct { + name string + delay time.Duration + output string +} + func (c *stagedOutputCommand) Name() string { return c.name } func (c *stagedOutputCommand) Usage() string { return c.name } func (c *stagedOutputCommand) Run(_ context.Context, execution *Execution) (any, error) { @@ -68,6 +74,18 @@ func (c *stagedOutputCommand) Run(_ context.Context, execution *Execution) (any, return nil, nil } +func (c *delayedCommand) Run(ctx context.Context, execution *Execution) (any, error) { + timer := time.NewTimer(c.delay) + defer timer.Stop() + select { + case <-timer.C: + _, err := fmt.Fprint(execution.Stdout, c.output) + return nil, err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + func (c *outputCommand) Name() string { return c.name } func (c *outputCommand) Usage() string { return c.name + " — test command" } func (c *outputCommand) Run(_ context.Context, execution *Execution) (any, error) { @@ -640,6 +658,115 @@ func TestBashExecuteHonorsTimeoutArg(t *testing.T) { } } +func TestBashArgsDistinguishesOmittedAndZeroTimeout(t *testing.T) { + omitted, err := tool.ParseArgs[BashArgs](`{"command":"work"}`) + if err != nil { + t.Fatal(err) + } + if omitted.TimeoutSpecified() { + t.Fatal("omitted timeout should use the tool default") + } + + unlimited, err := tool.ParseArgs[BashArgs](`{"command":"work","timeout":0}`) + if err != nil { + t.Fatal(err) + } + if !unlimited.TimeoutSpecified() || unlimited.Timeout != 0 { + t.Fatalf("explicit zero timeout was not preserved: %+v", unlimited) + } +} + +func TestBashWaitZeroStaysForeground(t *testing.T) { + registry := NewRegistry() + delayed := &delayedCommand{name: "delayed", delay: 200 * time.Millisecond, output: "finished"} + registry.Register(Command{Name: delayed.name, Usage: delayed.name, Run: delayed.Run}, "") + bash := NewBashTool(t.TempDir(), 2) + bash.SetCommandResolver(registry.Get) + defer bash.Close() + + started := time.Now() + res, err := bash.Execute(context.Background(), `{"command":"delayed","wait":0}`) + if err != nil { + t.Fatal(err) + } + if elapsed := time.Since(started); elapsed < 150*time.Millisecond { + t.Fatalf("wait=0 returned before completion after %s", elapsed) + } + if got := tool.ResultText(res); !strings.Contains(got, "finished") || strings.Contains(got, "background") { + t.Fatalf("result = %q", got) + } +} + +func TestBashExplicitWaitMovesRunningCommandToBackground(t *testing.T) { + registry := NewRegistry() + delayed := &delayedCommand{name: "delayed", delay: 1500 * time.Millisecond, output: "finished"} + registry.Register(Command{Name: delayed.name, Usage: delayed.name, Run: delayed.Run}, "") + bash := NewBashTool(t.TempDir(), 3) + bash.SetCommandResolver(registry.Get) + defer bash.Close() + + scoped := inbox.NewBuffered(8) + defer scoped.Close() + ctx := inbox.ContextWithInbox(context.Background(), scoped) + started := time.Now() + res, err := bash.Execute(ctx, `{"command":"delayed","wait":1}`) + if err != nil { + t.Fatal(err) + } + elapsed := time.Since(started) + if elapsed < 800*time.Millisecond || elapsed > 1400*time.Millisecond { + t.Fatalf("wait=1 background transition took %s", elapsed) + } + if got := tool.ResultText(res); !strings.Contains(got, "moved to background") { + t.Fatalf("result = %q", got) + } + if scoped.ActiveProducers() != 1 { + t.Fatalf("active producers = %d, want 1", scoped.ActiveProducers()) + } + + waitCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + foundCompletion := false + for !foundCompletion && scoped.Wait(waitCtx) { + for _, msg := range scoped.Drain() { + if msg.Meta["type"] == "completion" { + foundCompletion = true + } + } + } + if !foundCompletion { + t.Fatal("completion message missing") + } + deadline := time.Now().Add(time.Second) + for scoped.ActiveProducers() != 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if scoped.ActiveProducers() != 0 { + t.Fatalf("producer remained active after completion: %d", scoped.ActiveProducers()) + } +} + +func TestBashExplicitZeroTimeoutIsUnlimited(t *testing.T) { + registry := NewRegistry() + delayed := &delayedCommand{name: "delayed", delay: 1200 * time.Millisecond, output: "finished"} + registry.Register(Command{Name: delayed.name, Usage: delayed.name, Run: delayed.Run}, "") + bash := NewBashTool(t.TempDir(), 1) + bash.SetCommandResolver(registry.Get) + defer bash.Close() + + started := time.Now() + res, err := bash.Execute(context.Background(), `{"command":"delayed","wait":0,"timeout":0}`) + if err != nil { + t.Fatal(err) + } + if elapsed := time.Since(started); elapsed < time.Second { + t.Fatalf("timeout=0 did not remain unlimited; returned after %s", elapsed) + } + if got := tool.ResultText(res); !strings.Contains(got, "finished") || strings.Contains(got, "command stopped") { + t.Fatalf("result = %q", got) + } +} + func TestBashRunTimeoutStopsSession(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("shell assertions are unix-only") @@ -795,8 +922,13 @@ func TestExecuteTool_PanicDoesNotAffectSubsequentCalls(t *testing.T) { func TestBashBackgroundMonitorUsesInvocationInbox(t *testing.T) { tool := NewBashTool(t.TempDir(), 5) defer tool.Close() - scoped := inbox.NewBuffered(8) + scoped := inbox.NewBuffered(1) defer scoped.Close() + low := inbox.NewMessage(inbox.OriginSession, "user", "incremental") + low.Priority = inbox.PriorityLow + if err := scoped.Push(low); err != nil { + t.Fatal(err) + } release := make(chan struct{}) info, err := tool.tasks.CreateFunc(context.Background(), "scoped-inbox", 5*time.Second, func(context.Context, io.Writer) error { @@ -807,18 +939,66 @@ func TestBashBackgroundMonitorUsesInvocationInbox(t *testing.T) { t.Fatal(err) } tool.startMonitor(info, scoped) + if scoped.ActiveProducers() != 1 { + t.Fatalf("active producers = %d, want 1", scoped.ActiveProducers()) + } close(release) deadline := time.Now().Add(2 * time.Second) - received := false - for time.Now().Before(deadline) { - if len(scoped.Drain()) > 0 { - received = true - break + for scoped.ActiveProducers() != 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if scoped.ActiveProducers() != 0 { + t.Fatal("background producer was not closed") + } + receivedCompletion := false + for _, msg := range scoped.Drain() { + if msg.Meta["type"] == "completion" { + receivedCompletion = true + } + } + if !receivedCompletion { + t.Fatal("high-priority completion did not replace buffered incremental output") + } +} + +func TestBashBackgroundMonitorDeliversConcurrentCompletions(t *testing.T) { + tool := NewBashTool(t.TempDir(), 5) + defer tool.Close() + scoped := inbox.NewBuffered(64) + defer scoped.Close() + + const jobs = 16 + release := make(chan struct{}) + for i := 0; i < jobs; i++ { + info, err := tool.tasks.CreateFunc(context.Background(), fmt.Sprintf("job-%d", i), 5*time.Second, func(context.Context, io.Writer) error { + <-release + return nil + }) + if err != nil { + t.Fatal(err) } + tool.startMonitor(info, scoped) + } + if scoped.ActiveProducers() != jobs { + t.Fatalf("active producers = %d, want %d", scoped.ActiveProducers(), jobs) + } + close(release) + + deadline := time.Now().Add(3 * time.Second) + for scoped.ActiveProducers() != 0 && time.Now().Before(deadline) { time.Sleep(10 * time.Millisecond) } - if !received { - t.Fatal("scoped inbox did not receive background completion") + if scoped.ActiveProducers() != 0 { + t.Fatalf("active producers = %d after completion", scoped.ActiveProducers()) + } + completions := 0 + for _, msg := range scoped.Drain() { + if msg.Meta["type"] == "completion" { + completions++ + } + } + if completions != jobs { + t.Fatalf("completion messages = %d, want %d", completions, jobs) } } diff --git a/pkg/commands/factory.go b/pkg/commands/factory.go index ef1df105..cbfd8703 100644 --- a/pkg/commands/factory.go +++ b/pkg/commands/factory.go @@ -34,8 +34,14 @@ type Deps struct { SkillStore SkillSource RunnerMode bool - Provider provider.Provider - ScannerProxy string + Provider provider.Provider + ScannerProxy string + ScannerProxyCA string // CA PEM path for the MITM hub; injected so children trust intercepted HTTPS + // EgressResolver, when set, supersedes ScannerProxy/ScannerProxyCA per + // execution: given the current tool-call id it returns the proxy URL (with + // the id as the proxy username, so captured flows attribute to it) and the + // CA path from live hub state (empty while the hub is not intercepting). + EgressResolver func(callID string) (proxyURL, caPath string) Logger telemetry.Logger NodeName string NodeMeta map[string]any @@ -43,6 +49,9 @@ type Deps struct { PlaywrightSession string Events aop.EventEmitter Hooks *hooks.Registry + // FileAudit collects what the file tools and shell executions did to the + // filesystem. Nil leaves them unobserved. + FileAudit *FileAudit } // Provide stores a typed dependency, allocating the bag on first use so a diff --git a/pkg/commands/file_audit.go b/pkg/commands/file_audit.go new file mode 100644 index 00000000..54c6b38c --- /dev/null +++ b/pkg/commands/file_audit.go @@ -0,0 +1,402 @@ +package commands + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + filepb "github.com/chainreactors/aiscan/aop/file" + "github.com/chainreactors/aiscan/core/eventbus" + coretool "github.com/chainreactors/aiscan/core/tool" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// FileAudit is the runtime's file-access audit trail: what the agent read, what +// it wrote, and what its shell commands left behind. +// +// An agent that operates on a machine is answerable for what it touched there, +// and a tool call alone does not say it. This type is where the tools in this +// package report that, and it publishes each observation as an aop.file.Access +// so the file namespace answers both "operate on this file" and "who touched +// it" — one mechanism rather than two. +// +// Recording is off the critical path by construction: it is non-blocking and +// lossy under pressure. A file access must never be slower because someone is +// watching, so a full queue drops the observation and counts it. +// +// Coverage is honest rather than complete. Tool-level records are exact. Shell +// executions are covered by diffing the work dir around them (Around), which +// sees every write but no read at all — an unmodified read leaves nothing +// behind to find. That difference is on the wire as AccessSource. +// +// A nil *FileAudit is a working no-op, so a runtime that never wired one up +// costs nothing and no call site needs a nil check. +type FileAudit struct { + bus *eventbus.Bus[*filepb.Access] + + // prefix makes ids unique across processes, seq within one. Together they + // let a consumer store an observation idempotently when a reconnect + // redelivers it. + prefix string + seq atomic.Uint64 + + queue chan *filepb.Access + done chan struct{} + stop sync.Once + + mu sync.RWMutex + config AuditOptions + dropped atomic.Uint64 +} + +const ( + // auditQueueDepth is how many observations may wait for the publisher. + // Deep enough that one snapshot diff of a busy build keeps its tail, + // shallow enough that a stalled consumer cannot hold the whole listing. + auditQueueDepth = 512 + + // DefaultAuditMaxEntries bounds one snapshot walk. A work dir larger than + // this is not diffed at all — see TakeSnapshot. + DefaultAuditMaxEntries = 20000 +) + +// DefaultAuditIgnore are the path segments a snapshot never walks: the +// directories where a build or a checkout produces thousands of changes that +// say nothing about what the agent was doing. +var DefaultAuditIgnore = []string{".git", "node_modules", ".cairn", "__pycache__", ".venv"} + +// AuditOptions is the snapshot and reporting policy. A peer sets it at runtime +// through the file namespace's Configure. +type AuditOptions struct { + Enabled bool + Ignore []string + MaxEntries int +} + +func defaultAuditOptions() AuditOptions { + return AuditOptions{Enabled: true, Ignore: DefaultAuditIgnore, MaxEntries: DefaultAuditMaxEntries} +} + +// NewFileAudit starts an audit trail. Nothing is published until something +// subscribes, and subscribers see every observation recorded after they attach. +func NewFileAudit() *FileAudit { + a := &FileAudit{ + bus: eventbus.New[*filepb.Access](), + prefix: rand.Text()[:8], + queue: make(chan *filepb.Access, auditQueueDepth), + done: make(chan struct{}), + config: defaultAuditOptions(), + } + go a.publish() + return a +} + +func (a *FileAudit) publish() { + defer close(a.done) + for access := range a.queue { + a.bus.Emit(access) + } +} + +// Subscribe delivers every subsequent observation to handler, returning the +// detach function. Handlers run on the audit's own goroutine, never on the one +// that performed the file access. +func (a *FileAudit) Subscribe(handler func(*filepb.Access)) func() { + if a == nil || handler == nil { + return func() {} + } + return a.bus.Subscribe(handler) +} + +// Configure applies a peer's watch policy. A nil config restores the defaults, +// which is what a peer that asked to observe with no opinion should get. +func (a *FileAudit) Configure(config *filepb.WatchConfig) { + if a == nil { + return + } + next := defaultAuditOptions() + if config != nil { + next.Enabled = config.GetEnabled() + if ignore := config.GetIgnore(); len(ignore) > 0 { + next.Ignore = ignore + } + if max := int(config.GetMaxEntries()); max > 0 { + next.MaxEntries = max + } + } + a.mu.Lock() + a.config = next + a.mu.Unlock() +} + +// State is the reply a peer gets to Configure. Dropped rides along so a +// consumer can say the trail has a hole in it rather than presenting a short +// history as a complete one. +func (a *FileAudit) State() *filepb.WatchState { + if a == nil { + return &filepb.WatchState{} + } + state := &filepb.WatchState{Watching: a.Options().Enabled} + if dropped := a.dropped.Load(); dropped > 0 { + state.Error = fmt.Sprintf("%d observations dropped: consumer too slow", dropped) + } + return state +} + +// Options returns the active policy. Callers about to do expensive work — a +// snapshot walk above all — check Enabled first. +func (a *FileAudit) Options() AuditOptions { + if a == nil { + return AuditOptions{} + } + a.mu.RLock() + defer a.mu.RUnlock() + return a.config +} + +// Enabled reports whether observations are currently collected. +func (a *FileAudit) Enabled() bool { return a.Options().Enabled } + +// Record publishes one observation. The caller supplies what it knows; identity, +// timing and the invocation context are filled in here so no call site has to +// remember them. It never blocks. +func (a *FileAudit) Record(ctx context.Context, access *filepb.Access) { + if a == nil || access == nil || !a.Enabled() { + return + } + invocation := coretool.InvocationFromContext(ctx) + if access.ToolId == "" { + access.ToolId = invocation.CallID + } + if access.WorkDir == "" { + access.WorkDir = invocation.WorkDir + } + if access.Id == "" { + access.Id = a.prefix + "-" + strconv.FormatUint(a.seq.Add(1), 36) + } + if access.Timestamp == nil { + access.Timestamp = timestamppb.New(time.Now()) + } + select { + case a.queue <- access: + default: + a.dropped.Add(1) + } +} + +// RecordFile reports one exact, tool-level access. The size is read from the +// file unless the caller already knows it. +func (a *FileAudit) RecordFile(ctx context.Context, op filepb.AccessOp, path string, access *filepb.Access) { + if a == nil || !a.Enabled() { + return + } + if access == nil { + access = &filepb.Access{} + } + access.Op = op + access.Path = path + if access.Source == filepb.AccessSource_ACCESS_SOURCE_UNSPECIFIED { + access.Source = filepb.AccessSource_ACCESS_SOURCE_TOOL + } + if access.Size == 0 { + if info, err := os.Stat(path); err == nil { + access.Size = info.Size() + } + } + a.Record(ctx, access) +} + +// Close stops the publisher and waits for queued observations to drain. +func (a *FileAudit) Close() { + if a == nil { + return + } + a.stop.Do(func() { + close(a.queue) + <-a.done + }) +} + +// AuditDigest is the content hash carried by a write observation. It is what +// lets a consumer tell a rewrite that changed nothing from one that changed +// everything, without storing either version. +func AuditDigest(content []byte) string { + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]) +} + +// --- snapshots: the only way a shell command's writes become visible --- + +// auditEntry is what a snapshot remembers about one file. Size and modification +// time together tell a rewrite from an untouched file, and cost one stat rather +// than a read of every byte in the work dir. +type auditEntry struct { + modTime int64 + size int64 +} + +// AuditSnapshot is a work dir's regular files at one moment, keyed by absolute +// path. +type AuditSnapshot map[string]auditEntry + +// AuditChange is one difference between two snapshots. +type AuditChange struct { + Path string + Op filepb.AccessOp + Size int64 +} + +var errSnapshotTooLarge = fmt.Errorf("snapshot limit reached") + +// TakeSnapshot walks root and records every regular file it is willing to look +// at. +// +// It returns an error rather than a partial listing when the tree exceeds +// MaxEntries: a diff against a truncated snapshot invents a deletion for every +// file that fell off the end, and a wrong audit trail is worse than an absent +// one. The caller reports the refusal instead. +func TakeSnapshot(root string, options AuditOptions) (AuditSnapshot, error) { + if root == "" { + return nil, fmt.Errorf("file audit: work dir is required") + } + max := options.MaxEntries + if max <= 0 { + max = DefaultAuditMaxEntries + } + ignore := options.Ignore + if len(ignore) == 0 { + ignore = DefaultAuditIgnore + } + + snapshot := make(AuditSnapshot) + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + // A permission hole somewhere in the tree must not cost the audit + // of everything beside it. + if entry != nil && entry.IsDir() { + return fs.SkipDir + } + return nil + } + if entry.IsDir() { + if path != root && auditIgnored(entry.Name(), ignore) { + return fs.SkipDir + } + return nil + } + // Sockets, devices and symlinks have no content this audit can speak + // about, and following a link would double-count its target. + if !entry.Type().IsRegular() || auditIgnored(entry.Name(), ignore) { + return nil + } + if info, infoErr := entry.Info(); infoErr == nil { + if len(snapshot) >= max { + return errSnapshotTooLarge + } + snapshot[path] = auditEntry{modTime: info.ModTime().UnixNano(), size: info.Size()} + } + return nil + }) + if err != nil { + if err == errSnapshotTooLarge { + return nil, fmt.Errorf("file audit: %s holds more than %d files", root, max) + } + return nil, err + } + return snapshot, nil +} + +func auditIgnored(name string, patterns []string) bool { + for _, pattern := range patterns { + if pattern != "" && strings.Contains(name, pattern) { + return true + } + } + return false +} + +// DiffSnapshots reports what happened to the work dir between two snapshots, +// sorted by path so the same pair always produces the same sequence. +// +// A file present in both is reported only when its size or modification time +// moved. That misses a rewrite that restored the previous bytes within the +// filesystem's timestamp resolution — the price of not hashing every file in +// the tree twice per command. +func DiffSnapshots(before, after AuditSnapshot) []AuditChange { + var changes []AuditChange + for path, now := range after { + previous, existed := before[path] + switch { + case !existed: + changes = append(changes, AuditChange{Path: path, Op: filepb.AccessOp_ACCESS_OP_CREATE, Size: now.size}) + case previous != now: + changes = append(changes, AuditChange{Path: path, Op: filepb.AccessOp_ACCESS_OP_WRITE, Size: now.size}) + } + } + for path := range before { + if _, survives := after[path]; !survives { + changes = append(changes, AuditChange{Path: path, Op: filepb.AccessOp_ACCESS_OP_DELETE}) + } + } + sort.Slice(changes, func(i, j int) bool { return changes[i].Path < changes[j].Path }) + return changes +} + +// Around brackets fn with two snapshots of workDir and records the difference +// as SNAPSHOT observations attributed to the invocation in ctx. +// +// What it cannot do is separate the command's own writes from anything else +// that changed the work dir while it ran — a detached session, a background +// build — which is exactly what AccessSource SNAPSHOT tells a consumer. +func (a *FileAudit) Around(ctx context.Context, workDir string, fn func() error) error { + if a == nil || workDir == "" || !a.Enabled() { + return fn() + } + options := a.Options() + before, err := TakeSnapshot(workDir, options) + if err != nil { + // Run the command regardless — auditing is never a reason not to do the + // work — but say plainly that this execution has no file record. + a.recordSnapshotError(ctx, workDir, err) + return fn() + } + runErr := fn() + after, err := TakeSnapshot(workDir, options) + if err != nil { + a.recordSnapshotError(ctx, workDir, err) + return runErr + } + for _, change := range DiffSnapshots(before, after) { + a.Record(ctx, &filepb.Access{ + Op: change.Op, + Source: filepb.AccessSource_ACCESS_SOURCE_SNAPSHOT, + Path: change.Path, + WorkDir: workDir, + Size: change.Size, + }) + } + return runErr +} + +// recordSnapshotError reports that an execution went unaudited. It is an +// observation in its own right: a consumer that sees it knows the trail has a +// hole here, rather than reading an empty diff as "nothing was touched". +func (a *FileAudit) recordSnapshotError(ctx context.Context, workDir string, cause error) { + a.Record(ctx, &filepb.Access{ + Source: filepb.AccessSource_ACCESS_SOURCE_SNAPSHOT, + Path: workDir, + WorkDir: workDir, + Error: cause.Error(), + }) +} diff --git a/pkg/commands/file_audit_test.go b/pkg/commands/file_audit_test.go new file mode 100644 index 00000000..f0074d44 --- /dev/null +++ b/pkg/commands/file_audit_test.go @@ -0,0 +1,365 @@ +package commands + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + filepb "github.com/chainreactors/aiscan/aop/file" + coretool "github.com/chainreactors/aiscan/core/tool" +) + +// collector drains an audit trail into a slice. Observations are published on +// the audit's own goroutine, so every assertion waits for the expected count +// rather than reading immediately after the call that produced it. +type collector struct { + mu sync.Mutex + accesses []*filepb.Access + detach func() +} + +func collect(t *testing.T, audit *FileAudit) *collector { + t.Helper() + c := &collector{} + c.detach = audit.Subscribe(func(access *filepb.Access) { + c.mu.Lock() + c.accesses = append(c.accesses, access) + c.mu.Unlock() + }) + t.Cleanup(func() { c.detach() }) + return c +} + +func (c *collector) wait(t *testing.T, count int) []*filepb.Access { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + c.mu.Lock() + got := append([]*filepb.Access(nil), c.accesses...) + c.mu.Unlock() + if len(got) >= count { + return got + } + if time.Now().After(deadline) { + t.Fatalf("expected %d observations, got %d", count, len(got)) + } + time.Sleep(5 * time.Millisecond) + } +} + +func (c *collector) byPath(t *testing.T, path string) *filepb.Access { + t.Helper() + c.mu.Lock() + defer c.mu.Unlock() + for _, access := range c.accesses { + if access.GetPath() == path { + return access + } + } + t.Fatalf("no observation for %s", path) + return nil +} + +func TestNilFileAuditIsANoOp(t *testing.T) { + var audit *FileAudit + // Every entry point must survive a runtime that never wired one up. + audit.Record(context.Background(), &filepb.Access{Path: "/tmp/x"}) + audit.RecordFile(context.Background(), filepb.AccessOp_ACCESS_OP_READ, "/tmp/x", nil) + audit.Configure(&filepb.WatchConfig{Enabled: true}) + if audit.Enabled() { + t.Fatal("a nil audit must not report itself as enabled") + } + ran := false + if err := audit.Around(context.Background(), t.TempDir(), func() error { ran = true; return nil }); err != nil { + t.Fatalf("Around: %v", err) + } + if !ran { + t.Fatal("Around must still run the work when nothing is observing") + } +} + +func TestRecordFillsIdentityAndInvocation(t *testing.T) { + audit := NewFileAudit() + defer audit.Close() + c := collect(t, audit) + + ctx := coretool.ContextWithInvocation(context.Background(), coretool.Invocation{CallID: "call-1", WorkDir: "/work"}) + audit.Record(ctx, &filepb.Access{Op: filepb.AccessOp_ACCESS_OP_READ, Path: "/work/a.txt"}) + + access := c.wait(t, 1)[0] + if access.GetToolId() != "call-1" { + t.Fatalf("tool id = %q, want the invocation's call id", access.GetToolId()) + } + if access.GetWorkDir() != "/work" { + t.Fatalf("work dir = %q", access.GetWorkDir()) + } + if access.GetId() == "" || access.GetTimestamp() == nil { + t.Fatalf("identity and timing must be filled in: %+v", access) + } +} + +func TestDisabledAuditRecordsNothing(t *testing.T) { + audit := NewFileAudit() + defer audit.Close() + c := collect(t, audit) + + audit.Configure(&filepb.WatchConfig{Enabled: false}) + audit.Record(context.Background(), &filepb.Access{Path: "/tmp/x"}) + if state := audit.State(); state.GetWatching() { + t.Fatal("state must report that observation is off") + } + + time.Sleep(50 * time.Millisecond) + c.mu.Lock() + defer c.mu.Unlock() + if len(c.accesses) != 0 { + t.Fatalf("expected nothing while disabled, got %d", len(c.accesses)) + } +} + +func TestSnapshotDiffReportsCreateWriteDelete(t *testing.T) { + dir := t.TempDir() + kept := filepath.Join(dir, "kept.txt") + doomed := filepath.Join(dir, "doomed.txt") + if err := os.WriteFile(kept, []byte("one"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(doomed, []byte("two"), 0o644); err != nil { + t.Fatal(err) + } + + options := defaultAuditOptions() + before, err := TakeSnapshot(dir, options) + if err != nil { + t.Fatalf("TakeSnapshot: %v", err) + } + + born := filepath.Join(dir, "born.txt") + if err := os.WriteFile(born, []byte("three"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(kept, []byte("one and a half"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Remove(doomed); err != nil { + t.Fatal(err) + } + + after, err := TakeSnapshot(dir, options) + if err != nil { + t.Fatalf("TakeSnapshot: %v", err) + } + + want := map[string]filepb.AccessOp{ + born: filepb.AccessOp_ACCESS_OP_CREATE, + kept: filepb.AccessOp_ACCESS_OP_WRITE, + doomed: filepb.AccessOp_ACCESS_OP_DELETE, + } + changes := DiffSnapshots(before, after) + if len(changes) != len(want) { + t.Fatalf("expected %d changes, got %+v", len(want), changes) + } + for _, change := range changes { + if want[change.Path] != change.Op { + t.Fatalf("%s: op = %v, want %v", change.Path, change.Op, want[change.Path]) + } + } +} + +func TestSnapshotSkipsIgnoredDirectories(t *testing.T) { + dir := t.TempDir() + noisy := filepath.Join(dir, "node_modules", "pkg") + if err := os.MkdirAll(noisy, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(noisy, "index.js"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("package main"), 0o644); err != nil { + t.Fatal(err) + } + + snapshot, err := TakeSnapshot(dir, defaultAuditOptions()) + if err != nil { + t.Fatalf("TakeSnapshot: %v", err) + } + if len(snapshot) != 1 { + t.Fatalf("expected only the source file, got %+v", snapshot) + } +} + +// A tree over the limit must not produce a snapshot at all: diffing against a +// truncated one invents a deletion for every file that fell off the end. +func TestSnapshotRefusesOversizedTree(t *testing.T) { + dir := t.TempDir() + for i := range 5 { + if err := os.WriteFile(filepath.Join(dir, string(rune('a'+i))+".txt"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + if _, err := TakeSnapshot(dir, AuditOptions{Enabled: true, MaxEntries: 3}); err == nil { + t.Fatal("expected a refusal, got a snapshot") + } +} + +// The refusal itself is an observation: a consumer must be able to tell "no +// record was kept" from "nothing was touched". +func TestAroundReportsAnUnauditedExecution(t *testing.T) { + audit := NewFileAudit() + defer audit.Close() + c := collect(t, audit) + audit.Configure(&filepb.WatchConfig{Enabled: true, MaxEntries: 1}) + + dir := t.TempDir() + for _, name := range []string{"a.txt", "b.txt", "c.txt"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + ran := false + if err := audit.Around(context.Background(), dir, func() error { ran = true; return nil }); err != nil { + t.Fatalf("Around: %v", err) + } + if !ran { + t.Fatal("a refused snapshot must not stop the work") + } + if access := c.wait(t, 1)[0]; access.GetError() == "" { + t.Fatalf("expected the observation to carry the reason, got %+v", access) + } +} + +func TestAroundAttributesShellWritesToTheCall(t *testing.T) { + audit := NewFileAudit() + defer audit.Close() + c := collect(t, audit) + + dir := t.TempDir() + ctx := coretool.ContextWithInvocation(context.Background(), coretool.Invocation{CallID: "call-7", WorkDir: dir}) + target := filepath.Join(dir, "out.txt") + if err := audit.Around(ctx, dir, func() error { + return os.WriteFile(target, []byte("produced by a shell command"), 0o644) + }); err != nil { + t.Fatalf("Around: %v", err) + } + + access := c.wait(t, 1)[0] + if access.GetPath() != target { + t.Fatalf("path = %q, want %q", access.GetPath(), target) + } + if access.GetOp() != filepb.AccessOp_ACCESS_OP_CREATE { + t.Fatalf("op = %v, want CREATE", access.GetOp()) + } + if access.GetSource() != filepb.AccessSource_ACCESS_SOURCE_SNAPSHOT { + t.Fatalf("source = %v, want SNAPSHOT", access.GetSource()) + } + if access.GetToolId() != "call-7" { + t.Fatalf("tool id = %q, want the call that ran the command", access.GetToolId()) + } +} + +func TestWriteToolRecordsCreateWriteAndEdit(t *testing.T) { + audit := NewFileAudit() + defer audit.Close() + c := collect(t, audit) + + dir := t.TempDir() + tool := NewWriteTool(dir).WithAudit(audit) + path := filepath.Join(dir, "note.txt") + ctx := context.Background() + + if _, err := tool.Execute(ctx, `{"path": "note.txt", "content": "first\n"}`); err != nil { + t.Fatalf("write: %v", err) + } + if _, err := tool.Execute(ctx, `{"path": "note.txt", "content": "second\n"}`); err != nil { + t.Fatalf("rewrite: %v", err) + } + if _, err := tool.Execute(ctx, `{"path": "note.txt", "edits": [{"old_text": "second", "new_text": "third"}]}`); err != nil { + t.Fatalf("edit: %v", err) + } + + got := c.wait(t, 3) + want := []filepb.AccessOp{ + filepb.AccessOp_ACCESS_OP_CREATE, + filepb.AccessOp_ACCESS_OP_WRITE, + filepb.AccessOp_ACCESS_OP_EDIT, + } + for i, op := range want { + if got[i].GetOp() != op { + t.Fatalf("observation %d: op = %v, want %v", i, got[i].GetOp(), op) + } + if got[i].GetPath() != path { + t.Fatalf("observation %d: path = %q", i, got[i].GetPath()) + } + if got[i].GetDigest() == "" { + t.Fatalf("observation %d: a write must carry its content digest", i) + } + } + if got[2].GetEdits() != 1 { + t.Fatalf("edit count = %d, want 1", got[2].GetEdits()) + } + if got[0].GetDigest() == got[1].GetDigest() { + t.Fatal("two different contents must not share a digest") + } +} + +func TestReadToolRecordsWhatTheModelReceived(t *testing.T) { + audit := NewFileAudit() + defer audit.Close() + c := collect(t, audit) + + dir := t.TempDir() + path := filepath.Join(dir, "data.txt") + if err := os.WriteFile(path, []byte("alpha\nbeta\n"), 0o644); err != nil { + t.Fatal(err) + } + + tool := NewReadTool(dir).WithAudit(audit) + if _, err := tool.Execute(context.Background(), `{"path": "data.txt"}`); err != nil { + t.Fatalf("read: %v", err) + } + + c.wait(t, 1) + access := c.byPath(t, path) + if access.GetOp() != filepb.AccessOp_ACCESS_OP_READ { + t.Fatalf("op = %v, want READ", access.GetOp()) + } + if access.GetSource() != filepb.AccessSource_ACCESS_SOURCE_TOOL { + t.Fatalf("source = %v, want TOOL", access.GetSource()) + } + if access.GetSize() != 11 { + t.Fatalf("size = %d, want the file's 11 bytes", access.GetSize()) + } + if access.GetBytes() == 0 { + t.Fatal("a read must report how much reached the model") + } +} + +// A read that never touched the filesystem must not appear as one: an embedded +// skill has no path an operator could open. +func TestReadToolDoesNotRecordVirtualReads(t *testing.T) { + audit := NewFileAudit() + defer audit.Close() + c := collect(t, audit) + + tool := NewReadTool(t.TempDir(), staticVirtualReader{"aiscan://skill": "content"}).WithAudit(audit) + if _, err := tool.Execute(context.Background(), `{"path": "aiscan://skill"}`); err != nil { + t.Fatalf("read: %v", err) + } + + time.Sleep(50 * time.Millisecond) + c.mu.Lock() + defer c.mu.Unlock() + if len(c.accesses) != 0 { + t.Fatalf("expected no observation, got %+v", c.accesses) + } +} + +type staticVirtualReader map[string]string + +func (r staticVirtualReader) ReadVirtual(path string) (string, bool, error) { + content, ok := r[path] + return content, ok, nil +} diff --git a/pkg/commands/read.go b/pkg/commands/read.go index a77d5f2f..b62df6fd 100644 --- a/pkg/commands/read.go +++ b/pkg/commands/read.go @@ -10,6 +10,7 @@ import ( "unicode/utf8" aop "github.com/chainreactors/aiscan/aop" + filepb "github.com/chainreactors/aiscan/aop/file" coretool "github.com/chainreactors/aiscan/core/tool" "github.com/chainreactors/aiscan/core/truncate" ) @@ -23,12 +24,35 @@ const ( type ReadTool struct { workDir string readers []VirtualFileReader + audit *FileAudit } type VirtualFileReader interface { ReadVirtual(path string) (content string, handled bool, err error) } +// WithAudit attaches the file-access audit trail. A nil recorder leaves the +// tool unobserved, which is what a runtime that never wired one up gets. +func (t *ReadTool) WithAudit(recorder *FileAudit) *ReadTool { + t.audit = recorder + return t +} + +// audited records a successful read. Virtual reads never reach here: an +// embedded skill is not a file on this machine, and reporting it as one would +// put paths in the trail that no operator can open. +func (t *ReadTool) audited(ctx context.Context, path string, size int64, result *coretool.Result, err error) (*coretool.Result, error) { + if err == nil { + t.audit.RecordFile(ctx, filepb.AccessOp_ACCESS_OP_READ, path, &filepb.Access{ + Size: size, + // What the model actually received, which is the number that + // matters when the file was paginated or clipped. + Bytes: int64(len(coretool.ResultText(result))), + }) + } + return result, err +} + func NewReadTool(workDir string, readers ...VirtualFileReader) *ReadTool { return &ReadTool{workDir: workDir, readers: readers} } @@ -84,14 +108,17 @@ func (t *ReadTool) Execute(ctx context.Context, arguments string) (*coretool.Res } if mime := detectImageMime(resolved); mime != "" { - return readImageFile(resolved, args.Path, mime, info.Size()) + result, err := readImageFile(resolved, args.Path, mime, info.Size()) + return t.audited(ctx, resolved, info.Size(), result, err) } if isBinaryFile(resolved) { - return coretool.TextResult(fmt.Sprintf("[binary file: %s (%d bytes)]", args.Path, info.Size())), nil + result := coretool.TextResult(fmt.Sprintf("[binary file: %s (%d bytes)]", args.Path, info.Size())) + return t.audited(ctx, resolved, info.Size(), result, nil) } - return t.readFileLines(resolved, args.Path, args.Offset, args.Limit) + result, err := t.readFileLines(resolved, args.Path, args.Offset, args.Limit) + return t.audited(ctx, resolved, info.Size(), result, err) } func (t *ReadTool) readFileLines(resolved, displayPath string, offset, limit int) (*coretool.Result, error) { diff --git a/pkg/commands/register.go b/pkg/commands/register.go index 4158edc4..291a76bb 100644 --- a/pkg/commands/register.go +++ b/pkg/commands/register.go @@ -14,7 +14,7 @@ func init() { } timeout := deps.BashTimeout if timeout <= 0 { - timeout = 300 + timeout = defaultTimeout } var readers []VirtualFileReader var globbers []VirtualGlobber @@ -22,14 +22,14 @@ func init() { readers = append(readers, deps.SkillStore) globbers = append(globbers, deps.SkillStore) } - reg.RegisterTool(NewReadTool(workDir, readers...)) - reg.RegisterTool(NewWriteTool(workDir)) + reg.RegisterTool(NewReadTool(workDir, readers...).WithAudit(deps.FileAudit)) + reg.RegisterTool(NewWriteTool(workDir).WithAudit(deps.FileAudit)) if deps.RunnerMode { reg.RegisterTool(NewListTool(workDir)) } reg.RegisterTool(NewGlobTool(workDir, globbers...)) - bash := NewBashTool(workDir, timeout).WithScannerProxy(deps.ScannerProxy) + bash := NewBashTool(workDir, timeout).WithScannerProxy(deps.ScannerProxy).WithScannerProxyCA(deps.ScannerProxyCA).WithEgressResolver(deps.EgressResolver).WithAudit(deps.FileAudit) bash.SetCommandNames(reg.Names) bash.SetCommandResolver(reg.Get) reg.RegisterTool(bash) diff --git a/pkg/commands/tmux.go b/pkg/commands/tmux.go index bad7e19a..15d6ecd5 100644 --- a/pkg/commands/tmux.go +++ b/pkg/commands/tmux.go @@ -150,7 +150,7 @@ func (t *tmuxCommand) cmdNewSession(ctx context.Context, args []string) (string, } func (t *tmuxCommand) createSession(ctx context.Context, cmdLine, name string, timeout time.Duration) (tmux.Info, error) { - execution, err := t.start(ctx, cmdLine, BashExecOptions{Name: name, Timeout: timeout}) + execution, err := t.start(ctx, cmdLine, BashExecOptions{Name: name, Timeout: timeout, TimeoutSet: true}) if err != nil { return tmux.Info{}, err } diff --git a/pkg/commands/write.go b/pkg/commands/write.go index 03937c2f..51d54e88 100644 --- a/pkg/commands/write.go +++ b/pkg/commands/write.go @@ -8,12 +8,21 @@ import ( "sort" "strings" + filepb "github.com/chainreactors/aiscan/aop/file" coretool "github.com/chainreactors/aiscan/core/tool" "github.com/chainreactors/aiscan/core/truncate" ) type WriteTool struct { workDir string + audit *FileAudit +} + +// WithAudit attaches the file-access audit trail. A nil recorder leaves the +// tool unobserved. +func (t *WriteTool) WithAudit(recorder *FileAudit) *WriteTool { + t.audit = recorder + return t } func NewWriteTool(workDir string) *WriteTool { @@ -61,13 +70,13 @@ func (t *WriteTool) Execute(ctx context.Context, arguments string) (*coretool.Re } if len(args.Edits) > 0 { - return t.editFile(args) + return t.editFile(ctx, args) } - return t.writeFile(args) + return t.writeFile(ctx, args) } -func (t *WriteTool) writeFile(args WriteArgs) (*coretool.Result, error) { +func (t *WriteTool) writeFile(ctx context.Context, args WriteArgs) (*coretool.Result, error) { path := t.resolvePath(args.Path) dir := filepath.Dir(path) @@ -75,10 +84,24 @@ func (t *WriteTool) writeFile(args WriteArgs) (*coretool.Result, error) { return nil, fmt.Errorf("create directory: %w", err) } + // Whether the path existed decides CREATE vs WRITE, and it can only be + // asked before the write. + _, existed := os.Stat(path) + if err := os.WriteFile(path, []byte(args.Content), 0644); err != nil { return nil, fmt.Errorf("write file: %w", err) } + op := filepb.AccessOp_ACCESS_OP_WRITE + if existed != nil { + op = filepb.AccessOp_ACCESS_OP_CREATE + } + t.audit.RecordFile(ctx, op, path, &filepb.Access{ + Size: int64(len(args.Content)), + Bytes: int64(len(args.Content)), + Digest: AuditDigest([]byte(args.Content)), + }) + lineCount := strings.Count(args.Content, "\n") + 1 return coretool.TextResult(fmt.Sprintf("wrote %d bytes (%d lines) to %s", len(args.Content), lineCount, args.Path)), nil } @@ -90,7 +113,7 @@ type editMatch struct { newText string } -func (t *WriteTool) editFile(args WriteArgs) (*coretool.Result, error) { +func (t *WriteTool) editFile(ctx context.Context, args WriteArgs) (*coretool.Result, error) { path := t.resolvePath(args.Path) data, err := os.ReadFile(path) @@ -206,6 +229,15 @@ func (t *WriteTool) editFile(args WriteArgs) (*coretool.Result, error) { return nil, fmt.Errorf("write edited file: %w", err) } + // EDIT rather than WRITE: the patch count is what distinguishes a targeted + // change from a file the agent replaced wholesale. + t.audit.RecordFile(ctx, filepb.AccessOp_ACCESS_OP_EDIT, path, &filepb.Access{ + Size: int64(len(result)), + Bytes: int64(len(result)), + Edits: uint32(len(args.Edits)), + Digest: AuditDigest([]byte(result)), + }) + // Build summary var summary strings.Builder fmt.Fprintf(&summary, "edited %s: %d edit(s) applied", args.Path, len(args.Edits)) diff --git a/pkg/headless/hijack.go b/pkg/headless/hijack.go index 8224821e..585d4321 100644 --- a/pkg/headless/hijack.go +++ b/pkg/headless/hijack.go @@ -75,6 +75,9 @@ func (h *Hijack) Stop() error { // FetchGetResponseBody retrieves the response body for an intercepted request. func FetchGetResponseBody(page *rod.Page, e *proto.FetchRequestPaused) ([]byte, error) { + page = page.Timeout(defaultActionTimeout) + defer page.CancelTimeout() + m := proto.FetchGetResponseBody{RequestID: e.RequestID} r, err := m.Call(page) if err != nil { @@ -88,6 +91,9 @@ func FetchGetResponseBody(page *rod.Page, e *proto.FetchRequestPaused) ([]byte, // FetchContinueRequest continues a paused request without modification. func FetchContinueRequest(page *rod.Page, e *proto.FetchRequestPaused) error { + page = page.Timeout(defaultActionTimeout) + defer page.CancelTimeout() + m := proto.FetchContinueRequest{RequestID: e.RequestID} return m.Call(page) } diff --git a/pkg/headless/page.go b/pkg/headless/page.go index 620395b9..c3cc938c 100644 --- a/pkg/headless/page.go +++ b/pkg/headless/page.go @@ -306,9 +306,8 @@ func (p *Page) setupNativeHijack() { URLPattern: "*", RequestStage: proto.FetchRequestStageResponse, }) - go func() { - _ = hijack.Start(p.routingRuleHandlerNative)() - }() + wait := hijack.Start(p.routingRuleHandlerNative) + go func() { _ = wait() }() p.hijackNative = hijack } diff --git a/pkg/node/connection.go b/pkg/node/connection.go index 3c287ec4..c17015b2 100644 --- a/pkg/node/connection.go +++ b/pkg/node/connection.go @@ -38,7 +38,13 @@ type connectionConfig struct { Status func() *aop.AgentStatus Menu func() []*types.CommandSpec RunnerFileRPC bool - PTYRouter func() (*terminal.Router, error) + // FileAudit is the node's file-access trail, streamed on the file namespace + // and steerable by the peer through Configure. + FileAudit *commands.FileAudit + PTYRouter func() (*terminal.Router, error) + // ExtraNamespaces registers additional AOP namespaces on the connection mux + // after the built-ins (see ToolNodeConfig.ExtraNamespaces). + ExtraNamespaces []func(*aop.NamespaceMux) error } func connect(ctx context.Context, config connectionConfig) error { diff --git a/pkg/node/file_audit.go b/pkg/node/file_audit.go new file mode 100644 index 00000000..10331fee --- /dev/null +++ b/pkg/node/file_audit.go @@ -0,0 +1,34 @@ +package node + +import ( + "context" + + filepb "github.com/chainreactors/aiscan/aop/file" + "github.com/chainreactors/aiscan/pkg/commands" + protobuf "google.golang.org/protobuf/proto" +) + +func attachFileAccess(audit *commands.FileAudit, send func(string, protobuf.Message)) func() { + if audit == nil { + return nil + } + return audit.Subscribe(func(access *filepb.Access) { + if access == nil { + return + } + send(access.GetToolId(), &filepb.ProtocolMessage{ + Message: &filepb.ProtocolMessage_Access{Access: protobuf.CloneOf(access)}, + }) + }) +} + +func auditControlAccess(audit *commands.FileAudit, op filepb.AccessOp, base, path string, value fileResultValue) { + if audit == nil || value.err != nil || path == "" { + return + } + audit.RecordFile(context.Background(), op, resolveFileRPCPath(base, path), &filepb.Access{ + Source: filepb.AccessSource_ACCESS_SOURCE_CONTROL, + WorkDir: base, + Bytes: int64(len(value.result.GetData())), + }) +} diff --git a/pkg/node/file_audit_test.go b/pkg/node/file_audit_test.go new file mode 100644 index 00000000..901f79a9 --- /dev/null +++ b/pkg/node/file_audit_test.go @@ -0,0 +1,164 @@ +package node + +import ( + "context" + "io" + "testing" + "time" + + aop "github.com/chainreactors/aiscan/aop" + filepb "github.com/chainreactors/aiscan/aop/file" + "github.com/chainreactors/aiscan/core/telemetry" + "github.com/chainreactors/aiscan/pkg/commands" +) + +// fileAuditStream drives one connection through the handshake, delivers a +// Configure asking for observation, records an access, then holds the stream +// open long enough for the observation to be written. +type fileAuditStream struct { + helloID string + recvs int + sent chan *aop.Envelope + audit *commands.FileAudit +} + +func (s *fileAuditStream) Send(envelope *aop.Envelope) error { + if s.helloID == "" { + s.helloID = envelope.GetId() + } + select { + case s.sent <- envelope: + default: + } + return nil +} + +func (s *fileAuditStream) Recv() (*aop.Envelope, error) { + s.recvs++ + switch s.recvs { + case 1: + return aop.MustWrap("accepted", s.helloID, &aop.ProtocolMessage{ + Message: &aop.ProtocolMessage_AgentAccepted{AgentAccepted: &aop.AgentAccepted{NodeId: "runner-1"}}, + }), nil + case 2: + return aop.MustWrap("configure-1", "", &filepb.ProtocolMessage{ + Message: &filepb.ProtocolMessage_Configure{Configure: &filepb.Configure{ + Watch: &filepb.WatchConfig{Enabled: true}, + }}, + }), nil + case 3: + // A tool touched a file while the connection was up. + s.audit.Record(context.Background(), &filepb.Access{ + ToolId: "call-1", + Op: filepb.AccessOp_ACCESS_OP_EDIT, + Source: filepb.AccessSource_ACCESS_SOURCE_TOOL, + Path: "/root/work/main.go", + }) + } + time.Sleep(200 * time.Millisecond) + return nil, io.EOF +} + +// The file namespace carries the audit trail as well as its RPCs. This pins the +// whole path open: a peer asks for observation, the node answers with its watch +// state, and every access recorded afterwards reaches the wire addressed to the +// tool call that produced it. +func TestFileAuditReachesTheWire(t *testing.T) { + audit := commands.NewFileAudit() + defer audit.Close() + + stream := &fileAuditStream{sent: make(chan *aop.Envelope, 32), audit: audit} + cc := connectionConfig{ + Name: "runner-1", + NodeID: "runner-1", + Registry: commands.NewRegistry(), + FileAudit: audit, + } + + if err := serveAgentConnection(context.Background(), cc, telemetry.NopLogger(), stream); err != io.EOF { + t.Fatalf("serveAgentConnection error = %v, want EOF", err) + } + + var sawState, sawAccess bool + for { + select { + case envelope := <-stream.sent: + message, err := aop.Unwrap(envelope) + if err != nil { + continue + } + value, ok := message.(*filepb.ProtocolMessage) + if !ok { + continue + } + if state := value.GetState(); state != nil { + if !state.GetWatching() { + t.Fatal("the node answered Configure by saying it is not watching") + } + if envelope.GetReplyTo() != "configure-1" { + t.Fatalf("state reply_to = %q, want configure-1", envelope.GetReplyTo()) + } + sawState = true + } + if access := value.GetAccess(); access != nil { + if access.GetPath() != "/root/work/main.go" { + t.Fatalf("path = %q", access.GetPath()) + } + // Addressed to the call that produced it, so a consumer can + // attribute the observation without a second lookup. + if envelope.GetReplyTo() != "call-1" { + t.Fatalf("access reply_to = %q, want call-1", envelope.GetReplyTo()) + } + if access.GetId() == "" || access.GetTimestamp() == nil { + t.Fatalf("the audit must stamp identity and timing: %+v", access) + } + sawAccess = true + } + default: + if !sawState { + t.Fatal("the watch state never reached the wire") + } + if !sawAccess { + t.Fatal("the recorded access never reached the wire") + } + return + } + } +} + +// Without an audit the namespace still answers, so a peer learns that this node +// reports nothing rather than waiting for a stream that will never start. +func TestFileConfigureWithoutAnAuditStillAnswers(t *testing.T) { + stream := &fileAuditStream{sent: make(chan *aop.Envelope, 32), audit: commands.NewFileAudit()} + defer stream.audit.Close() + cc := connectionConfig{Name: "runner-1", NodeID: "runner-1", Registry: commands.NewRegistry()} + + if err := serveAgentConnection(context.Background(), cc, telemetry.NopLogger(), stream); err != io.EOF { + t.Fatalf("serveAgentConnection error = %v, want EOF", err) + } + + for { + select { + case envelope := <-stream.sent: + message, err := aop.Unwrap(envelope) + if err != nil { + continue + } + value, ok := message.(*filepb.ProtocolMessage) + if !ok { + continue + } + if state := value.GetState(); state != nil { + if state.GetWatching() { + t.Fatal("a node with no audit must not claim to be watching") + } + return + } + if value.GetAccess() != nil { + t.Fatal("a node with no audit must not stream observations") + } + default: + t.Fatal("the watch state never reached the wire") + } + } +} diff --git a/pkg/node/namespace_reply.go b/pkg/node/namespace_reply.go new file mode 100644 index 00000000..6e497ed9 --- /dev/null +++ b/pkg/node/namespace_reply.go @@ -0,0 +1,15 @@ +package node + +import aop "github.com/chainreactors/aiscan/aop" + +func registerExtraNamespaces(mux *aop.NamespaceMux, registrars []func(*aop.NamespaceMux) error) error { + for _, register := range registrars { + if register == nil { + continue + } + if err := register(mux); err != nil { + return err + } + } + return nil +} diff --git a/pkg/node/namespace_reply_test.go b/pkg/node/namespace_reply_test.go new file mode 100644 index 00000000..d5b85031 --- /dev/null +++ b/pkg/node/namespace_reply_test.go @@ -0,0 +1,115 @@ +package node + +import ( + "context" + "io" + "testing" + "time" + + aop "github.com/chainreactors/aiscan/aop" + trafficpb "github.com/chainreactors/aiscan/aop/traffic" + "github.com/chainreactors/aiscan/core/telemetry" + "github.com/chainreactors/aiscan/pkg/commands" + protobuf "google.golang.org/protobuf/proto" +) + +// namespaceReplyStream drives one connection through the handshake, delivers a +// single message addressed to an extra namespace, then blocks until the +// connection has written something back before ending the stream. Everything +// the connection sends is collected so a test can assert what reached the wire. +type namespaceReplyStream struct { + helloID string + recvs int + sent chan *aop.Envelope + payload *aop.Envelope +} + +func (s *namespaceReplyStream) Send(envelope *aop.Envelope) error { + if s.helloID == "" { + s.helloID = envelope.GetId() + } + select { + case s.sent <- envelope: + default: + } + return nil +} + +func (s *namespaceReplyStream) Recv() (*aop.Envelope, error) { + s.recvs++ + switch s.recvs { + case 1: + return aop.MustWrap("accepted", s.helloID, &aop.ProtocolMessage{ + Message: &aop.ProtocolMessage_AgentAccepted{AgentAccepted: &aop.AgentAccepted{NodeId: "runner-1"}}, + }), nil + case 2: + return s.payload, nil + } + // Hold the stream open long enough for the handler's reply to be written. + time.Sleep(200 * time.Millisecond) + return nil, io.EOF +} + +// An ExtraNamespaces handler can only answer through the SendFunc the dispatch +// hands it: unlike the built-in namespaces, it has no closure over the +// connection's own sender. That argument used to be a discard, so a host +// namespace could be registered, could report success, and could never reply or +// stream — a runner whose capture was "on" looked exactly like a quiet target. +// This pins the reply path open. +func TestExtraNamespaceRepliesReachTheWire(t *testing.T) { + stream := &namespaceReplyStream{ + sent: make(chan *aop.Envelope, 16), + payload: aop.MustWrap("query-1", "", &trafficpb.ProtocolMessage{ + Message: &trafficpb.ProtocolMessage_Query{Query: &trafficpb.Query{State: true}}, + }), + } + + cc := connectionConfig{ + Name: "runner-1", + NodeID: "runner-1", + Registry: commands.NewRegistry(), + ExtraNamespaces: []func(*aop.NamespaceMux) error{ + func(mux *aop.NamespaceMux) error { + return mux.Register(&trafficpb.ProtocolMessage{}, func( + _ context.Context, envelope *aop.Envelope, _ protobuf.Message, send aop.SendFunc, + ) error { + reply, err := aop.Wrap("reply-1", envelope.GetId(), &trafficpb.ProtocolMessage{ + Message: &trafficpb.ProtocolMessage_State{State: &trafficpb.State{ + Capture: &trafficpb.CaptureState{Capturing: true}, + }}, + }) + if err != nil { + return err + } + return send(reply) + }) + }, + }, + } + + if err := serveAgentConnection(context.Background(), cc, telemetry.NopLogger(), stream); err != io.EOF { + t.Fatalf("serveAgentConnection error = %v, want EOF", err) + } + + for { + select { + case envelope := <-stream.sent: + message, err := aop.Unwrap(envelope) + if err != nil { + continue + } + value, ok := message.(*trafficpb.ProtocolMessage) + if !ok { + continue + } + if value.GetState().GetCapture().GetCapturing() { + if envelope.GetReplyTo() != "query-1" { + t.Fatalf("reply_to = %q, want query-1", envelope.GetReplyTo()) + } + return + } + default: + t.Fatal("the namespace handler's reply never reached the wire") + } + } +} diff --git a/pkg/node/proto_connection.go b/pkg/node/proto_connection.go index b3a69c5c..50eeb809 100644 --- a/pkg/node/proto_connection.go +++ b/pkg/node/proto_connection.go @@ -325,6 +325,9 @@ func serveAgentConnection(ctx context.Context, cc connectionConfig, logger telem if detach := attachToolProgress(cc.Progress, send); detach != nil { defer detach() } + if detach := attachFileAccess(cc.FileAudit, send); detach != nil { + defer detach() + } // The catalog is the first post-handshake message the hub treats as a // readiness signal. Attach event and progress subscribers before publishing // it so callers cannot emit into the small acceptance-to-subscribe gap. @@ -386,6 +389,14 @@ func serveAgentConnection(ctx context.Context, cc connectionConfig, logger telem if err != nil { return fmt.Errorf("register connection namespaces: %w", err) } + // The reply path handed to every namespace handler. The built-in namespaces + // close over sendEnvelope directly and ignore this argument, which is why it + // could be a discard for so long; an ExtraNamespaces handler has no such + // closure and can only answer — or stream — through here. + reply := func(envelope *aop.Envelope) error { + sendEnvelope(envelope) + return nil + } for { envelope, err := stream.Recv() if err != nil { @@ -396,7 +407,7 @@ func serveAgentConnection(ctx context.Context, cc connectionConfig, logger telem } return err } - handled, err := namespaceMux.Dispatch(connectionCtx, envelope, func(*aop.Envelope) error { return nil }) + handled, err := namespaceMux.Dispatch(connectionCtx, envelope, reply) if err != nil { send(envelope.GetId(), protocolFailure("INVALID_PAYLOAD", err.Error())) continue @@ -494,6 +505,9 @@ func newAgentConnectionNamespaceMux( }); err != nil { return nil, err } + if err := registerExtraNamespaces(mux, cc.ExtraNamespaces); err != nil { + return nil, err + } return mux, nil } @@ -595,9 +609,19 @@ func handleAgentFileMessage(cc connectionConfig, envelope *aop.Envelope, value * fail := func(message string) { send(replyTo, protocolFailure("OPERATION_FAILED", message)) } switch payload := value.Message.(type) { case *filepb.ProtocolMessage_ReadRequest: - go sendFileResult(replyTo, fileRead(payload.ReadRequest, workingDir(cc.Runtime)), send) + go func() { + base := workingDir(cc.Runtime) + value := fileRead(payload.ReadRequest, base) + sendFileResult(replyTo, value, send) + auditControlAccess(cc.FileAudit, filepb.AccessOp_ACCESS_OP_READ, base, payload.ReadRequest.GetPath(), value) + }() case *filepb.ProtocolMessage_WriteRequest: - go sendFileResult(replyTo, fileWrite(payload.WriteRequest, workingDir(cc.Runtime)), send) + go func() { + base := workingDir(cc.Runtime) + value := fileWrite(payload.WriteRequest, base) + sendFileResult(replyTo, value, send) + auditControlAccess(cc.FileAudit, filepb.AccessOp_ACCESS_OP_WRITE, base, payload.WriteRequest.GetPath(), value) + }() case *filepb.ProtocolMessage_ListRequest: if !cc.RunnerFileRPC { fail("file list is unavailable") @@ -610,6 +634,9 @@ func handleAgentFileMessage(cc connectionConfig, envelope *aop.Envelope, value * return } go sendFileResult(replyTo, fileMkdir(payload.MkdirRequest, workingDir(cc.Runtime)), send) + case *filepb.ProtocolMessage_Configure: + cc.FileAudit.Configure(payload.Configure.GetWatch()) + send(replyTo, &filepb.ProtocolMessage{Message: &filepb.ProtocolMessage_State{State: cc.FileAudit.State()}}) case *filepb.ProtocolMessage_UploadRequest: go func() { if cc.Chat == nil { diff --git a/pkg/node/toolnode.go b/pkg/node/toolnode.go index b1bdc948..22000766 100644 --- a/pkg/node/toolnode.go +++ b/pkg/node/toolnode.go @@ -41,6 +41,13 @@ type ToolNodeConfig struct { // DisableCommandCatalog prevents the AIScan-specific command namespace from // being sent to generic AOP hubs. Tool definitions remain in AgentHello. DisableCommandCatalog bool + // FileAudit is the file-access trail the registry's tools report into. When + // set, every observation is streamed to the hub on the file namespace. + FileAudit *commands.FileAudit + // ExtraNamespaces lets a host register additional AOP namespaces on the + // connection mux (e.g. the traffic namespace backed by the host's proxy + // hub). Each registrar is applied after the built-in namespaces. + ExtraNamespaces []func(*aop.NamespaceMux) error } // RunToolNode connects to the hub as a tool-only node and serves until ctx is @@ -82,20 +89,22 @@ func RunToolNode(ctx context.Context, cfg ToolNodeConfig) error { subscribe = cfg.Events.Subscribe } return connect(ctx, connectionConfig{ - ServerURL: cfg.ServerURL, - WSPath: cfg.WSPath, - Name: runnerID, - Token: cfg.Token, - Registry: cfg.Registry, - AgentSubscribe: subscribe, - Progress: cfg.Progress, - Logger: logger, - NodeID: runnerID, - Runtime: runnerRuntime, - Capabilities: []string{"pty", "file", "exec", "tool", "artifact"}, - Menu: menu, - RunnerFileRPC: true, - JSONFrames: cfg.JSONFrames, + ServerURL: cfg.ServerURL, + WSPath: cfg.WSPath, + Name: runnerID, + Token: cfg.Token, + Registry: cfg.Registry, + AgentSubscribe: subscribe, + Progress: cfg.Progress, + Logger: logger, + NodeID: runnerID, + Runtime: runnerRuntime, + Capabilities: []string{"pty", "file", "exec", "tool", "artifact"}, + Menu: menu, + RunnerFileRPC: true, + FileAudit: cfg.FileAudit, + JSONFrames: cfg.JSONFrames, + ExtraNamespaces: cfg.ExtraNamespaces, }) } diff --git a/pkg/runner/app.go b/pkg/runner/app.go index b6a552de..5c00cb9d 100644 --- a/pkg/runner/app.go +++ b/pkg/runner/app.go @@ -17,11 +17,11 @@ import ( "github.com/chainreactors/aiscan/core/eventbus" "github.com/chainreactors/aiscan/core/output" "github.com/chainreactors/aiscan/core/telemetry" - "github.com/chainreactors/aiscan/core/truncate" "github.com/chainreactors/aiscan/pkg/commands" types "github.com/chainreactors/aiscan/pkg/types" "github.com/chainreactors/aiscan/skills" ioatools "github.com/chainreactors/aiscan/tools/ioa" + proxytool "github.com/chainreactors/aiscan/tools/proxy" ioaclient "github.com/chainreactors/ioa/client" "github.com/chainreactors/ioa/protocols" ) @@ -37,18 +37,26 @@ type App struct { SkillDiagnostics []skills.Diagnostic IOAClient *ioaclient.Client IOAStreamClient ioaclient.StreamAPI - EventBus *eventbus.Bus[*aop.Event] - Events *sessionEmitter - Progress *eventbus.Bus[*toolpb.Progress] - Recorder *output.JSONLRecorder - recorderMu sync.Mutex - closeOnce sync.Once - enginesReady chan struct{} - enginesEnabled bool - healthMu sync.RWMutex - llmHealth LLMHealth - loggerMu sync.RWMutex - logger telemetry.Logger + // FileAudit is the trail the file tools and shell executions report into. + // It belongs to the application rather than any one transport, so a local + // run and a remote tool node observe the same thing. + FileAudit *commands.FileAudit + EventBus *eventbus.Bus[*aop.Event] + Events *sessionEmitter + Progress *eventbus.Bus[*toolpb.Progress] + Recorder *output.JSONLRecorder + deps *commands.Deps + proxyInfra *proxytool.Infra + cancel context.CancelFunc + assemblyMu sync.Mutex + recorderMu sync.Mutex + closeOnce sync.Once + enginesReady chan struct{} + enginesEnabled bool + healthMu sync.RWMutex + llmHealth LLMHealth + loggerMu sync.RWMutex + logger telemetry.Logger } // LLMHealth is the latest lightweight provider connectivity check. It is kept @@ -69,7 +77,17 @@ const ( ) func NewApp(ctx context.Context, rc ApplicationConfig) (*App, error) { - a := &App{} + if ctx == nil { + ctx = context.Background() + } + appCtx, cancel := context.WithCancel(ctx) + a := &App{cancel: cancel} + ready := false + defer func() { + if !ready { + cancel() + } + }() logger := rc.Logger if logger == nil { logger = telemetry.NopLogger() @@ -127,7 +145,8 @@ func NewApp(ctx context.Context, rc ApplicationConfig) (*App, error) { a.setLLMHealth(LLMHealth{State: LLMHealthNotConfigured}) } - a.Commands = initCoreCommands(rc, a.Provider, a.Skills, a.Hooks, a.Events, logger) + a.FileAudit = commands.NewFileAudit() + a.initCommands(rc, logger) if rc.RecordFile != "" { if err := a.StartRecording(rc.RecordFile); err != nil { a.Close() @@ -136,21 +155,22 @@ func NewApp(ctx context.Context, rc ApplicationConfig) (*App, error) { } a.enginesReady = make(chan struct{}) - a.enginesEnabled = ScannerInitFunc != nil && !rc.SkipEngines + a.enginesEnabled = !rc.SkipEngines go func() { if a.enginesEnabled { - ScannerInitFunc(ctx, a, rc, logger) + a.initScanner(appCtx, rc, logger) } close(a.enginesReady) }() if rc.IOA != nil { - if err := a.InitIOA(ctx, *rc.IOA); err != nil { + if err := a.InitIOA(appCtx, *rc.IOA); err != nil { a.Close() return nil, err } } + ready = true return a, nil } @@ -237,6 +257,12 @@ func (a *App) Close() { return } a.closeOnce.Do(func() { + if a.cancel != nil { + a.cancel() + } + if a.enginesReady != nil { + <-a.enginesReady + } a.recorderMu.Lock() if a.Recorder != nil { if err := a.Recorder.Close(); err != nil { @@ -260,6 +286,12 @@ func (a *App) Close() { if closer, ok := a.Engines.(interface{ Close() }); ok { closer.Close() } + if a.proxyInfra != nil && a.proxyInfra.Hub != nil { + a.proxyInfra.Hub.Shutdown(context.Background()) + } + if a.FileAudit != nil { + a.FileAudit.Close() + } }) } @@ -364,34 +396,43 @@ func llmConfigLabel(providerName, model string) string { return providerName + "/" + model } -func initCoreCommands(rc ApplicationConfig, llmProvider agent.Provider, skillStore *skills.Store, hookRegistry *hooks.Registry, events aop.EventEmitter, logger telemetry.Logger) *commands.CommandRegistry { - cmdReg := commands.NewRegistry() +func (a *App) initCommands(rc ApplicationConfig, logger telemetry.Logger) { + a.Commands = commands.NewRegistry() workDir, _ := os.Getwd() - deps := &commands.Deps{ + a.deps = &commands.Deps{ WorkDir: workDir, + RunnerMode: rc.Tools.RunnerMode, BashTimeout: rc.Tools.BashTimeout, - SkillStore: skillStore, - Provider: llmProvider, + SkillStore: a.Skills, + Provider: a.Provider, + ScannerProxy: rc.Scanner.Proxy, Logger: logger, TavilyKeys: rc.Tools.TavilyKeys, PlaywrightSession: rc.Tools.PlaywrightSession, - Hooks: hookRegistry, - Events: events, + Hooks: a.Hooks, + Events: a.Events, + FileAudit: a.FileAudit, } + var err error + a.proxyInfra, err = proxytool.InstallInfra(a.deps, captureEnabled(rc.Tools.MitmCapture)) + if err != nil { + logger.Warnf("proxy hub unavailable, tools use direct/original proxy: %s", err) + } + plan := capability.Select(capability.Options{ - Groups: linkedToolGroups(), + Groups: linkedBaseGroups(), OptionalTools: rc.Tools.OptionalTools, }) - commands.BuildPlan(plan, deps, cmdReg) - cmdReg.SetLogger(logger) - return cmdReg + commands.BuildPlan(plan, a.deps, a.Commands) + a.Commands.SetLogger(logger) } -func linkedToolGroups() []string { +func linkedBaseGroups() []string { seen := make(map[string]bool) var groups []string for _, descriptor := range capability.All() { - if descriptor.Kind != capability.KindTool || descriptor.Group == "" || seen[descriptor.Group] { + baseService := descriptor.Kind == capability.KindService && len(descriptor.Requires) == 0 + if (descriptor.Kind != capability.KindTool && !baseService) || descriptor.Group == "" || seen[descriptor.Group] { continue } seen[descriptor.Group] = true @@ -400,62 +441,16 @@ func linkedToolGroups() []string { return groups } -func executeRegistryCommand(ctx context.Context, reg *commands.CommandRegistry, commandLine string, timeout time.Duration) (string, error) { - tool, ok := reg.GetTool("bash") - if !ok { - return "", fmt.Errorf("bash tool is not registered") - } - bash, ok := tool.(*commands.BashTool) - if !ok { - return "", fmt.Errorf("registered bash tool has unexpected type") - } - var output strings.Builder - execution, err := bash.RunForeground(ctx, commandLine, commands.BashExecOptions{ - Timeout: timeout, - OnOutput: func(data []byte) { _, _ = output.Write(data) }, - }) - if err != nil { - return output.String(), err - } - if execution.ExitCode != 0 { - return output.String(), fmt.Errorf("command exited with code %d", execution.ExitCode) - } - return output.String(), nil -} - -func appendDeepBrowserStep(sb *strings.Builder, name, commandLine, output string, err error) { - sb.WriteString("\n## ") - sb.WriteString(name) - sb.WriteString("\nCommand: `") - sb.WriteString(commandLine) - sb.WriteString("`\n") - if err != nil { - sb.WriteString("Error: ") - sb.WriteString(err.Error()) - sb.WriteString("\n") - } - output = strings.TrimSpace(output) - if output != "" { - if tr := truncate.Head(output, truncate.Options{}); tr.Truncated { - sb.WriteString(tr.Content) - sb.WriteString(fmt.Sprintf("\n[step truncated: %d/%d lines]", tr.OutputLines, tr.TotalLines)) - } else { - sb.WriteString(tr.Content) - } - sb.WriteString("\n") - } +func captureEnabled(configured *bool) bool { + return configured == nil || *configured } -func quoteCommandArg(value string) string { - if value == "" { - return `""` - } - if !strings.ContainsAny(value, " \t\r\n'\"\\") { - return value +// RegisterTrafficNamespace exposes the application's single proxy hub over AOP. +func (a *App) RegisterTrafficNamespace(mux *aop.NamespaceMux) error { + if a == nil || a.proxyInfra == nil || a.proxyInfra.Hub == nil { + return nil } - value = strings.ReplaceAll(value, `\`, `\\`) - value = strings.ReplaceAll(value, `"`, `\"`) - return `"` + value + `"` + return proxytool.NewTrafficHandler(a.proxyInfra).Register(mux) } func (a *App) InitIOA(ctx context.Context, ioa IOAConfig) error { @@ -473,13 +468,13 @@ func (a *App) InitIOA(ctx context.Context, ioa IOAConfig) error { } } a.IOAStreamClient = client - if ioa.RegisterTools && a.Commands != nil { - deps := &commands.Deps{ - NodeName: ioa.NodeName, - NodeMeta: ioa.NodeMeta, - } - commands.Provide(deps, ioatools.ClientKey, protocols.ClientAPI(client)) - commands.BuildPlan(capability.Select(capability.Options{Groups: []string{"ioa"}}), deps, a.Commands) + if ioa.RegisterTools && a.Commands != nil && a.deps != nil { + a.assemblyMu.Lock() + a.deps.NodeName = ioa.NodeName + a.deps.NodeMeta = ioa.NodeMeta + commands.Provide(a.deps, ioatools.ClientKey, protocols.ClientAPI(client)) + commands.BuildPlan(capability.Select(capability.Options{Groups: []string{"ioa"}}), a.deps, a.Commands) + a.assemblyMu.Unlock() } if ioa.AutoRegister { if err := client.EnsureRegistered(ctx, ioa.NodeName, "", ioa.NodeMeta); err != nil { @@ -533,76 +528,3 @@ func newIOAClient(ioa IOAConfig) (*ioaclient.Client, error) { } return ioaclient.NewClient(ioa.URL, ioa.NodeID) } - -func CollectDeepBrowserArtifacts(ctx context.Context, reg *commands.CommandRegistry, targetURL string, logger telemetry.Logger) (string, error) { - if reg == nil || !reg.Has("playwright") { - return "", fmt.Errorf("playwright command unavailable; rebuild web with browser tag") - } - targetURL = strings.TrimSpace(targetURL) - if targetURL == "" { - return "", fmt.Errorf("target URL is empty") - } - - session := fmt.Sprintf("deep%d", time.Now().UnixNano()) - closed := false - defer func() { - if closed { - return - } - closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, _ = executeRegistryCommand(closeCtx, reg, "playwright close "+session, 5*time.Second) - }() - - script := `(()=>JSON.stringify({url:location.href,title:document.title,forms:[...document.forms].map((f,i)=>({i,action:f.action,method:f.method,inputs:[...f.elements].map(e=>({tag:e.tagName,type:e.type,name:e.name,id:e.id,placeholder:e.placeholder}))})),buttons:[...document.querySelectorAll("button,input[type=button],input[type=submit],a")].slice(0,80).map(e=>({tag:e.tagName,text:(e.innerText||e.value||e.getAttribute("aria-label")||"").trim(),href:e.href||"",type:e.type||"",id:e.id||"",name:e.name||""})),scripts:[...document.scripts].map(s=>s.src).filter(Boolean).slice(0,50),localStorage:Object.keys(localStorage),sessionStorage:Object.keys(sessionStorage)}))()` - steps := []struct { - name string - command string - }{ - {"open", fmt.Sprintf("playwright open %s --session %s --op-timeout 8 --record", quoteCommandArg(targetURL), session)}, - {"network-start", "playwright network " + session + " --start"}, - {"reload", "playwright reload " + session}, - {"wait-idle", "playwright wait-for " + session + " --idle"}, - {"url", "playwright url " + session}, - {"discover", "playwright discover " + session}, - {"inner-text", "playwright inner-text " + session + " body"}, - {"storage-links-scripts", fmt.Sprintf("playwright evaluate %s %s", session, quoteCommandArg(script))}, - {"network-dump", "playwright network " + session + " --dump"}, - } - - const stepTimeout = 12 * time.Second - var sb strings.Builder - sb.WriteString("Target: ") - sb.WriteString(targetURL) - sb.WriteString("\nSession: ") - sb.WriteString(session) - sb.WriteString("\n") - for _, step := range steps { - if err := ctx.Err(); err != nil { - appendDeepBrowserStep(&sb, step.name, step.command, "", err) - break - } - out, err := executeRegistryCommand(ctx, reg, step.command, stepTimeout) - appendDeepBrowserStep(&sb, step.name, step.command, out, err) - if err != nil && logger != nil { - logger.Debugf("deep browser step=%s error=%q", step.name, err) - } - if err != nil { - break - } - } - - closeCtx, cancel := context.WithTimeout(context.Background(), 8*time.Second) - out, err := executeRegistryCommand(closeCtx, reg, "playwright close "+session, 8*time.Second) - cancel() - closed = true - appendDeepBrowserStep(&sb, "close", "playwright close "+session, out, err) - - artifact := sb.String() - if tr := truncate.Head(artifact, truncate.Options{}); tr.Truncated { - artifact = tr.Content + fmt.Sprintf( - "\n\n[deep browser truncated: showing %d/%d lines (%s of %s)]", - tr.OutputLines, tr.TotalLines, truncate.FormatSize(tr.OutputBytes), truncate.FormatSize(tr.TotalBytes)) - } - return artifact, nil -} diff --git a/pkg/runner/app_test.go b/pkg/runner/app_test.go index 20c51d3b..9130444e 100644 --- a/pkg/runner/app_test.go +++ b/pkg/runner/app_test.go @@ -15,12 +15,32 @@ import ( "github.com/chainreactors/aiscan/agent" aop "github.com/chainreactors/aiscan/aop" toolpb "github.com/chainreactors/aiscan/aop/tool" + coredeps "github.com/chainreactors/aiscan/core/deps" "github.com/chainreactors/aiscan/core/telemetry" + proxytool "github.com/chainreactors/aiscan/tools/proxy" "github.com/chainreactors/utils/parsers" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/anypb" ) +func TestAppOwnsOneSharedProxyInfrastructure(t *testing.T) { + app, err := NewApp(context.Background(), ApplicationConfig{SkipEngines: true, Logger: telemetry.NopLogger()}) + if err != nil { + t.Fatal(err) + } + defer app.Close() + if app.deps == nil || app.proxyInfra == nil || app.proxyInfra.Hub == nil { + t.Fatal("application proxy infrastructure is incomplete") + } + provided, ok := coredeps.Get(app.deps.Bag, proxytool.InfraKey) + if !ok || provided != app.proxyInfra { + t.Fatal("command factories do not share the application proxy infrastructure") + } + if app.deps.ScannerProxy != app.proxyInfra.Hub.ProxyURL() { + t.Fatalf("scanner proxy = %q, hub = %q", app.deps.ScannerProxy, app.proxyInfra.Hub.ProxyURL()) + } +} + func TestLogLLMProbeStatusReady(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/chat/completions" { diff --git a/pkg/runner/application_builder.go b/pkg/runner/application_builder.go index 7187073e..74944466 100644 --- a/pkg/runner/application_builder.go +++ b/pkg/runner/application_builder.go @@ -43,7 +43,7 @@ func AppConfigFromDistribute(dc *types.DistributeConfig, features RuntimeFeature }, Tools: ToolConfig{ Enabled: features.ToolsEnabled, - BashTimeout: 300, + BashTimeout: 600, TavilyKeys: dc.GetSearch().GetTavilyKeys(), OptionalTools: append([]string(nil), dc.GetAgent().GetTools()...), }, @@ -59,6 +59,7 @@ func MergeOptionExtras(rc ApplicationConfig, option *cfg.Option) ApplicationConf } rc.Scanner.UncoverCredentials = cloneStringMap(option.UncoverCredentials) rc.Tools.PlaywrightSession = option.PlaywrightSession + rc.Tools.MitmCapture = cloneBool(option.Mitm) rc.CLISkillPaths = skillPathsFromOptions(option) rc.RecordFile = option.OutputFile return rc @@ -87,10 +88,11 @@ func AppConfig(option *cfg.Option, features RuntimeFeatures, logger telemetry.Lo }, Tools: ToolConfig{ Enabled: features.ToolsEnabled, - BashTimeout: 300, + BashTimeout: 600, TavilyKeys: resolveTavilyKeys(option.TavilyKey, option.SearchConfig.TavilyKeys, cfg.DefaultTavilyKeys), PlaywrightSession: option.PlaywrightSession, OptionalTools: option.Tools, + MitmCapture: cloneBool(option.Mitm), }, Logger: logger, CLISkillPaths: skillPathsFromOptions(option), @@ -140,3 +142,11 @@ func cloneStringMap(src map[string]string) map[string]string { } return dst } + +func cloneBool(src *bool) *bool { + if src == nil { + return nil + } + value := *src + return &value +} diff --git a/pkg/runner/application_builder_test.go b/pkg/runner/application_builder_test.go new file mode 100644 index 00000000..a5f8b201 --- /dev/null +++ b/pkg/runner/application_builder_test.go @@ -0,0 +1,29 @@ +package runner + +import ( + "testing" + + cfg "github.com/chainreactors/aiscan/core/config" + "github.com/chainreactors/aiscan/core/telemetry" +) + +func TestAppConfigPreservesAutomaticCaptureDefault(t *testing.T) { + option := new(cfg.Option) + config := AppConfig(option, RuntimeFeatures{}, telemetry.NopLogger()) + if config.Tools.MitmCapture != nil { + t.Fatal("unset MITM option must remain unset until application defaults are applied") + } + if !captureEnabled(config.Tools.MitmCapture) { + t.Fatal("unset MITM option must enable capture") + } + + disabled := false + option.Mitm = &disabled + config = AppConfig(option, RuntimeFeatures{}, telemetry.NopLogger()) + if config.Tools.MitmCapture == nil || *config.Tools.MitmCapture { + t.Fatal("explicit MITM disable must be preserved") + } + if captureEnabled(config.Tools.MitmCapture) { + t.Fatal("explicit MITM disable must select relay mode") + } +} diff --git a/pkg/runner/application_config.go b/pkg/runner/application_config.go index 385c0d58..b970e385 100644 --- a/pkg/runner/application_config.go +++ b/pkg/runner/application_config.go @@ -40,10 +40,12 @@ type ScannerConfig struct { type ToolConfig struct { Enabled bool + RunnerMode bool BashTimeout int TavilyKeys string PlaywrightSession string OptionalTools []string // optional tool groups to enable + MitmCapture *bool // nil defaults to capture; false keeps routing without interception } type IOAConfig struct { diff --git a/pkg/runner/hooks.go b/pkg/runner/hooks.go deleted file mode 100644 index 55ff5021..00000000 --- a/pkg/runner/hooks.go +++ /dev/null @@ -1,24 +0,0 @@ -package runner - -import ( - "context" - - cfg "github.com/chainreactors/aiscan/core/config" - "github.com/chainreactors/aiscan/core/telemetry" -) - -// ScannerInitFunc initializes scanner engines and registers scanner commands. -// Set via init() from the package imported by cmd/aiscan. -var ScannerInitFunc func(ctx context.Context, a *App, rc ApplicationConfig, logger telemetry.Logger) - -// ScannerWithAgentFunc runs a scanner command with AI agent assistance. -// Set via init() from the package imported by cmd/aiscan. -var ScannerWithAgentFunc func(ctx context.Context, option *cfg.Option, application *App, scannerArgs []string, logger telemetry.Logger) error - -// IOAServeFunc starts the IOA HTTP server. -// Set via init() from cmd/aiscan setup. -var IOAServeFunc func(ctx context.Context, option *cfg.Option, logger telemetry.Logger) error - -// IOAClientCommandFunc dispatches IOA client CLI commands (spaces, messages, etc.). -// Set via init() from cmd/aiscan setup. -var IOAClientCommandFunc func(ctx context.Context, mode cfg.RunMode, option *cfg.Option, args cfg.IOAClientArgs, logger telemetry.Logger) error diff --git a/pkg/runner/ioa.go b/pkg/runner/ioa.go index 121edf9b..dbbfb868 100644 --- a/pkg/runner/ioa.go +++ b/pkg/runner/ioa.go @@ -5,25 +5,64 @@ import ( "crypto/rand" "encoding/hex" "fmt" + "net/url" + "os" "strconv" "time" cfg "github.com/chainreactors/aiscan/core/config" "github.com/chainreactors/aiscan/core/telemetry" + "github.com/chainreactors/aiscan/pkg/tui" + ioaclient "github.com/chainreactors/ioa/client" + "github.com/chainreactors/ioa/protocols" + ioaserver "github.com/chainreactors/ioa/server" ) func RunIOAServe(ctx context.Context, option *cfg.Option, logger telemetry.Logger) error { - if IOAServeFunc == nil { - return fmt.Errorf("ioa server not available in this build") + store := ioaserver.NewMemoryStore() + logger.Importantf("aiscan server store=memory") + defer func() { _ = store.Close() }() + + accessKey := option.IOAToken + if accessKey == "" { + accessKey = protocols.NewToken() + } + listenURL := option.IOAURL + if listenURL == "" { + listenURL = "http://127.0.0.1:8765" + } + if parsed, err := url.Parse(listenURL); err == nil { + logger.Infof(" agent IOA connect: aiscan agent --transport local --ioa-url http://%s@%s", accessKey, parsed.Host) } - return IOAServeFunc(ctx, option, logger) + return ioaserver.RunServer(ctx, ioaserver.ServerOptions{URL: listenURL, AccessKey: accessKey, Store: store}) } func RunIOAClientCommand(ctx context.Context, mode cfg.RunMode, option *cfg.Option, args cfg.IOAClientArgs, logger telemetry.Logger) error { - if IOAClientCommandFunc == nil { - return fmt.Errorf("ioa commands not available in this build") + ioaURL := option.IOAURL + if ioaURL == "" { + ioaURL = "http://127.0.0.1:8765" + } + client, err := ioaclient.NewClient(ioaURL, "") + if err != nil { + return fmt.Errorf("connect to server: %w", err) + } + if client.AccessKey() != "" { + if err := client.EnsureRegistered(ctx, "aiscan-cli", "", nil); err != nil { + return fmt.Errorf("server auth register: %w", err) + } + } + switch mode { + case cfg.RunModeIOASpaces: + return tui.RunIOASpaces(ctx, client, option, os.Stdout, os.Stderr) + case cfg.RunModeIOAMessages: + return tui.RunIOAMessages(ctx, client, option, args, os.Stdout, os.Stderr) + case cfg.RunModeIOAContext: + return tui.RunIOAContext(ctx, client, option, args, os.Stdout, os.Stderr) + case cfg.RunModeIOANodes: + return tui.RunIOANodes(ctx, client, option, args, os.Stdout, os.Stderr) + default: + return fmt.Errorf("unknown server mode: %s", mode) } - return IOAClientCommandFunc(ctx, mode, option, args, logger) } func ResolveIOANodeName(option *cfg.Option) string { diff --git a/pkg/runner/node_info_test.go b/pkg/runner/node_info_test.go index 1f0028ac..6efdf3d0 100644 --- a/pkg/runner/node_info_test.go +++ b/pkg/runner/node_info_test.go @@ -53,7 +53,7 @@ func TestCommandCatalogIncludesNodeRegistryCommands(t *testing.T) { if got["!tmux"] == nil || got["!tmux"].usage != "!tmux " { t.Fatalf("!tmux = %+v", got["!tmux"]) } - if got["!tmux"].description != "PTY session manager built into aiscan. All bash commands run inside tmux sessions; long commands auto-background with inbox delivery." { + if got["!tmux"].description != "PTY session manager built into aiscan. Bash commands stay foreground by default and move to background only when the agent sets wait." { t.Fatalf("!tmux description = %q", got["!tmux"].description) } } diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 665ca698..152f5f06 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -623,10 +623,7 @@ func RunDirectScannerMode(ctx context.Context, option *cfg.Option, rest []string } if option.AI && scannerArgs[0] != "scan" { - if ScannerWithAgentFunc == nil { - return fmt.Errorf("scanner agent mode not available in this build") - } - return ScannerWithAgentFunc(ctx, option, application, scannerArgs, logger) + return runScannerWithAgent(ctx, option, application, scannerArgs, logger) } if option.NoColor && scannerArgs[0] == "scan" && !HasScannerFlag(scannerArgs[1:], "--no-color") { diff --git a/pkg/runner/scanner.go b/pkg/runner/scanner.go index 291b8dbe..e6616a62 100644 --- a/pkg/runner/scanner.go +++ b/pkg/runner/scanner.go @@ -1,14 +1,29 @@ package runner import ( + "context" "fmt" + "os" "strings" + "time" - "github.com/chainreactors/aiscan/core/config" + "github.com/chainreactors/aiscan/agent" + aop "github.com/chainreactors/aiscan/aop" + "github.com/chainreactors/aiscan/core/capability" + cfg "github.com/chainreactors/aiscan/core/config" + "github.com/chainreactors/aiscan/core/pidlock" + "github.com/chainreactors/aiscan/core/resources" + "github.com/chainreactors/aiscan/core/telemetry" + "github.com/chainreactors/aiscan/core/truncate" + "github.com/chainreactors/aiscan/pkg/commands" + "github.com/chainreactors/aiscan/pkg/tui" + "github.com/chainreactors/aiscan/skills" + "github.com/chainreactors/aiscan/tools/scan" + "github.com/chainreactors/aiscan/tools/scan/engine" ) func DirectScannerRuntimeFeatures(rest []string) (RuntimeFeatures, []string, error) { - return DirectScannerRuntimeFeaturesWithDefault(rest, config.DefaultVerify) + return DirectScannerRuntimeFeaturesWithDefault(rest, cfg.DefaultVerify) } func DirectScannerRuntimeFeaturesWithDefault(rest []string, defaultVerify string) (RuntimeFeatures, []string, error) { @@ -93,7 +108,7 @@ func ShouldStreamScannerOutput(rest []string) bool { } func isDirectScannerJSONOutput(rest []string) bool { - if len(rest) == 0 || !config.ScannerCommandAvailable(rest[0]) { + if len(rest) == 0 || !cfg.ScannerCommandAvailable(rest[0]) { return false } for _, arg := range rest[1:] { @@ -171,3 +186,275 @@ func removeScannerFlag(args []string, flag string) []string { } return out } + +func (a *App) initScanner(ctx context.Context, rc ApplicationConfig, logger telemetry.Logger) { + engineSet := initEngines(ctx, rc.Scanner, logger) + a.Engines = engineSet + + var options []scan.Option + if rc.Scanner.AIEnabled && a.Provider != nil { + parent := agent.NewAgent(agent.Config{ + Provider: a.Provider, + Tools: a.Commands, + Model: a.ProviderConfig.Model, + MaxTokens: a.ProviderConfig.MaxTokens, + ContextWindow: a.ProviderConfig.ContextWindow, + Logger: logger, + Bus: a.Events, + }) + options = append(options, + scan.WithParent(parent), + scan.WithDeepBrowserFunc(func(ctx context.Context, targetURL string) (string, error) { + return CollectDeepBrowserArtifacts(ctx, a.Commands, targetURL, logger) + }), + ) + if a.Skills != nil { + options = append(options, scan.WithSkillReader(func(name string) string { + content, ok, err := a.Skills.ReadVirtual("aiscan://skills/scan/" + name + ".md") + if !ok || err != nil { + return "" + } + return content + })) + } + } + options = append(options, scan.WithLogger(logger)) + + a.assemblyMu.Lock() + defer a.assemblyMu.Unlock() + commands.Provide(a.deps, scan.OptsKey, options) + if engineSet != nil { + commands.Provide(a.deps, engine.SetKey, engineSet) + commands.Provide(a.deps, resources.SetKey, engineSet.Resources) + } + commands.BuildPlan(capability.Select(capability.Options{Groups: []string{"scanner"}}), a.deps, a.Commands) + logger.Infof("%s", telemetry.StartupOK("scanner", strings.Join(a.Commands.GroupNames("scanner"), ","))) +} + +func initEngines(ctx context.Context, scanner ScannerConfig, logger telemetry.Logger) *engine.Set { + engineSet, err := engine.InitWithOptions(ctx, resources.Options{ + CyberhubURL: scanner.CyberhubURL, + APIKey: scanner.CyberhubKey, + Mode: scanner.CyberhubMode, + Proxy: scanner.Proxy, + }, logger) + if err != nil { + logger.Warnf("scanner engines init error=%q action=continue_without_scanners", err) + return nil + } + engineSet.SetupUncover(engine.ReconOptions{ + FofaKey: scanner.FofaKey, + HunterAPIKey: scanner.HunterAPIKey, + IngressProxy: scanner.ReconProxy, + Limit: scanner.ReconLimit, + Credentials: scanner.UncoverCredentials, + }, logger) + return engineSet +} + +func runScannerWithAgent(ctx context.Context, option *cfg.Option, application *App, scannerArgs []string, logger telemetry.Logger) error { + if application.Provider == nil { + return fmt.Errorf("--ai requires a configured LLM provider") + } + lock, err := pidlock.Acquire(pidlock.AgentPIDFilePath(), logger) + if err != nil { + return err + } + defer lock.Release() + + intent, err := resolveScannerIntent(option, application.Skills, scannerArgs[0]) + if err != nil { + return err + } + runtime, err := NewAgentRuntime(ctx, option, logger, &RuntimeConfig{ + ExistingApp: application, + PromptConfig: &PromptConfig{ + Tools: application.Commands, + ScannerDocs: application.Commands.UsageDocs(), + Skills: application.Skills.Skills, + ScannerAgentMode: true, + ScannerName: scannerArgs[0], + }, + }) + if err != nil { + return err + } + defer runtime.Close() + + prompt := scan.FormatAgentTaskPrompt(scannerArgs, intent) + output := tui.NewStaticAgentOutput(option) + unsubscribe := runtime.Subscribe(output.HandleEvent) + defer unsubscribe() + output.Start("scanner", strings.Join(scannerArgs, " ")) + session, err := runtime.OpenSession(ctx, SessionOptions{ID: "scanner"}) + if err != nil { + return err + } + run, err := session.Run(ctx, RunInput{Content: []*aop.Content{aop.Text(prompt)}}) + if err != nil { + return err + } + result, err := run.Wait() + if strings.TrimSpace(result.Output) != "" { + output.Final(result.Output) + } + _ = runtime.CloseSession(context.Background(), "scanner", SessionCloseCompleted) + return err +} + +func resolveScannerIntent(option *cfg.Option, store *skills.Store, command string) (string, error) { + var sections []string + if conceptURI := scan.ScannerConceptURI(command); conceptURI != "" && cfg.ScannerCommandAvailable(command) { + if body, ok, err := store.ReadVirtualBody(conceptURI); err == nil && ok && body != "" { + sections = append(sections, skills.FormatVirtualInvocation(command, conceptURI, body)) + } + } + intent, err := cfg.ResolvePrompt(option.Prompt) + if err != nil { + return "", err + } + if intent == "" && option.TaskFile != "" { + data, err := os.ReadFile(option.TaskFile) + if err != nil { + return "", fmt.Errorf("read task file: %w", err) + } + intent = strings.TrimSpace(string(data)) + } + if intent == "" { + intent = "Process the scanner output according to the user's intent. If no specific intent is provided, briefly explain the important evidence in the output." + } + intent, err = cfg.ApplySelectedSkills(intent, scan.FilterAutoSkill(option.Skills, command), store) + if err != nil { + return "", err + } + return strings.Join(append(sections, intent), "\n\n"), nil +} + +func executeRegistryCommand(ctx context.Context, registry *commands.CommandRegistry, commandLine string, timeout time.Duration) (string, error) { + tool, ok := registry.GetTool("bash") + if !ok { + return "", fmt.Errorf("bash tool is not registered") + } + bash, ok := tool.(*commands.BashTool) + if !ok { + return "", fmt.Errorf("registered bash tool has unexpected type") + } + var output strings.Builder + execution, err := bash.RunForeground(ctx, commandLine, commands.BashExecOptions{ + Timeout: timeout, + OnOutput: func(data []byte) { + _, _ = output.Write(data) + }, + }) + if err != nil { + return output.String(), err + } + if execution.ExitCode != 0 { + return output.String(), fmt.Errorf("command exited with code %d", execution.ExitCode) + } + return output.String(), nil +} + +func appendDeepBrowserStep(output *strings.Builder, name, commandLine, content string, err error) { + output.WriteString("\n## ") + output.WriteString(name) + output.WriteString("\nCommand: `") + output.WriteString(commandLine) + output.WriteString("`\n") + if err != nil { + output.WriteString("Error: ") + output.WriteString(err.Error()) + output.WriteString("\n") + } + content = strings.TrimSpace(content) + if content == "" { + return + } + truncated := truncate.Head(content, truncate.Options{}) + output.WriteString(truncated.Content) + if truncated.Truncated { + output.WriteString(fmt.Sprintf("\n[step truncated: %d/%d lines]", truncated.OutputLines, truncated.TotalLines)) + } + output.WriteString("\n") +} + +func quoteCommandArg(value string) string { + if value == "" { + return `""` + } + if !strings.ContainsAny(value, " \t\r\n'\"\\") { + return value + } + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, `"`, `\"`) + return `"` + value + `"` +} + +func CollectDeepBrowserArtifacts(ctx context.Context, registry *commands.CommandRegistry, targetURL string, logger telemetry.Logger) (string, error) { + if registry == nil || !registry.Has("playwright") { + return "", fmt.Errorf("playwright command unavailable") + } + targetURL = strings.TrimSpace(targetURL) + if targetURL == "" { + return "", fmt.Errorf("target URL is empty") + } + + session := fmt.Sprintf("deep%d", time.Now().UnixNano()) + closed := false + defer func() { + if closed { + return + } + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, _ = executeRegistryCommand(closeCtx, registry, "playwright close "+session, 5*time.Second) + }() + + script := `(()=>JSON.stringify({url:location.href,title:document.title,forms:[...document.forms].map((f,i)=>({i,action:f.action,method:f.method,inputs:[...f.elements].map(e=>({tag:e.tagName,type:e.type,name:e.name,id:e.id,placeholder:e.placeholder}))})),buttons:[...document.querySelectorAll("button,input[type=button],input[type=submit],a")].slice(0,80).map(e=>({tag:e.tagName,text:(e.innerText||e.value||e.getAttribute("aria-label")||"").trim(),href:e.href||"",type:e.type||"",id:e.id||"",name:e.name||""})),scripts:[...document.scripts].map(s=>s.src).filter(Boolean).slice(0,50),localStorage:Object.keys(localStorage),sessionStorage:Object.keys(sessionStorage)}))()` + steps := []struct { + name string + command string + }{ + {"open", fmt.Sprintf("playwright open %s --session %s --op-timeout 8 --record", quoteCommandArg(targetURL), session)}, + {"network-start", "playwright network " + session + " --start"}, + {"reload", "playwright reload " + session}, + {"wait-idle", "playwright wait-for " + session + " --idle"}, + {"url", "playwright url " + session}, + {"discover", "playwright discover " + session}, + {"inner-text", "playwright inner-text " + session + " body"}, + {"storage-links-scripts", fmt.Sprintf("playwright evaluate %s %s", session, quoteCommandArg(script))}, + {"network-dump", "playwright network " + session + " --dump"}, + } + + var output strings.Builder + output.WriteString("Target: " + targetURL + "\nSession: " + session + "\n") + for _, step := range steps { + if err := ctx.Err(); err != nil { + appendDeepBrowserStep(&output, step.name, step.command, "", err) + break + } + content, err := executeRegistryCommand(ctx, registry, step.command, 12*time.Second) + appendDeepBrowserStep(&output, step.name, step.command, content, err) + if err != nil { + if logger != nil { + logger.Debugf("deep browser step=%s error=%q", step.name, err) + } + break + } + } + + closeCtx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + content, err := executeRegistryCommand(closeCtx, registry, "playwright close "+session, 8*time.Second) + cancel() + closed = true + appendDeepBrowserStep(&output, "close", "playwright close "+session, content, err) + + artifact := truncate.Head(output.String(), truncate.Options{}) + if !artifact.Truncated { + return artifact.Content, nil + } + return artifact.Content + fmt.Sprintf( + "\n\n[deep browser truncated: showing %d/%d lines (%s of %s)]", + artifact.OutputLines, artifact.TotalLines, truncate.FormatSize(artifact.OutputBytes), truncate.FormatSize(artifact.TotalBytes), + ), nil +} diff --git a/pkg/runner/tool_call.go b/pkg/runner/tool_call.go index d587d014..cf969435 100644 --- a/pkg/runner/tool_call.go +++ b/pkg/runner/tool_call.go @@ -72,11 +72,16 @@ func executeCall(ctx context.Context, executor ToolExecutor, call *aop.ToolCall, if err != nil { return nil, err } + if err := args.Validate(); err != nil { + return nil, err + } progress := newProgressStreamer(progressBus, call.Name, callID) - result, err := registry.ExecuteBashForeground(ctx, args.Command, commands.BashExecOptions{ - Timeout: time.Duration(args.Timeout) * time.Second, - OnOutput: progress.Write, - }) + options := commands.BashExecOptions{OnOutput: progress.Write} + if args.TimeoutSpecified() { + options.Timeout = time.Duration(args.Timeout) * time.Second + options.TimeoutSet = true + } + result, err := registry.ExecuteBashForeground(ctx, args.Command, options) progress.Flush() return result, err } diff --git a/pkg/runner/tool_call_test.go b/pkg/runner/tool_call_test.go index 073f15af..dd07509b 100644 --- a/pkg/runner/tool_call_test.go +++ b/pkg/runner/tool_call_test.go @@ -138,6 +138,23 @@ func TestExecuteToolRequestForeground(t *testing.T) { } } +func TestExecuteToolRequestForegroundPreservesExplicitZeroTimeout(t *testing.T) { + registry := commands.NewRegistry() + bash := &recordingBash{} + registry.RegisterTool(bash) + + _, err := ExecuteToolRequest(context.Background(), "task-zero-timeout", toolRequest(t, "task-zero-timeout", "bash", map[string]any{ + "command": "echo test", + "timeout": 0, + }), registry, nil) + if err != nil { + t.Fatal(err) + } + if !bash.options.TimeoutSet || bash.options.Timeout != 0 { + t.Fatalf("bash options = %+v, want explicit unlimited timeout", bash.options) + } +} + func TestProgressStreamerSanitizesInvalidUTF8(t *testing.T) { progressBus := eventbus.New[*toolpb.Progress]() var progress []*toolpb.Progress diff --git a/skills/aiscan/SKILL.md b/skills/aiscan/SKILL.md index 936b04ea..44251de8 100644 --- a/skills/aiscan/SKILL.md +++ b/skills/aiscan/SKILL.md @@ -15,7 +15,7 @@ Use these capabilities to inspect inputs, execute supporting analysis, and colle - `bash`: run shell commands and pseudo-commands (see below). - `web_search`: search the web for CVEs, advisories, exploits, and documentation. - `fetch`: fetch and read a specific URL. -- `record` (Windows/Linux full builds): capture desktop or visible application-window screenshots and H.264 recordings. It accepts HWND/X11 Window IDs or resolves a PID to its main visible window. +- `record` (optional Windows/Linux SDK builds): capture desktop or visible application-window screenshots and H.264 recordings. It accepts HWND/X11 Window IDs or resolves a PID to its main visible window. ## ASM and Penetration Tools @@ -76,9 +76,13 @@ When producing a scan report, follow the format and verification semantics in `a ## Execution Environment -`bash` accepts a single `command` argument — no `background` or `timeout` fields. Every command runs in a tmux session. Pseudo-commands run in-process; others run as shell commands in a PTY. Keep invocations self-contained — no shell state carryover. +`bash` accepts `command`, `wait`, and `timeout`. Every command runs in a tmux session. Pseudo-commands run in-process; others run as shell commands in a PTY. Keep invocations self-contained — no shell state carryover. -Long-running commands auto-background after 15s, returning a session id. Incremental output arrives via inbox automatically — no polling needed. +- `wait: 0` (default): stay in the foreground until completion. +- `wait: N`: move a still-running command to background after N seconds and return its session id. This is not a failure or cancellation. +- omitted `timeout`: use the 600s safety timeout. `timeout: N` cancels the command after N total seconds, including background time. `timeout: 0` disables the command timeout. + +Background completion is delivered through the inbox automatically. Incremental output is best-effort; completion delivery is retained with higher priority. Interactive shells (`su`, `python`, `mysql` prompts) do not work. Use "one command in → stdout out" pattern. diff --git a/skills/aiscan/okf/runtime/tmux.md b/skills/aiscan/okf/runtime/tmux.md index 2745ded3..f8a0fde3 100644 --- a/skills/aiscan/okf/runtime/tmux.md +++ b/skills/aiscan/okf/runtime/tmux.md @@ -1,7 +1,7 @@ --- type: Tool Playbook title: tmux -description: PTY session manager built into aiscan. All bash commands run inside tmux sessions; long commands auto-background with inbox delivery. +description: PTY session manager built into aiscan. Bash commands stay foreground by default and move to background only when the agent sets wait. tags: [runtime, session] status: stable generated: { by: process:okf-maintain, at: 2026-08-02T11:46:25Z } @@ -9,7 +9,7 @@ generated: { by: process:okf-maintain, at: 2026-08-02T11:46:25Z } # tmux - Session Manager -tmux is the PTY session manager built into aiscan. All `bash` commands run inside tmux sessions. Commands completing within 15 seconds return output inline; longer commands are auto-backgrounded with incremental output delivered to the agent inbox automatically. +tmux is the PTY session manager built into aiscan. All `bash` commands run inside tmux sessions. Commands stay in the foreground by default. The agent explicitly sets `wait: N` when a still-running command should move to background after N seconds. ## Commands @@ -49,13 +49,15 @@ tmux capture-pane -t --full -n 100 ``` Re-reads the entire buffer (or last N lines of it). Use sparingly; prefer incremental or `-n` for most reads. -## Auto-Background & Inbox Monitoring +## Explicit Background & Inbox Monitoring -When a `bash` command exceeds 15 seconds, it is automatically backgrounded: +When a `bash` call sets `wait: N` and the command is still running after N seconds: 1. The bash tool returns immediately with the session id. 2. A **monitor goroutine** starts, pushing incremental output to the agent inbox every 10 seconds as `` messages. 3. When the session completes, a `` message is pushed to inbox with exit code and last 20 lines. +With `wait: 0` or an omitted `wait`, the command remains foreground until completion. `timeout` is independent: omitted uses 600 seconds, a positive value cancels after that total runtime, and `timeout: 0` means unlimited. + This means for long-running commands: - **You do not need to poll** with `tmux capture-pane`. Output arrives automatically via inbox. - Wait for inbox messages to review progress, then decide next action. @@ -66,8 +68,8 @@ This means for long-running commands: ### Long scan — let monitoring deliver output ``` -bash: gogo -i 10.0.0.0/24 -p top2 -# auto-backgrounded → session id returned +bash: {"command":"gogo -i 10.0.0.0/24 -p top2","wait":15} +# still running after 15s → session id returned # wait for inbox messages with scan progress # wait for inbox when done ``` diff --git a/test-skips.json b/test-skips.json index 10cb8e48..af437a1d 100644 --- a/test-skips.json +++ b/test-skips.json @@ -49,7 +49,7 @@ "reason": "Live multi-provider cache coverage requires explicit endpoint credentials." }, { - "path": "archtest/architecture_test.go", + "path": "internal/repositorytest/architecture_test.go", "format": "repository governance requires a Git checkout: %v", "count": 1, "category": "external_runtime", diff --git a/tools/proxy/command.go b/tools/proxy/command.go index 88ddfdb9..512e3571 100644 --- a/tools/proxy/command.go +++ b/tools/proxy/command.go @@ -13,21 +13,19 @@ import ( goflags "github.com/jessevdk/go-flags" ) -type OnProxyChange func(newProxyURL string) - type CommandExecutor func(ctx context.Context, tokens []string, execution *commands.Execution) (any, error) type Command struct { - state *State - onProxyChange OnProxyChange - execCommand CommandExecutor + state *State + hub *ProxyHub + execCommand CommandExecutor } func New(state *State) *Command { return &Command{state: state} } -func (c *Command) SetOnProxyChange(fn OnProxyChange) { c.onProxyChange = fn } +func (c *Command) SetHub(hub *ProxyHub) { c.hub = hub } func (c *Command) SetCommandExecutor(fn CommandExecutor) { c.execCommand = fn } func (c *Command) Name() string { return "proxy" } @@ -131,19 +129,14 @@ func (c *Command) execPassthrough(ctx context.Context, proxyURL string, cmdArgs if c.execCommand == nil { return nil, fmt.Errorf("proxy passthrough not available (no command executor)") } - if _, err := url.Parse(proxyURL); err != nil { - return nil, fmt.Errorf("invalid proxy URL: %w", err) - } - - prev := c.state.ActiveProxy() - if c.onProxyChange != nil { - c.onProxyChange(proxyURL) + // Route this one command through proxyURL by temporarily swapping the hub's + // upstream. Children keep pointing at the stable hub address; only the + // egress chain changes for the duration of the wrapped command. + restore, err := c.state.WithOverrideDial(proxyURL) + if err != nil { + return nil, err } - defer func() { - if c.onProxyChange != nil { - c.onProxyChange(prev) - } - }() + defer restore() return c.execCommand(ctx, cmdArgs, execution) } @@ -202,10 +195,6 @@ func (c *Command) execAuto(_ context.Context, args []string) (string, error) { } c.state.SetAutoDial(clashURL, dial) - if c.onProxyChange != nil { - c.onProxyChange(clashURL) - } - supported := clash.SupportedNodes(sub) var sb strings.Builder sb.WriteString("[proxy] auto mode enabled\n") @@ -258,9 +247,6 @@ func (c *Command) execSwitch(args []string) (string, error) { return "", err } newProxy := c.state.ActiveProxy() - if c.onProxyChange != nil { - c.onProxyChange(newProxy) - } return fmt.Sprintf("[proxy] switched to %q\nProxy URL: %s", c.state.ActiveNodeName(), newProxy), nil } @@ -316,9 +302,6 @@ func (c *Command) execCurrent() (string, error) { func (c *Command) execClear() (string, error) { c.state.Clear() original := c.state.OriginalProxy() - if c.onProxyChange != nil { - c.onProxyChange(original) - } if original != "" { return fmt.Sprintf("[proxy] cleared. Reverted to original proxy: %s", original), nil } diff --git a/tools/proxy/command_test.go b/tools/proxy/command_test.go index f5555f86..106dd61d 100644 --- a/tools/proxy/command_test.go +++ b/tools/proxy/command_test.go @@ -96,8 +96,6 @@ func TestSwitchNoSubscription(t *testing.T) { func TestClear(t *testing.T) { state := NewState("socks5://127.0.0.1:1080") cmd := New(state) - var lastProxy string - cmd.SetOnProxyChange(func(p string) { lastProxy = p }) out, err := runProxy(cmd, "clear") if err != nil { @@ -106,8 +104,13 @@ func TestClear(t *testing.T) { if !strings.Contains(out, "cleared") { t.Fatalf("expected 'cleared', got: %q", out) } - if lastProxy != "socks5://127.0.0.1:1080" { - t.Fatalf("expected revert to original proxy, got: %q", lastProxy) + // Clear reverts the egress to the original proxy; the message reports it and + // the republished chain must remain usable. + if !strings.Contains(out, "socks5://127.0.0.1:1080") { + t.Fatalf("expected revert to original proxy in output, got: %q", out) + } + if state.CurrentDial() == nil { + t.Fatal("CurrentDial must remain non-nil after clear") } } @@ -136,12 +139,13 @@ func TestPassthroughNoExecutor(t *testing.T) { } func TestPassthroughSetsAndRevertsProxy(t *testing.T) { - state := NewState("original://proxy") + state := NewState("socks5://127.0.0.1:1080") cmd := New(state) + base := state.dialPtr() - var proxyChanges []string - cmd.SetOnProxyChange(func(p string) { proxyChanges = append(proxyChanges, p) }) + var duringExec = base cmd.SetCommandExecutor(func(_ context.Context, tokens []string, execution *commands.Execution) (any, error) { + duringExec = state.dialPtr() fmt.Fprint(execution.Stdout, "executed: "+strings.Join(tokens, " ")) return nil, nil }) @@ -153,14 +157,13 @@ func TestPassthroughSetsAndRevertsProxy(t *testing.T) { if !strings.Contains(out, "executed: echo hello") { t.Fatalf("expected command output, got: %q", out) } - if len(proxyChanges) != 2 { - t.Fatalf("expected 2 proxy changes (set + revert), got %d: %v", len(proxyChanges), proxyChanges) - } - if proxyChanges[0] != "socks5://127.0.0.1:9999" { - t.Fatalf("first proxy change = %q, want socks5://127.0.0.1:9999", proxyChanges[0]) + // The override republishes a different chain for the duration of the wrapped + // command, then restores the previous one. + if duringExec == base { + t.Fatal("expected egress chain to be overridden during passthrough execution") } - if proxyChanges[1] != "original://proxy" { - t.Fatalf("second proxy change = %q, want original://proxy (revert)", proxyChanges[1]) + if state.dialPtr() != base { + t.Fatal("expected egress chain to be restored after passthrough") } } diff --git a/tools/proxy/hub.go b/tools/proxy/hub.go new file mode 100644 index 00000000..9413f282 --- /dev/null +++ b/tools/proxy/hub.go @@ -0,0 +1,216 @@ +package proxy + +import ( + "context" + "encoding/pem" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + traffic "github.com/chainreactors/aiscan/aop/traffic" + mitmproxy "github.com/chainreactors/utils/mitmproxy/proxy" +) + +// ProxyHub is the runner-level, long-lived MITM proxy that every tool routes +// through. It is the STABLE front hop: its local address is injected once into +// child process env and in-process HTTP clients and never changes. The DYNAMIC +// back hop — the actual egress proxy chain — lives in State and is swapped live +// via State.CurrentDial(), which hub.dial reads on every connection. Switching +// proxy nodes therefore takes effect immediately without re-injecting anything +// into already-running children. +// +// hub.dial is installed as mitmproxy Options.Dialer so it covers plain HTTP as +// well as HTTPS/CONNECT (see the local mitmproxy fork patch adding that field). +type ProxyHub struct { + state *State + store *FlowStore + + mu sync.Mutex + server *mitmproxy.Proxy + addr string + caPath string + started bool + startErr error + startOnce sync.Once + + // Capture is runtime-mutable so the control plane can toggle it via the + // traffic namespace without restarting the listener. recording gates whether + // flows are stored and streamed; decrypt gates HTTPS MITM interception. Both + // are read on every connection, so a change takes effect for subsequent + // connections while in-flight children are undisturbed. + recording atomic.Bool + decrypt atomic.Bool + + subsMu sync.Mutex + subs map[int]chan *traffic.Flow + nextSub int +} + +const hubStreamLargeBodies = 10 * 1024 * 1024 + +// NewProxyHub builds the hub around an existing State (egress source of truth) +// and FlowStore (capture sink). Both are owned by the caller so the mitm query +// verbs and the hub share one store. +// +// capture selects the mode. The hub is ALWAYS the routing substrate — tools +// route through it and `proxy switch` swaps its upstream live in either mode. +// - capture=true (mitm on): intercept + record HTTPS (MITM) and HTTP flows. +// - capture=false (mitm off): pure relay — no interception, no recording, no +// CA needed. Routing still works; nothing is decrypted or stored. +// +// Start must be called before use. +func NewProxyHub(state *State, store *FlowStore, caRootPath string, capture bool) *ProxyHub { + if store == nil { + store = NewFlowStore(10000) + } + h := &ProxyHub{state: state, store: store, subs: make(map[int]chan *traffic.Flow)} + // The CA path is always prepared so capture can be toggled on at runtime; + // CAPath only advertises it to children while interception is actually on. + h.caPath = filepath.Join(caRootPath, "mitmproxy-ca-cert.pem") + h.recording.Store(capture) + h.decrypt.Store(capture) + return h +} + +// Capturing reports whether the hub currently records traffic (mitm on) or only +// relays. +func (h *ProxyHub) Capturing() bool { return h.recording.Load() } + +// SetCapture toggles capture at runtime without restarting the listener. record +// gates storing/streaming; decryptHTTPS gates HTTPS MITM interception, which +// only affects connections opened after the change because a child's CA trust +// is fixed at spawn time. +func (h *ProxyHub) SetCapture(record, decryptHTTPS bool) { + h.recording.Store(record) + h.decrypt.Store(decryptHTTPS) +} + +// Start brings up the MITM listener on an ephemeral loopback port and exports +// the CA certificate so external processes can trust intercepted HTTPS. It is +// idempotent: repeated calls return the first outcome. +func (h *ProxyHub) Start(caRootPath string) error { + h.startOnce.Do(func() { + h.startErr = h.start(caRootPath) + }) + return h.startErr +} + +func (h *ProxyHub) start(caRootPath string) error { + if caRootPath != "" { + if err := os.MkdirAll(caRootPath, 0o755); err != nil { + return fmt.Errorf("proxy hub: create CA dir: %w", err) + } + } + server, err := mitmproxy.NewProxy(&mitmproxy.Options{ + Addr: "127.0.0.1:0", + SslInsecure: true, + StreamLargeBodies: hubStreamLargeBodies, + CaRootPath: caRootPath, + Dialer: h.dial, + }) + if err != nil { + return fmt.Errorf("proxy hub: create MITM proxy: %w", err) + } + // The addon is always installed; recording gates whether it stores/streams + // (see ingest). HTTPS CONNECTs are MITM-decrypted only while capture and + // decrypt are both on, so a relay-mode child that tunnels HTTPS is never + // handed a forged certificate its env does not trust. + server.AddAddon(&captureAddon{hub: h}) + server.SetShouldInterceptRule(func(*http.Request) bool { + return h.recording.Load() && h.decrypt.Load() + }) + + listenAddr, _, err := server.StartAsync() + if err != nil { + return fmt.Errorf("proxy hub: start MITM proxy: %w", err) + } + + h.mu.Lock() + h.server = server + h.addr = listenAddr.String() + h.started = true + h.mu.Unlock() + + // Export the CA up front so children can trust intercepted HTTPS whenever + // capture is toggled on later. A failure only degrades HTTPS interception to + // CONNECT metadata; it is not fatal to the proxy itself. + if err := h.exportCA(server); err != nil { + h.mu.Lock() + h.caPath = "" + h.mu.Unlock() + } + return nil +} + +// dial is the stable indirection: it reads the current egress chain from State +// on every connection, so `proxy switch/auto/clear` swaps the upstream live. +func (h *ProxyHub) dial(ctx context.Context, network, address string) (net.Conn, error) { + if h.state == nil { + return (&net.Dialer{}).DialContext(ctx, network, address) + } + return h.state.CurrentDial()(ctx, network, address) +} + +// exportCA writes the proxy's root CA to caPath in PEM so external tools can be +// pointed at it via CURL_CA_BUNDLE / SSL_CERT_FILE / NODE_EXTRA_CA_CERTS. +func (h *ProxyHub) exportCA(server *mitmproxy.Proxy) error { + if h.caPath == "" { + return nil + } + crt := server.GetCertificate() + if len(crt.Raw) == 0 { + return fmt.Errorf("proxy hub: empty root CA") + } + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: crt.Raw}) + if err := os.WriteFile(h.caPath, pemBytes, 0o644); err != nil { + return fmt.Errorf("proxy hub: write CA: %w", err) + } + return nil +} + +// ProxyURL is the stable http:// address injected into children and in-process +// clients. Empty until Start succeeds. +func (h *ProxyHub) ProxyURL() string { + h.mu.Lock() + defer h.mu.Unlock() + if h.addr == "" { + return "" + } + return "http://" + h.addr +} + +// CAPath is the exported CA PEM path to advertise to children, or "" when the +// hub is not currently MITM-decrypting HTTPS. It returns a path only while +// capture and decrypt are both on: a child must trust the hub's CA exactly when +// the hub forges certificates for it, and must not when HTTPS is tunneled (a +// CA-only bundle would then fail to validate the real server certificate). +func (h *ProxyHub) CAPath() string { + if !(h.recording.Load() && h.decrypt.Load()) { + return "" + } + h.mu.Lock() + defer h.mu.Unlock() + return h.caPath +} + +// Shutdown stops the listener. Safe to call on a never-started hub. +func (h *ProxyHub) Shutdown(ctx context.Context) { + h.mu.Lock() + server := h.server + h.server = nil + h.mu.Unlock() + if server == nil { + return + } + if ctx == nil { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + } + _ = server.Shutdown(ctx) +} diff --git a/tools/proxy/hub_test.go b/tools/proxy/hub_test.go new file mode 100644 index 00000000..b94fd5ab --- /dev/null +++ b/tools/proxy/hub_test.go @@ -0,0 +1,406 @@ +package proxy + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/chainreactors/aiscan/pkg/commands" +) + +// newTestHub starts a hub (capture or relay) with an isolated CA dir and returns +// it plus an HTTP client that routes through it and trusts its CA. Keep-alives +// are disabled so every request forces a fresh upstream dial — otherwise a +// pooled connection to the same host masks egress-chain changes. +func newTestHub(t *testing.T, capture bool) (*ProxyHub, *State, *http.Client) { + t.Helper() + state := NewState("") + store := NewFlowStore(10000) + caRoot := t.TempDir() + hub := NewProxyHub(state, store, caRoot, capture) + if err := hub.Start(caRoot); err != nil { + t.Fatalf("hub start: %v", err) + } + t.Cleanup(func() { hub.Shutdown(context.Background()) }) + + pool := x509.NewCertPool() + if ca := hub.CAPath(); ca != "" { + pem, err := os.ReadFile(ca) + if err != nil { + t.Fatalf("read hub CA: %v", err) + } + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("hub CA not added to pool") + } + } + proxyURL, err := url.Parse(hub.ProxyURL()) + if err != nil { + t.Fatalf("parse hub url: %v", err) + } + client := &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{RootCAs: pool}, + DisableKeepAlives: true, + }, + } + return hub, state, client +} + +// runMitm executes a mitm verb and returns its stdout. +func runMitm(t *testing.T, store *FlowStore, hub *ProxyHub, args ...string) string { + t.Helper() + cmd := NewMitmCommand(nil, store, hub) + var out bytes.Buffer + exec := &commands.Execution{Args: args, Stdout: &out, Stderr: &out} + if _, err := cmd.Run(context.Background(), exec); err != nil { + t.Fatalf("mitm %v: %v", args, err) + } + return out.String() +} + +// --------------------------------------------------------------------------- +// Capture scenarios (mitm on) +// --------------------------------------------------------------------------- + +func TestCaptureHTTPAndHTTPS(t *testing.T) { + httpSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, "plain") + })) + defer httpSrv.Close() + tlsSrv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"ok":true}`) + })) + defer tlsSrv.Close() + + hub, _, client := newTestHub(t, true) + + // Plain HTTP — exercises the fork Options.Dialer patch path. + if body := get(t, client, httpSrv.URL); !strings.Contains(body, "plain") { + t.Fatalf("http body = %q", body) + } + // HTTPS — MITM decrypt with the client trusting the hub CA. Use a hostname + // target: the hub forges a cert by CN, and a bare-IP CN yields no IP SAN, + // which strict verifiers reject (a real MITM-of-IP-HTTPS limitation). + if body := get(t, client, localhost(tlsSrv.URL)); !strings.Contains(body, `"ok":true`) { + t.Fatalf("https body = %q", body) + } + + flows := hub.Store().Query(QueryOpts{}) + if len(flows) < 2 { + t.Fatalf("want >=2 flows, got %d", len(flows)) + } + var sawHTTP, sawHTTPSDecoded bool + for _, f := range flows { + if !f.TLS && strings.Contains(string(f.ResponseBodySnip), "plain") { + sawHTTP = true + } + if f.TLS && strings.Contains(string(f.ResponseBodySnip), `"ok":true`) { + sawHTTPSDecoded = true // decrypted body proves real MITM + } + } + if !sawHTTP { + t.Error("plain HTTP flow with body not captured") + } + if !sawHTTPSDecoded { + t.Error("HTTPS flow not decrypted/captured (MITM or CA trust failed)") + } +} + +func TestCapturePostRequestBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(io.Discard, r.Body) + w.WriteHeader(201) + })) + defer srv.Close() + hub, _, client := newTestHub(t, true) + + resp, err := client.Post(srv.URL, "application/json", strings.NewReader(`{"probe":"payload-marker"}`)) + if err != nil { + t.Fatalf("post: %v", err) + } + resp.Body.Close() + + flows := hub.Store().Query(QueryOpts{}) + var found bool + for _, f := range flows { + if f.Method == "POST" && strings.Contains(string(f.RequestBodySnip), "payload-marker") { + found = true + if f.StatusCode != 201 { + t.Errorf("status = %d, want 201", f.StatusCode) + } + } + } + if !found { + t.Error("POST request body not captured") + } +} + +func TestCaptureFiltersAndVerbs(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/ok", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + }) + mux.HandleFunc("/missing", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) }) + mux.HandleFunc("/boom", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }) + srv := httptest.NewServer(mux) + defer srv.Close() + hub, _, client := newTestHub(t, true) + + for _, p := range []string{"/ok", "/missing", "/boom"} { + resp, err := client.Get(srv.URL + p) + if err != nil { + t.Fatalf("get %s: %v", p, err) + } + resp.Body.Close() + } + + store := hub.Store() + if got := len(store.Query(QueryOpts{Status: "404"})); got != 1 { + t.Errorf("status 404 filter = %d, want 1", got) + } + if got := len(store.Query(QueryOpts{Status: "5xx"})); got != 1 { + t.Errorf("status 5xx filter = %d, want 1", got) + } + if got := len(store.Query(QueryOpts{CType: "json"})); got != 1 { + t.Errorf("content-type json filter = %d, want 1", got) + } + if got := len(store.Query(QueryOpts{Last: 2})); got != 2 { + t.Errorf("last 2 = %d, want 2", got) + } + + // Verbs: flows, flow , analyze, clear. + if out := runMitm(t, store, hub, "flows"); !strings.Contains(out, "flows") { + t.Errorf("flows output = %q", out) + } + first := store.Query(QueryOpts{Last: 1}) + if len(first) == 1 { + out := runMitm(t, store, hub, "flow", fmt.Sprintf("%d", first[0].ID)) + if !strings.Contains(out, "Request Headers") { + t.Errorf("flow detail missing headers: %q", out) + } + } + if out := runMitm(t, store, hub, "analyze"); !strings.Contains(out, "Summary") { + t.Errorf("analyze output = %q", out) + } + runMitm(t, store, hub, "clear") + if store.Count() != 0 { + t.Errorf("store not cleared: %d", store.Count()) + } +} + +func TestCaptureLargeBodyIsSnipped(t *testing.T) { + big := strings.Repeat("A", maxBodySnip*3) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, big) + })) + defer srv.Close() + hub, _, client := newTestHub(t, true) + get(t, client, srv.URL) + + for _, f := range hub.Store().Query(QueryOpts{}) { + if len(f.ResponseBodySnip) > maxBodySnip { + t.Fatalf("body snip = %d, want <= %d", len(f.ResponseBodySnip), maxBodySnip) + } + } +} + +func TestCaptureConcurrent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })) + defer srv.Close() + hub, _, client := newTestHub(t, true) + + const n = 20 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if resp, err := client.Get(srv.URL); err == nil { + resp.Body.Close() + } + }() + } + wg.Wait() + if got := hub.Store().Count(); got != n { + t.Errorf("captured %d flows, want %d", got, n) + } +} + +func TestCaptureConnectionError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + deadURL := srv.URL + srv.Close() // now refused + hub, _, client := newTestHub(t, true) + + resp, err := client.Get(deadURL) + if err == nil { + resp.Body.Close() + } + // The failed upstream is recorded as a flow carrying the error. + var sawErr bool + for _, f := range hub.Store().Query(QueryOpts{}) { + if f.Error != "" { + sawErr = true + } + } + if !sawErr { + t.Error("connection error not captured as a flow") + } +} + +// --------------------------------------------------------------------------- +// Proxy mechanism +// --------------------------------------------------------------------------- + +// TestChainTraversesUpstreamProxy proves tool → hub → upstream proxy → target: +// a counting CONNECT proxy set as the egress upstream must see the connection. +func TestChainTraversesUpstreamProxy(t *testing.T) { + target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "reached-via-chain") + })) + defer target.Close() + + upstreamAddr, connects := startCountingConnectProxy(t) + + _, state, client := newTestHub(t, true) + restore, err := state.WithOverrideDial("http://" + upstreamAddr) + if err != nil { + t.Fatalf("override: %v", err) + } + defer restore() + + body := get(t, client, localhost(target.URL)) + if !strings.Contains(body, "reached-via-chain") { + t.Fatalf("target not reached through chain: %q", body) + } + if atomic.LoadInt32(connects) == 0 { + t.Fatal("upstream proxy was not traversed (egress chain not applied)") + } +} + +// TestFailClosedNoDirectLeak proves an unreachable egress upstream causes the +// request to fail rather than silently leaking a direct connection. +func TestFailClosedNoDirectLeak(t *testing.T) { + target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "should-not-be-reached") + })) + defer target.Close() + + _, state, client := newTestHub(t, true) + restore, err := state.WithOverrideDial("socks5://127.0.0.1:1") // nothing listening + if err != nil { + t.Fatalf("override: %v", err) + } + defer restore() + + resp, err := client.Get(target.URL) + if err == nil { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 64)) + resp.Body.Close() + t.Fatalf("request unexpectedly succeeded (leaked direct): %q", b) + } +} + +// TestRelayModeRoutesButDoesNotCapture proves capture=false forwards traffic +// (routing works) without intercepting/recording it. +func TestRelayModeRoutesButDoesNotCapture(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "relayed-ok") + })) + defer target.Close() + + hub, _, client := newTestHub(t, false) // relay mode + if hub.Capturing() { + t.Fatal("hub should not be capturing in relay mode") + } + if hub.CAPath() != "" { + t.Error("relay mode should not export a CA") + } + body := get(t, client, target.URL) + if !strings.Contains(body, "relayed-ok") { + t.Fatalf("relay routing failed: %q", body) + } + // Plain HTTP has no addon in relay mode, so nothing is recorded. + if got := hub.Store().Count(); got != 0 { + t.Errorf("relay mode captured %d flows, want 0", got) + } +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +// localhost rewrites a 127.0.0.1 test URL to a hostname so the hub's forged +// leaf cert (CN-based, DNS SAN) verifies. Bare-IP targets have no IP SAN. +func localhost(u string) string { return strings.Replace(u, "127.0.0.1", "localhost", 1) } + +func get(t *testing.T, c *http.Client, u string) string { + t.Helper() + resp, err := c.Get(u) + if err != nil { + t.Fatalf("GET %s: %v", u, err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + return string(b) +} + +// startCountingConnectProxy is a minimal HTTP CONNECT proxy that counts tunnels +// and relays bytes, used to prove egress actually traverses the upstream. +func startCountingConnectProxy(t *testing.T) (addr string, count *int32) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + var c int32 + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go serveConnect(conn, &c) + } + }() + t.Cleanup(func() { ln.Close() }) + return ln.Addr().String(), &c +} + +func serveConnect(client net.Conn, count *int32) { + defer client.Close() + br := bufio.NewReader(client) + req, err := http.ReadRequest(br) + if err != nil || req.Method != http.MethodConnect { + return + } + atomic.AddInt32(count, 1) + server, err := net.DialTimeout("tcp", req.Host, 5*time.Second) + if err != nil { + io.WriteString(client, "HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + } + defer server.Close() + io.WriteString(client, "HTTP/1.1 200 Connection Established\r\n\r\n") + go io.Copy(server, br) + io.Copy(client, server) +} diff --git a/tools/proxy/hub_traffic.go b/tools/proxy/hub_traffic.go new file mode 100644 index 00000000..0ba85189 --- /dev/null +++ b/tools/proxy/hub_traffic.go @@ -0,0 +1,100 @@ +package proxy + +import ( + "net/http" + "strconv" + "sync" + + traffic "github.com/chainreactors/aiscan/aop/traffic" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func (h *ProxyHub) Store() *FlowStore { return h.store } + +func (h *ProxyHub) ingest(flow Flow) { + if !h.recording.Load() { + return + } + stored := h.store.Add(flow) + h.publish(&stored) +} + +func (h *ProxyHub) publish(flow *Flow) { + if flow == nil { + return + } + h.subsMu.Lock() + if len(h.subs) == 0 { + h.subsMu.Unlock() + return + } + message := flowToProto(flow) + for _, subscriber := range h.subs { + select { + case subscriber <- message: + default: + } + } + h.subsMu.Unlock() +} + +func (h *ProxyHub) Subscribe(buffer int) (<-chan *traffic.Flow, func()) { + if buffer <= 0 { + buffer = 256 + } + channel := make(chan *traffic.Flow, buffer) + h.subsMu.Lock() + if h.subs == nil { + h.subs = make(map[int]chan *traffic.Flow) + } + id := h.nextSub + h.nextSub++ + h.subs[id] = channel + h.subsMu.Unlock() + + var once sync.Once + cancel := func() { + once.Do(func() { + h.subsMu.Lock() + if existing, ok := h.subs[id]; ok { + delete(h.subs, id) + close(existing) + } + h.subsMu.Unlock() + }) + } + return channel, cancel +} + +func flowToProto(flow *Flow) *traffic.Flow { + if flow == nil { + return nil + } + message := &traffic.Flow{ + Id: strconv.Itoa(flow.ID), + ToolId: flow.ToolID, + Method: flow.Method, + Url: flow.URL, + StatusCode: int32(flow.StatusCode), + RequestHeaders: headersToProto(flow.RequestHeaders), + ResponseHeaders: headersToProto(flow.ResponseHeaders), + RequestBody: flow.RequestBodySnip, + ResponseBody: flow.ResponseBodySnip, + Error: flow.Error, + Complete: flow.Error == "" && flow.StatusCode != 0, + } + if !flow.Timestamp.IsZero() { + message.Timestamp = timestamppb.New(flow.Timestamp) + } + return message +} + +func headersToProto(headers http.Header) []*traffic.Header { + var result []*traffic.Header + for name, values := range headers { + for _, value := range values { + result = append(result, &traffic.Header{Name: name, Value: value}) + } + } + return result +} diff --git a/tools/proxy/hub_traffic_test.go b/tools/proxy/hub_traffic_test.go new file mode 100644 index 00000000..ffa82755 --- /dev/null +++ b/tools/proxy/hub_traffic_test.go @@ -0,0 +1,124 @@ +package proxy + +import ( + "context" + "io" + "net/http" + "net/url" + "testing" + "time" +) + +// hubClient builds an HTTP client that routes through the hub with callID as the +// proxy username, mirroring how bash injects the tool-call id as proxy userinfo. +func hubClient(t *testing.T, hub *ProxyHub, callID string) *http.Client { + t.Helper() + u, err := url.Parse(hub.ProxyURL()) + if err != nil { + t.Fatalf("parse hub url: %v", err) + } + if callID != "" { + u.User = url.User(callID) + } + return &http.Client{ + Transport: &http.Transport{Proxy: http.ProxyURL(u), DisableKeepAlives: true}, + Timeout: 5 * time.Second, + } +} + +func getThrough(t *testing.T, client *http.Client, target string) { + t.Helper() + resp, err := client.Get(target) + if err != nil { + t.Fatalf("request through hub: %v", err) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() +} + +func waitForFlows(t *testing.T, store *FlowStore, want int) []Flow { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if flows := store.Query(QueryOpts{}); len(flows) >= want { + return flows + } + time.Sleep(10 * time.Millisecond) + } + return store.Query(QueryOpts{}) +} + +func startHub(t *testing.T, capture bool) *ProxyHub { + t.Helper() + caRoot := t.TempDir() + hub := NewProxyHub(NewState(""), NewFlowStore(1000), caRoot, capture) + if err := hub.Start(caRoot); err != nil { + t.Fatalf("start hub: %v", err) + } + t.Cleanup(func() { hub.Shutdown(context.Background()) }) + return hub +} + +// TestHubStampsToolID verifies the hub attributes a captured flow to the tool- +// call id carried as the proxy username (via the mitmproxy fork's ProxyAuthUser). +func TestHubStampsToolID(t *testing.T) { + target := startTestTarget(64) + defer target.Close() + hub := startHub(t, true) + + getThrough(t, hubClient(t, hub, "tool-abc"), target.URL) + + flows := waitForFlows(t, hub.Store(), 1) + if len(flows) == 0 { + t.Fatal("no flow captured") + } + if flows[0].ToolID != "tool-abc" { + t.Fatalf("ToolID = %q, want %q", flows[0].ToolID, "tool-abc") + } +} + +// TestHubCaptureToggle verifies capture is runtime-mutable: a relay-mode hub +// records nothing until SetCapture turns recording on, without restarting. +func TestHubCaptureToggle(t *testing.T) { + target := startTestTarget(64) + defer target.Close() + hub := startHub(t, false) // relay + addr := hub.ProxyURL() + + getThrough(t, hubClient(t, hub, "tool-1"), target.URL) + time.Sleep(100 * time.Millisecond) + if n := hub.Store().Count(); n != 0 { + t.Fatalf("relay mode recorded %d flows, want 0", n) + } + + hub.SetCapture(true, true) + if hub.ProxyURL() != addr { + t.Fatalf("hub address changed on capture toggle: %q != %q", hub.ProxyURL(), addr) + } + getThrough(t, hubClient(t, hub, "tool-2"), target.URL) + if flows := waitForFlows(t, hub.Store(), 1); len(flows) == 0 { + t.Fatal("no flow captured after enabling capture") + } +} + +// TestHubSubscribe verifies captured flows fan out to subscribers as protocol +// messages carrying the tool-call id. +func TestHubSubscribe(t *testing.T) { + target := startTestTarget(64) + defer target.Close() + hub := startHub(t, true) + + ch, cancel := hub.Subscribe(16) + defer cancel() + + getThrough(t, hubClient(t, hub, "tool-xyz"), target.URL) + + select { + case flow := <-ch: + if flow.GetToolId() != "tool-xyz" { + t.Fatalf("streamed ToolId = %q, want %q", flow.GetToolId(), "tool-xyz") + } + case <-time.After(2 * time.Second): + t.Fatal("no flow received on subscription") + } +} diff --git a/tools/proxy/infra.go b/tools/proxy/infra.go new file mode 100644 index 00000000..edcb62c1 --- /dev/null +++ b/tools/proxy/infra.go @@ -0,0 +1,90 @@ +package proxy + +import ( + "net/url" + "path/filepath" + "strings" + + "github.com/chainreactors/aiscan/core/deps" + "github.com/chainreactors/aiscan/pkg/commands" + "github.com/chainreactors/proxyclient" +) + +// InfraKey carries the long-lived proxy infrastructure from the assembly layer +// into the proxy command factory through the Deps bag. +var InfraKey = deps.NewKey[*Infra]("proxy.infra") + +// Infra bundles the runner-level proxy infrastructure that must exist BEFORE +// tool factories run: the egress State (source of truth), the shared capture +// FlowStore, and the long-lived MITM ProxyHub. Creating it up front lets the +// assembly set Deps.ScannerProxy to the stable hub address, so bash and every +// scanner engine route through the hub uniformly with no factory-order coupling. +type Infra struct { + State *State + Store *FlowStore + Hub *ProxyHub +} + +// InstallInfra creates and starts the proxy infrastructure, points Deps at the +// hub (ScannerProxy + ScannerProxyCA), and stores the Infra in the bag for the +// proxy factory. The originating Deps.ScannerProxy becomes the hub's default +// upstream, so tool → hub → configured-proxy holds. On hub start failure it +// leaves Deps unchanged (tools fall back to the original proxy / direct) and +// still returns the Infra so the factory can register verbs against the State. +// +// capture selects the hub mode (see NewProxyHub): true records traffic (mitm +// on), false is a pure routing relay (mitm off). Routing works in both, so +// `proxy` keeps managing egress either way; only capture is gated. +func InstallInfra(d *commands.Deps, capture bool) (*Infra, error) { + originalProxy := d.ScannerProxy + state := NewState(originalProxy) + + // A clash:// original proxy is a subscription/auto spec, not a single node: + // activate it as the auto dial so the hub's default upstream load-balances. + if strings.HasPrefix(strings.ToUpper(originalProxy), "CLASH://") { + if u, err := url.Parse(originalProxy); err == nil { + if dial, dialErr := proxyclient.NewClient(u); dialErr == nil { + state.SetAutoDial(originalProxy, dial) + } + } + } + + store := NewFlowStore(10000) + caRoot := filepath.Join(d.WorkDir, ".aiscan", "mitm") + hub := NewProxyHub(state, store, caRoot, capture) + + infra := &Infra{State: state, Store: store, Hub: hub} + commands.Provide(d, InfraKey, infra) + + if err := hub.Start(caRoot); err != nil { + return infra, err + } + if hubURL := hub.ProxyURL(); hubURL != "" { + d.ScannerProxy = hubURL + d.ScannerProxyCA = hub.CAPath() // "" while not intercepting; no CA to trust + // Resolve egress per execution from live hub state: tag the proxy URL + // with the tool-call id so captured flows attribute to it, and read the + // CA path fresh so it tracks runtime capture toggles. + d.EgressResolver = func(callID string) (string, string) { + return egressURL(hub.ProxyURL(), callID), hub.CAPath() + } + } + return infra, nil +} + +// egressURL inserts callID as the proxy username so the hub can attribute every +// captured flow on the connection to the originating tool-call. The value is +// percent-encoded by url.String and decoded back to callID by the HTTP client's +// Proxy-Authorization, so it round-trips even with unusual ids. An empty callID +// or base leaves the URL unchanged. +func egressURL(base, callID string) string { + if base == "" || callID == "" { + return base + } + u, err := url.Parse(base) + if err != nil { + return base + } + u.User = url.User(callID) + return u.String() +} diff --git a/tools/proxy/mitm.go b/tools/proxy/mitm.go index 0eef4e6c..392e9a48 100644 --- a/tools/proxy/mitm.go +++ b/tools/proxy/mitm.go @@ -21,16 +21,19 @@ import ( type MitmCommand struct { store *FlowStore + hub *ProxyHub execCommand CommandExecutor registry *commands.CommandRegistry - execMu sync.Mutex } -func NewMitmCommand(reg *commands.CommandRegistry) *MitmCommand { - return &MitmCommand{ - store: NewFlowStore(10000), - registry: reg, +// NewMitmCommand wires the mitm verbs to the long-lived hub's shared FlowStore +// so `mitm flows/analyze/flow` query traffic captured from every tool, not just +// a per-invocation proxy. +func NewMitmCommand(reg *commands.CommandRegistry, store *FlowStore, hub *ProxyHub) *MitmCommand { + if store == nil { + store = NewFlowStore(10000) } + return &MitmCommand{store: store, hub: hub, registry: reg} } func (c *MitmCommand) SetCommandExecutor(fn CommandExecutor) { @@ -40,20 +43,17 @@ func (c *MitmCommand) SetCommandExecutor(fn CommandExecutor) { func (c *MitmCommand) Name() string { return "mitm" } func (c *MitmCommand) Usage() string { - return `mitm - Run a command with MITM traffic capture + return `mitm - Inspect traffic captured from tool execution -Usage: - mitm [args...] Run command with traffic interception - mitm flows [--host X] [--last N] List captured flows from last run - mitm flow Show full flow details - mitm analyze [--host X] [--last N] Summarize captured functional traffic - mitm clear Clear captured flows +Tool traffic is captured automatically (default on). Inspect it with: + mitm flows [--host X] [--status 2xx] [--type json] [--last N] List captured flows + mitm flow Show one flow (headers + bodies) + mitm analyze [--host X] [--last N] Summarize captured traffic + mitm clear Clear the capture store + mitm [args...] Run a command, report flows it added Examples: - mitm scan -i http://example.com --mode quick - mitm spray -i http://target.com - mitm gogo -i 10.0.0.1 -p top2 - mitm flows --last 20 + mitm flows --host example.com --last 20 mitm analyze --host example.com` } @@ -65,6 +65,17 @@ func (c *MitmCommand) Run(ctx context.Context, execution *commands.Execution) (_ return nil, nil } + // In relay mode (config mitm:false) nothing is recorded; steer the model + // away from querying an empty store rather than returning misleading "no + // flows". Routing still works, so passthrough (default) stays allowed. + switch args[0] { + case "flows", "flow", "analyze": + if c.hub != nil && !c.hub.Capturing() { + fmt.Fprint(execution.Stdout, "[mitm] traffic capture is disabled (proxy routing only). Enable with config mitm: true") + return nil, nil + } + } + var result string switch args[0] { @@ -94,45 +105,18 @@ func (c *MitmCommand) execWithCapture(ctx context.Context, args []string, execut if c.execCommand == nil { return nil, fmt.Errorf("mitm: command executor not available") } - - // Scanner commands share mutable proxy configuration. Serialize captured - // executions so one run cannot steal another run's proxy or flows. - c.execMu.Lock() - defer c.execMu.Unlock() - - state := &mitmState{store: c.store} - if err := state.start(); err != nil { - return nil, err - } - - // Set MITM proxy on the target command only - targetName := args[0] - var prevProxy string - if cmd, ok := c.registry.Get(targetName); ok { - if cmd.SetProxy != nil { - if cmd.GetProxy != nil { - prevProxy = cmd.GetProxy() - } - cmd.SetProxy(state.proxyURL()) - defer cmd.SetProxy(prevProxy) - } - } - defer state.stop() - + // Every tool already routes through the long-lived hub, so the wrapped + // command is captured automatically. Report the flows it added. The delta + // is approximate under concurrency (the shared store also receives other + // commands' flows), which is acceptable for this summary. + before := c.store.Count() details, err := c.execCommand(ctx, args, execution) - - flowCount := len(state.Records()) - summary := fmt.Sprintf("\n[mitm] %d flows captured.", flowCount) - fmt.Fprint(execution.Stdout, summary) - return &CaptureResult{Command: details, Flows: state.Records()}, err -} - -// CaptureResult is returned as tool-result details. FlowRecord is the canonical -// immutable traffic snapshot from utils/mitmproxy; callers should persist it -// directly instead of translating it through another flow DTO. -type CaptureResult struct { - Command any `json:"command,omitempty"` - Flows []*mitmproxy.FlowRecord `json:"flows"` + added := c.store.Count() - before + if added < 0 { + added = 0 + } + fmt.Fprintf(execution.Stdout, "\n[mitm] %d flows captured.", added) + return details, err } type flowQueryFlags struct { @@ -178,65 +162,6 @@ func (c *MitmCommand) analyze(args []string) (string, error) { return formatFlowAnalysis(c.store.Query(QueryOpts{Host: f.Host, Last: f.Last})), nil } -// --------------------------------------------------------------------------- -// mitmState — lightweight MITM proxy lifecycle (no exported API needed) -// --------------------------------------------------------------------------- - -type mitmState struct { - server *mitmproxy.Proxy - addr string - store *FlowStore - recordMu sync.Mutex - records []*mitmproxy.FlowRecord -} - -func (s *mitmState) start() error { - p, err := mitmproxy.NewProxy(&mitmproxy.Options{ - Addr: "127.0.0.1:0", - SslInsecure: true, - StreamLargeBodies: 10 * 1024 * 1024, - }) - if err != nil { - return fmt.Errorf("create MITM proxy: %w", err) - } - p.AddAddon(&captureAddon{store: s.store, record: s.addRecord}) - listenAddr, _, err := p.StartAsync() - if err != nil { - return fmt.Errorf("start MITM proxy: %w", err) - } - s.server = p - s.addr = listenAddr.String() - return nil -} - -func (s *mitmState) addRecord(record *mitmproxy.FlowRecord) { - if record == nil { - return - } - s.recordMu.Lock() - s.records = append(s.records, record) - s.recordMu.Unlock() -} - -func (s *mitmState) Records() []*mitmproxy.FlowRecord { - s.recordMu.Lock() - defer s.recordMu.Unlock() - return append([]*mitmproxy.FlowRecord(nil), s.records...) -} - -func (s *mitmState) stop() { - if s.server != nil { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - _ = s.server.Shutdown(ctx) - cancel() - s.server = nil - } -} - -func (s *mitmState) proxyURL() string { - return "http://" + s.addr -} - // --------------------------------------------------------------------------- // captureAddon — passive HTTP flow capture // --------------------------------------------------------------------------- @@ -245,19 +170,25 @@ const maxBodySnip = 4096 type captureAddon struct { mitmproxy.BaseAddon - store *FlowStore - record func(*mitmproxy.FlowRecord) + hub *ProxyHub pending sync.Map } +// toolIDOf returns the AOP tool-call id that opened this flow's connection, read +// from the per-connection proxy-auth username the client injected. Empty when no +// identity was presented (e.g. relay use or a non-Cairn client). +func toolIDOf(f *mitmproxy.Flow) string { + if f != nil && f.ConnContext != nil { + return f.ConnContext.ProxyAuthUser + } + return "" +} + func (a *captureAddon) Requestheaders(f *mitmproxy.Flow) { a.pending.Store(f.Id.String(), time.Now()) } func (a *captureAddon) Response(f *mitmproxy.Flow) { - if a.record != nil { - a.record(mitmproxy.NewFlowRecord(f, 0)) - } var dur time.Duration if start, ok := a.pending.LoadAndDelete(f.Id.String()); ok { if t, ok := start.(time.Time); ok { @@ -266,6 +197,7 @@ func (a *captureAddon) Response(f *mitmproxy.Flow) { } flow := Flow{ Timestamp: f.StartTime, + ToolID: toolIDOf(f), Method: f.Request.Method, URL: f.Request.URL.String(), Host: f.Request.URL.Hostname(), @@ -284,23 +216,19 @@ func (a *captureAddon) Response(f *mitmproxy.Flow) { flow.ResponseBodySnip = snip(f.Response.Body, maxBodySnip) } } - a.store.Add(flow) + a.hub.ingest(flow) } func (a *captureAddon) RequestError(f *mitmproxy.Flow, err error) { - if a.record != nil { - record := mitmproxy.NewFlowRecord(f, 0) - record.Error = err.Error() - a.record(record) - } var dur time.Duration if start, ok := a.pending.LoadAndDelete(f.Id.String()); ok { if t, ok := start.(time.Time); ok { dur = time.Since(t) } } - a.store.Add(Flow{ + a.hub.ingest(Flow{ Timestamp: f.StartTime, + ToolID: toolIDOf(f), Method: f.Request.Method, URL: f.Request.URL.String(), Host: f.Request.URL.Hostname(), @@ -324,6 +252,7 @@ func snip(b []byte, max int) []byte { type Flow struct { ID int + ToolID string Timestamp time.Time Method string URL string @@ -360,7 +289,9 @@ func NewFlowStore(cap int) *FlowStore { return &FlowStore{flows: make([]Flow, 0, 256), cap: cap} } -func (s *FlowStore) Add(f Flow) { +// Add stores f, assigns it a monotonic ID, and returns the stored copy so the +// caller can fan the ID-bearing flow out to subscribers. +func (s *FlowStore) Add(f Flow) Flow { s.mu.Lock() defer s.mu.Unlock() s.seq++ @@ -371,6 +302,7 @@ func (s *FlowStore) Add(f Flow) { } else { s.flows = append(s.flows, f) } + return f } func (s *FlowStore) Query(opts QueryOpts) []Flow { diff --git a/tools/proxy/mitm_test.go b/tools/proxy/mitm_test.go index f9dd2631..e051e947 100644 --- a/tools/proxy/mitm_test.go +++ b/tools/proxy/mitm_test.go @@ -31,6 +31,13 @@ func startTestTarget(bodySize int) *httptest.Server { })) } +// newCapturingHub wraps a store in a recording ProxyHub so a bare captureAddon +// can route flows through hub.ingest in tests without starting the hub's own +// listener (the test attaches the addon to its own proxy). +func newCapturingHub(store *FlowStore) *ProxyHub { + return NewProxyHub(nil, store, "", true) +} + // startMITMProxy creates a MITM proxy with a captureAddon and returns its address. func startMITMProxy(t *testing.T) (*mitmproxy.Proxy, *FlowStore, string) { t.Helper() @@ -43,7 +50,7 @@ func startMITMProxy(t *testing.T) (*mitmproxy.Proxy, *FlowStore, string) { if err != nil { t.Fatal(err) } - p.AddAddon(&captureAddon{store: store}) + p.AddAddon(&captureAddon{hub: newCapturingHub(store)}) addr, _, err := p.StartAsync() if err != nil { t.Fatal(err) @@ -278,7 +285,7 @@ func BenchmarkMITM_HTTPProxy(b *testing.B) { store := NewFlowStore(b.N + 100) p, _ := mitmproxy.NewProxy(&mitmproxy.Options{Addr: "127.0.0.1:0", SslInsecure: true}) - p.AddAddon(&captureAddon{store: store}) + p.AddAddon(&captureAddon{hub: newCapturingHub(store)}) addr, _, _ := p.StartAsync() defer p.Shutdown(context.Background()) @@ -307,7 +314,7 @@ func BenchmarkMITM_CONNECT(b *testing.B) { store := NewFlowStore(b.N + 100) p, _ := mitmproxy.NewProxy(&mitmproxy.Options{Addr: "127.0.0.1:0", SslInsecure: true}) - p.AddAddon(&captureAddon{store: store}) + p.AddAddon(&captureAddon{hub: newCapturingHub(store)}) addr, _, _ := p.StartAsync() defer p.Shutdown(context.Background()) diff --git a/tools/proxy/register_command.go b/tools/proxy/register_command.go index ed14252f..37de3ea0 100644 --- a/tools/proxy/register_command.go +++ b/tools/proxy/register_command.go @@ -3,12 +3,11 @@ package proxy import ( - "net/url" - "strings" + "context" "github.com/chainreactors/aiscan/core/capability" + "github.com/chainreactors/aiscan/core/deps" "github.com/chainreactors/aiscan/pkg/commands" - "github.com/chainreactors/proxyclient" // Register extra proxy protocols so proxyclient.NewClient can handle them. _ "github.com/chainreactors/proxyclient/extra/anytls" @@ -22,25 +21,15 @@ func init() { capability.Register(capability.Descriptor{ID: "proxy", Kind: capability.KindService, Group: "proxy"}) commands.RegisterFactory(commands.Factory{ Capability: "proxy", - Build: func(deps *commands.Deps, reg *commands.CommandRegistry) { - state := NewState(deps.ScannerProxy) + Build: func(d *commands.Deps, reg *commands.CommandRegistry) { + // The infrastructure (State + FlowStore + long-lived MITM hub) is + // created by InstallInfra before BuildPlan so Deps.ScannerProxy + // already points every tool at the stable hub address. Here we only + // register the verbs that observe and steer it. + state, store, hub := resolveInfra(d) + cmd := New(state) - cmd.SetOnProxyChange(func(newProxy string) { - // 1. update BashTool scanner proxy env (for shell commands) - if bt, ok := reg.GetTool("bash"); ok { - if bash, ok := bt.(*commands.BashTool); ok { - bash.SetScannerProxy(newProxy) - } - } - // 2. update individual scanner command proxy fields; - // each command passes proxy to the SDK engine via - // Context.SetProxy / RunOptions.ProxyDial on next execution. - for _, pc := range reg.All() { - if pc.SetProxy != nil { - pc.SetProxy(newProxy) - } - } - }) + cmd.SetHub(hub) cmd.SetCommandExecutor(reg.Run) reg.Register(commands.Command{ Name: cmd.Name(), Usage: cmd.Usage(), @@ -48,24 +37,30 @@ func init() { Run: cmd.Run, }, "proxy") - mitmCmd := NewMitmCommand(reg) + mitmCmd := NewMitmCommand(reg, store, hub) mitmCmd.SetCommandExecutor(reg.Run) reg.Register(commands.Command{ Name: mitmCmd.Name(), Usage: mitmCmd.Usage(), DescriptionPath: "aiscan://skills/aiscan/okf/runtime/mitm.md", Run: mitmCmd.Run, - }, "proxy") - - // If --proxy / config proxy is a clash:// URL, auto-activate - if strings.HasPrefix(strings.ToUpper(deps.ScannerProxy), "CLASH://") { - u, err := url.Parse(deps.ScannerProxy) - if err == nil { - dial, dialErr := proxyclient.NewClient(u) - if dialErr == nil { - state.SetAutoDial(deps.ScannerProxy, dial) + Close: func() { + if hub != nil { + hub.Shutdown(context.Background()) } - } - } + }, + }, "proxy") }, }) } + +// resolveInfra returns the shared proxy infrastructure installed by +// InstallInfra, or a hub-less fallback (direct egress, no capture) for build +// paths — chiefly tests — that register the proxy group without it. +func resolveInfra(d *commands.Deps) (*State, *FlowStore, *ProxyHub) { + if d.Bag != nil { + if infra, ok := deps.Get(d.Bag, InfraKey); ok && infra != nil { + return infra.State, infra.Store, infra.Hub + } + } + return NewState(d.ScannerProxy), NewFlowStore(10000), nil +} diff --git a/tools/proxy/state.go b/tools/proxy/state.go index 43270819..5554e7ba 100644 --- a/tools/proxy/state.go +++ b/tools/proxy/state.go @@ -7,9 +7,11 @@ import ( "io" "net" "net/http" + "net/url" "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/chainreactors/proxyclient" @@ -25,10 +27,18 @@ type State struct { activeURL string autoURL string // clash:// URL for auto mode autoDial proxyclient.Dial // pre-built dial for auto mode + singleDial proxyclient.Dial // pre-built dial for a single persistent proxy URL + + // chain is the composed egress dial for the current selection, republished + // on every state change. ProxyHub's upstream reads it on each connection so + // switching nodes takes effect live without touching in-flight children. + chain atomic.Pointer[proxyclient.Dial] } func NewState(originalProxy string) *State { - return &State{originalProxy: originalProxy} + s := &State{originalProxy: originalProxy} + s.publishChainLocked() + return s } func (s *State) LoadSubscription(sub *clash.Subscription, subscribeURL string) { @@ -68,6 +78,10 @@ func (s *State) Switch(nameOrIndex string) error { } s.activeNode = node s.activeURL = node.URL.String() + s.autoURL = "" + s.autoDial = nil + s.singleDial = nil + s.publishChainLocked() return nil } @@ -80,6 +94,10 @@ func (s *State) Switch(nameOrIndex string) error { } s.activeNode = &nodes[i] s.activeURL = nodes[i].URL.String() + s.autoURL = "" + s.autoDial = nil + s.singleDial = nil + s.publishChainLocked() return nil } } @@ -91,8 +109,33 @@ func (s *State) SetAutoDial(clashURL string, dial proxyclient.Dial) { defer s.mu.Unlock() s.autoURL = clashURL s.autoDial = dial + s.singleDial = nil s.activeNode = nil s.activeURL = "" + s.publishChainLocked() +} + +// SetProxyURL routes egress through a single persistent proxy URL (socks5://, +// trojan://, …). Unlike WithOverrideDial it is not scoped to one command; it +// stays the active egress until changed or cleared. +func (s *State) SetProxyURL(rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return fmt.Errorf("invalid proxy URL: %w", err) + } + d, err := proxyclient.NewClient(u) + if err != nil { + return fmt.Errorf("create proxy client: %w", err) + } + s.mu.Lock() + defer s.mu.Unlock() + s.singleDial = d + s.activeURL = rawURL + s.activeNode = nil + s.autoURL = "" + s.autoDial = nil + s.publishChainLocked() + return nil } func (s *State) ActiveProxy() string { @@ -137,6 +180,8 @@ func (s *State) Clear() { s.activeURL = "" s.autoURL = "" s.autoDial = nil + s.singleDial = nil + s.publishChainLocked() } func (s *State) TestNode(ctx context.Context, node *clash.ProxyNode) (time.Duration, error) { diff --git a/tools/proxy/state_chain.go b/tools/proxy/state_chain.go new file mode 100644 index 00000000..94c19512 --- /dev/null +++ b/tools/proxy/state_chain.go @@ -0,0 +1,54 @@ +package proxy + +import ( + "fmt" + "net/url" + + "github.com/chainreactors/proxyclient" +) + +func (s *State) CurrentDial() proxyclient.Dial { + if pointer := s.chain.Load(); pointer != nil && *pointer != nil { + return *pointer + } + return proxyclient.DefaultDial +} + +func (s *State) WithOverrideDial(proxyURL string) (func(), error) { + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("invalid proxy URL: %w", err) + } + dial, err := proxyclient.NewClient(parsed) + if err != nil { + return nil, fmt.Errorf("create proxy client: %w", err) + } + previous := s.chain.Load() + current := proxyclient.Dial(dial) + s.chain.Store(¤t) + return func() { s.chain.Store(previous) }, nil +} + +func (s *State) publishChainLocked() { + var dial proxyclient.Dial + switch { + case s.autoDial != nil: + dial = s.autoDial + case s.singleDial != nil: + dial = s.singleDial + case s.activeNode != nil && s.activeNode.URL != nil: + if client, err := proxyclient.NewClient(s.activeNode.URL); err == nil { + dial = client + } + case s.originalProxy != "": + if parsed, err := url.Parse(s.originalProxy); err == nil { + if client, err := proxyclient.NewClient(parsed); err == nil { + dial = client + } + } + } + if dial == nil { + dial = proxyclient.DefaultDial + } + s.chain.Store(&dial) +} diff --git a/tools/proxy/state_chain_test.go b/tools/proxy/state_chain_test.go new file mode 100644 index 00000000..d02f6bf5 --- /dev/null +++ b/tools/proxy/state_chain_test.go @@ -0,0 +1,53 @@ +package proxy + +import ( + "testing" + + "github.com/chainreactors/proxyclient" +) + +// dialPtr returns the identity of the currently published chain pointer so a +// swap can be observed without comparing func values (which are not comparable). +func (s *State) dialPtr() *proxyclient.Dial { return s.chain.Load() } + +func TestCurrentDialNeverNil(t *testing.T) { + s := NewState("") + if s.CurrentDial() == nil { + t.Fatal("CurrentDial must never return nil, even with no selection") + } + if s.dialPtr() == nil { + t.Fatal("NewState must publish an initial chain") + } +} + +func TestOriginalProxyBecomesChain(t *testing.T) { + s := NewState("socks5://127.0.0.1:1080") + if s.CurrentDial() == nil { + t.Fatal("original proxy should yield a dial") + } +} + +func TestWithOverrideDialSwapAndRestore(t *testing.T) { + s := NewState("") + base := s.dialPtr() + + restore, err := s.WithOverrideDial("socks5://127.0.0.1:1080") + if err != nil { + t.Fatalf("override failed: %v", err) + } + if s.dialPtr() == base { + t.Fatal("override should republish a different chain pointer") + } + + restore() + if s.dialPtr() != base { + t.Fatal("restore should return to the previous chain pointer") + } +} + +func TestWithOverrideDialRejectsBadURL(t *testing.T) { + s := NewState("") + if _, err := s.WithOverrideDial("://not a url"); err == nil { + t.Fatal("expected error for malformed proxy URL") + } +} diff --git a/tools/proxy/traffic_handler.go b/tools/proxy/traffic_handler.go new file mode 100644 index 00000000..cb6d33c1 --- /dev/null +++ b/tools/proxy/traffic_handler.go @@ -0,0 +1,271 @@ +package proxy + +import ( + "context" + "fmt" + "net/url" + "strconv" + "sync" + "sync/atomic" + + aop "github.com/chainreactors/aiscan/aop" + traffic "github.com/chainreactors/aiscan/aop/traffic" + "github.com/chainreactors/proxyclient" + "github.com/chainreactors/proxyclient/extra/clash" + protobuf "google.golang.org/protobuf/proto" +) + +// TrafficHandler bridges the AOP traffic namespace to the runner's traffic +// infrastructure: it applies Configure (routing + capture) against State/Hub, +// answers Query with a State snapshot or recorded flows, and streams captured +// flows back while a stream is requested. One handler is created per connection +// so its stream lifecycle is tied to that connection. +type TrafficHandler struct { + infra *Infra + + mu sync.Mutex + stopStream func() // cancels the active flow stream, nil when none +} + +// NewTrafficHandler returns a handler backed by infra. infra must be non-nil and +// fully started (hub listening). +func NewTrafficHandler(infra *Infra) *TrafficHandler { + return &TrafficHandler{infra: infra} +} + +// Register installs the traffic namespace on mux. The returned mux routes +// traffic.ProtocolMessage envelopes to this handler. +func (h *TrafficHandler) Register(mux *aop.NamespaceMux) error { + return mux.Register(&traffic.ProtocolMessage{}, func(ctx context.Context, env *aop.Envelope, msg protobuf.Message, send aop.SendFunc) error { + pm, ok := msg.(*traffic.ProtocolMessage) + if !ok { + return fmt.Errorf("traffic: unexpected message %T", msg) + } + return h.handle(ctx, env, pm, send) + }) +} + +// Close tears down any active stream. Call when the connection ends. +func (h *TrafficHandler) Close() { h.stopStreaming() } + +func (h *TrafficHandler) handle(ctx context.Context, env *aop.Envelope, pm *traffic.ProtocolMessage, send aop.SendFunc) error { + switch m := pm.Message.(type) { + case *traffic.ProtocolMessage_Configure: + return h.handleConfigure(ctx, env, m.Configure, send) + case *traffic.ProtocolMessage_Query: + return h.handleQuery(env, m.Query, send) + default: + // State and Flow are outbound-only; ignore if echoed back. + return nil + } +} + +func (h *TrafficHandler) handleConfigure(ctx context.Context, env *aop.Envelope, cfg *traffic.Configure, send aop.SendFunc) error { + var errMsg string + if rc := cfg.GetRouting(); rc != nil { + if err := applyRouting(h.infra.State, rc); err != nil { + errMsg = err.Error() + } + } + if cap := cfg.GetCapture(); cap != nil && cap.GetMode() != traffic.CaptureMode_CAPTURE_MODE_UNSPECIFIED { + record := cap.GetMode() == traffic.CaptureMode_CAPTURE_MODE_RECORD + h.infra.Hub.SetCapture(record, cap.GetDecryptHttps()) + if record && cap.GetStream() { + h.startStream(ctx, env.Id, send) + } else { + h.stopStreaming() + } + } + return h.replyState(env.Id, send, errMsg) +} + +func (h *TrafficHandler) handleQuery(env *aop.Envelope, q *traffic.Query, send aop.SendFunc) error { + if q.GetFlows() { + for _, f := range h.infra.Store.Query(queryOptsFromFilter(q.GetFilter())) { + flow := f + if err := h.sendFlow(env.Id, send, flowToProto(&flow)); err != nil { + return err + } + } + } + // Always answer with a State unless the caller asked only for flows. + if q.GetState() || !q.GetFlows() { + return h.replyState(env.Id, send, "") + } + return nil +} + +// startStream subscribes to the hub and forwards captured flows as Flow messages +// correlated to replyTo until the connection context ends or capture is +// reconfigured. A prior stream is replaced. +func (h *TrafficHandler) startStream(ctx context.Context, replyTo string, send aop.SendFunc) { + ch, cancelSub := h.infra.Hub.Subscribe(256) + streamCtx, cancelCtx := context.WithCancel(ctx) + + h.mu.Lock() + if h.stopStream != nil { + h.stopStream() + } + h.stopStream = func() { + cancelCtx() + cancelSub() + } + h.mu.Unlock() + + go func() { + for { + select { + case <-streamCtx.Done(): + return + case flow, ok := <-ch: + if !ok { + return + } + if err := h.sendFlow(replyTo, send, flow); err != nil { + return + } + } + } + }() +} + +func (h *TrafficHandler) stopStreaming() { + h.mu.Lock() + defer h.mu.Unlock() + if h.stopStream != nil { + h.stopStream() + h.stopStream = nil + } +} + +func (h *TrafficHandler) sendFlow(replyTo string, send aop.SendFunc, flow *traffic.Flow) error { + env, err := aop.Wrap(trafficEnvID(), replyTo, &traffic.ProtocolMessage{ + Message: &traffic.ProtocolMessage_Flow{Flow: flow}, + }) + if err != nil { + return err + } + return send(env) +} + +func (h *TrafficHandler) replyState(replyTo string, send aop.SendFunc, errMsg string) error { + env, err := aop.Wrap(trafficEnvID(), replyTo, &traffic.ProtocolMessage{ + Message: &traffic.ProtocolMessage_State{State: h.snapshot(errMsg)}, + }) + if err != nil { + return err + } + return send(env) +} + +func (h *TrafficHandler) snapshot(errMsg string) *traffic.State { + s := h.infra.State + mode := traffic.CaptureMode_CAPTURE_MODE_RELAY + if h.infra.Hub.Capturing() { + mode = traffic.CaptureMode_CAPTURE_MODE_RECORD + } + return &traffic.State{ + Routing: &traffic.RoutingState{ + ActiveNode: s.ActiveNodeName(), + EgressUrl: s.ActiveProxy(), + Auto: s.IsAutoMode(), + }, + Capture: &traffic.CaptureState{Mode: mode, Capturing: h.infra.Hub.Capturing()}, + Error: errMsg, + } +} + +// applyRouting steers the egress chain per the routing config. UNSPECIFIED +// leaves routing untouched so a capture-only Configure does not disturb it. +func applyRouting(state *State, rc *traffic.RoutingConfig) error { + switch rc.GetMode() { + case traffic.RoutingMode_ROUTING_MODE_UNSPECIFIED: + return nil + case traffic.RoutingMode_ROUTING_MODE_DIRECT, traffic.RoutingMode_ROUTING_MODE_CLEAR: + state.Clear() + return nil + case traffic.RoutingMode_ROUTING_MODE_PROXY: + if rc.GetUrl() == "" { + return fmt.Errorf("routing proxy requires url") + } + return state.SetProxyURL(rc.GetUrl()) + case traffic.RoutingMode_ROUTING_MODE_SUBSCRIBE: + sub, err := clash.FetchSubscriptionWithUA(rc.GetUrl(), clashSubscriptionUA) + if err != nil { + return fmt.Errorf("fetch subscription: %w", err) + } + state.LoadSubscription(sub, rc.GetUrl()) + return nil + case traffic.RoutingMode_ROUTING_MODE_AUTO: + return applyAutoRouting(state, rc) + case traffic.RoutingMode_ROUTING_MODE_SWITCH: + return state.Switch(rc.GetSelector()) + default: + return fmt.Errorf("unknown routing mode %v", rc.GetMode()) + } +} + +// applyAutoRouting mirrors the `proxy auto` verb: fetch the subscription and +// install an adaptive load-balancing clash dial as the persistent egress. +func applyAutoRouting(state *State, rc *traffic.RoutingConfig) error { + if rc.GetUrl() == "" { + return fmt.Errorf("routing auto requires url") + } + sub, err := clash.FetchSubscriptionWithUA(rc.GetUrl(), clashSubscriptionUA) + if err != nil { + return fmt.Errorf("fetch subscription: %w", err) + } + state.LoadSubscription(sub, rc.GetUrl()) + + q := url.Values{} + q.Set("url", rc.GetUrl()) + q.Set("ua", clashSubscriptionUA) + strategy := rc.GetStrategy() + if strategy == "" { + strategy = "adaptive" + } + q.Set("strategy", strategy) + if rc.GetType() != "" { + q.Set("type", rc.GetType()) + } + if rc.GetName() != "" { + q.Set("name", rc.GetName()) + } + if rc.GetCountry() != "" { + q.Set("country", rc.GetCountry()) + } + clashURL := "clash://?" + q.Encode() + u, err := url.Parse(clashURL) + if err != nil { + return fmt.Errorf("build clash url: %w", err) + } + dial, err := proxyclient.NewClient(u) + if err != nil { + return fmt.Errorf("create dialer: %w", err) + } + state.SetAutoDial(clashURL, dial) + return nil +} + +func queryOptsFromFilter(f *traffic.FlowFilter) QueryOpts { + if f == nil { + return QueryOpts{} + } + return QueryOpts{ + Host: f.GetHost(), + Status: f.GetStatus(), + CType: f.GetType(), + Last: int(f.GetLast()), + } +} + +const clashSubscriptionUA = "clash-verge/v2.0.0" + +var trafficEnvSeq atomic.Uint64 + +// trafficEnvID returns a process-unique envelope id for outbound traffic +// replies. A monotonic counter avoids time/random sources (unavailable in some +// hosts) while staying unique within a process. +func trafficEnvID() string { + return "traffic:" + strconv.FormatUint(trafficEnvSeq.Add(1), 36) +} diff --git a/tools/proxy/traffic_handler_test.go b/tools/proxy/traffic_handler_test.go new file mode 100644 index 00000000..6b324c9a --- /dev/null +++ b/tools/proxy/traffic_handler_test.go @@ -0,0 +1,126 @@ +package proxy + +import ( + "context" + "testing" + + aop "github.com/chainreactors/aiscan/aop" + traffic "github.com/chainreactors/aiscan/aop/traffic" +) + +// collectReplies dispatches one envelope through a mux registered for the +// traffic handler and returns every reply the handler sends synchronously. +func dispatchTraffic(t *testing.T, h *TrafficHandler, msg *traffic.ProtocolMessage) []*traffic.ProtocolMessage { + t.Helper() + mux := aop.NewNamespaceMux() + if err := h.Register(mux); err != nil { + t.Fatalf("register: %v", err) + } + env := aop.MustWrap("req-1", "", msg) + var replies []*traffic.ProtocolMessage + _, err := mux.Dispatch(context.Background(), env, func(reply *aop.Envelope) error { + m, err := aop.Unwrap(reply) + if err != nil { + return err + } + if pm, ok := m.(*traffic.ProtocolMessage); ok { + replies = append(replies, pm) + } + return nil + }) + if err != nil { + t.Fatalf("dispatch: %v", err) + } + return replies +} + +func TestTrafficHandlerConfigureCapture(t *testing.T) { + hub := startHub(t, false) // relay + infra := &Infra{State: hub.state, Store: hub.store, Hub: hub} + h := NewTrafficHandler(infra) + defer h.Close() + + if hub.Capturing() { + t.Fatal("hub should start in relay mode") + } + + replies := dispatchTraffic(t, h, &traffic.ProtocolMessage{ + Message: &traffic.ProtocolMessage_Configure{Configure: &traffic.Configure{ + Capture: &traffic.CaptureConfig{Mode: traffic.CaptureMode_CAPTURE_MODE_RECORD, DecryptHttps: true}, + }}, + }) + + if !hub.Capturing() { + t.Fatal("Configure did not enable capture") + } + if len(replies) != 1 { + t.Fatalf("want 1 State reply, got %d", len(replies)) + } + state := replies[0].GetState() + if state == nil { + t.Fatalf("reply is not a State: %#v", replies[0]) + } + if state.GetCapture().GetMode() != traffic.CaptureMode_CAPTURE_MODE_RECORD || !state.GetCapture().GetCapturing() { + t.Fatalf("State capture = %#v, want RECORD/capturing", state.GetCapture()) + } + + // A relay Configure turns capture back off. + dispatchTraffic(t, h, &traffic.ProtocolMessage{ + Message: &traffic.ProtocolMessage_Configure{Configure: &traffic.Configure{ + Capture: &traffic.CaptureConfig{Mode: traffic.CaptureMode_CAPTURE_MODE_RELAY}, + }}, + }) + if hub.Capturing() { + t.Fatal("relay Configure did not disable capture") + } +} + +func TestTrafficHandlerConfigureRoutingProxy(t *testing.T) { + hub := startHub(t, false) + infra := &Infra{State: hub.state, Store: hub.store, Hub: hub} + h := NewTrafficHandler(infra) + defer h.Close() + + replies := dispatchTraffic(t, h, &traffic.ProtocolMessage{ + Message: &traffic.ProtocolMessage_Configure{Configure: &traffic.Configure{ + Routing: &traffic.RoutingConfig{Mode: traffic.RoutingMode_ROUTING_MODE_PROXY, Url: "socks5://127.0.0.1:1080"}, + }}, + }) + if len(replies) != 1 || replies[0].GetState() == nil { + t.Fatalf("want 1 State reply, got %#v", replies) + } + if got := replies[0].GetState().GetRouting().GetEgressUrl(); got != "socks5://127.0.0.1:1080" { + t.Fatalf("egress url = %q, want socks5://127.0.0.1:1080", got) + } +} + +func TestTrafficHandlerQueryFlows(t *testing.T) { + target := startTestTarget(64) + defer target.Close() + hub := startHub(t, true) // record + infra := &Infra{State: hub.state, Store: hub.store, Hub: hub} + h := NewTrafficHandler(infra) + defer h.Close() + + getThrough(t, hubClient(t, hub, "tool-q"), target.URL) + if flows := waitForFlows(t, hub.Store(), 1); len(flows) == 0 { + t.Fatal("no flow captured") + } + + replies := dispatchTraffic(t, h, &traffic.ProtocolMessage{ + Message: &traffic.ProtocolMessage_Query{Query: &traffic.Query{Flows: true}}, + }) + + var flowReplies int + for _, r := range replies { + if f := r.GetFlow(); f != nil { + flowReplies++ + if f.GetToolId() != "tool-q" { + t.Fatalf("queried flow tool_id = %q, want tool-q", f.GetToolId()) + } + } + } + if flowReplies == 0 { + t.Fatal("Query flows returned no Flow replies") + } +} diff --git a/tools/register_command_full_integration_test.go b/tools/register_command_full_integration_test.go index 46e22d58..fdd6d2de 100644 --- a/tools/register_command_full_integration_test.go +++ b/tools/register_command_full_integration_test.go @@ -17,7 +17,6 @@ import ( toolpb "github.com/chainreactors/aiscan/aop/tool" "github.com/chainreactors/aiscan/core/capability" "github.com/chainreactors/aiscan/core/eventbus" - "github.com/chainreactors/aiscan/core/output" "github.com/chainreactors/aiscan/core/telemetry" "github.com/chainreactors/aiscan/pkg/commands" _ "github.com/chainreactors/aiscan/tools/katana" diff --git a/tools/register_command_integration_test.go b/tools/register_command_integration_test.go index f368eb38..b68bc0d0 100644 --- a/tools/register_command_integration_test.go +++ b/tools/register_command_integration_test.go @@ -14,7 +14,6 @@ import ( toolpb "github.com/chainreactors/aiscan/aop/tool" "github.com/chainreactors/aiscan/core/capability" "github.com/chainreactors/aiscan/core/eventbus" - "github.com/chainreactors/aiscan/core/output" "github.com/chainreactors/aiscan/core/resources" "github.com/chainreactors/aiscan/core/telemetry" "github.com/chainreactors/aiscan/pkg/commands" diff --git a/web/frontend/cyber-ui b/web/frontend/cyber-ui index 2a00bac6..de122ac9 160000 --- a/web/frontend/cyber-ui +++ b/web/frontend/cyber-ui @@ -1 +1 @@ -Subproject commit 2a00bac673570fef65535c4753ffc4f6a95ff502 +Subproject commit de122ac9009328959774aa29b3b7a3fe1b2890b1 diff --git a/web/frontend/e2e/user-journey.spec.ts b/web/frontend/e2e/user-journey.spec.ts index 7d0ec22f..3d3eed84 100644 --- a/web/frontend/e2e/user-journey.spec.ts +++ b/web/frontend/e2e/user-journey.spec.ts @@ -105,8 +105,9 @@ test('operator completes a full AIScan Web journey', async ({ page, request }) = await terminalInput.focus() await terminalInput.pressSequentially('/status') await terminalInput.press('Enter') - await expect.poll(async () => page.locator('.xterm-rows').innerText(), { timeout: 20_000 }).toContain('Provider:') - await expect.poll(async () => (await page.locator('.xterm-rows').innerText()).replace(/\s+/g, '')).toContain(E2E_MODEL.replace(/\s+/g, '')) + const compactTerminalText = async () => (await page.locator('.xterm-rows').innerText()).replace(/\s+/g, '') + await expect.poll(compactTerminalText, { timeout: 20_000 }).toContain('Provider:') + await expect.poll(compactTerminalText).toContain(E2E_MODEL.replace(/\s+/g, '')) await page.getByRole('button', { name: 'Show details' }).click() const agentDrawer = page.getByRole('dialog').filter({ hasText: 'Agent Console' })