fuzzing.yml

name: Fuzzing

# spell-checker:ignore (people) taiki-e
# spell-checker:ignore (misc) fuzzer PIPESTATUS

env:
  CARGO_INCREMENTAL: "0"

on:
  pull_request:
    paths: &fuzz_paths
      - 'fuzz/**'
      - 'src/uu/**'
      - 'src/uucore/**'
      - 'src/uucore_procs/**'
      - 'Cargo.toml'
      - 'Cargo.lock'
      - '.github/workflows/fuzzing.yml'
  push:
    branches:
      - '*'
    paths: *fuzz_paths

permissions:
  contents: read # to fetch code (actions/checkout)

# End the current execution if there is a new changeset in the PR.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  detect:
    name: Detect changes on crates
    runs-on: ubuntu-latest
    outputs:
      targets: ${{ steps.detect.outputs.targets }}
      all: ${{ steps.detect.outputs.all }}
    steps:
      - uses: actions/checkout@v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      - id: detect
        shell: bash
        run: |
          # target:should_pass:owning crate (a src/uu package, or "uucore" for
          # the ones that fuzz shared parsing code)
          TARGETS="
          fuzz_test:true:test
          fuzz_date:true:date
          fuzz_expr:true:expr
          fuzz_printf:true:printf
          fuzz_echo:true:echo
          fuzz_parse_glob:true:uucore
          fuzz_parse_size:false:uucore
          fuzz_parse_time:false:uucore
          fuzz_seq_parse_number:false:seq
          fuzz_seq:false:seq
          fuzz_sort:false:sort
          fuzz_wc:false:wc
          fuzz_cut:false:cut
          fuzz_split:false:split
          fuzz_tr:false:tr
          fuzz_env:false:env
          fuzz_cksum:false:cksum
          fuzz_non_utf8_paths:true:uucore
          fuzz_dirname:true:dirname
          "

          if [ "${{ github.event_name }}" = "pull_request" ]; then
            base="${{ github.event.pull_request.base.sha }}"
          else
            # for push diff of the previous
            base="${{ github.event.before }}"
          fi

          if ! git cat-file -e "$base^{commit}" 2>/dev/null; then
            # new branch, force push, ...: no usable base, fuzz everything
            changed=""
            all=true
          else
            changed=$(git diff --name-only "$base" "${{ github.sha }}")
            # anything shared (uucore, the fuzzers themselves, the manifests or
            # this workflow) can affect every target
            if echo "$changed" | grep -qE '^(fuzz/|Cargo\.(toml|lock)$|src/(uucore|uucore_procs)/|\.github/workflows/fuzzing\.yml$)'; then
              all=true
            else
              all=false
            fi
          fi

          if [ "$all" = true ]; then
            selected=$(echo "$TARGETS" | awk -F: 'NF==3')
          else
            crates=$(echo "$changed" | awk -F/ '/^src\/uu\// {print $3}' | sort -u)
            selected=$(echo "$TARGETS" | awk -F: -v crates="$crates" '
              BEGIN { n = split(crates, c, "\n"); for (i = 1; i <= n; i++) want[c[i]] = 1 }
              NF == 3 && ($3 in want)')
          fi

          targets=$(echo "$selected" | jq -Rsc '
            split("\n") | map(select(length > 0) | split(":"))
            | map({name: .[0], should_pass: .[1]})')

          echo "all=$all" >> "$GITHUB_OUTPUT"
          echo "targets=$targets" >> "$GITHUB_OUTPUT"
          echo "Fuzzing everything: $all"
          echo "Targets: $targets"

  uufuzz-examples:
    name: Build and test uufuzz examples
    needs: detect
    if: needs.detect.outputs.all == 'true'
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - uses: Swatinem/rust-cache@v2
      with:
        workspaces: "fuzz -> target"
    - name: Build uufuzz library
      run: |
        cd fuzz/uufuzz
        cargo build --release
    - name: Run uufuzz tests
      run: |
        cd fuzz/uufuzz
        cargo test --lib
    - name: Build and run uufuzz examples
      run: |
        cd fuzz/uufuzz
        echo "Building all examples..."
        cargo build --examples --release

        # Run all examples except integration_testing (which has FD issues in CI)
        for example in examples/*.rs; do
          example_name=$(basename "$example" .rs)
          if [ "$example_name" != "integration_testing" ]; then
            cargo run --example "$example_name" --release
          fi
        done

  fuzz-build:
    name: Build the fuzzers
    needs: detect
    if: needs.detect.outputs.all == 'true'
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - name: Install `cargo-fuzz`
      uses: taiki-e/install-action@v2
      with:
        tool: cargo-fuzz
    - uses: Swatinem/rust-cache@v2
      with:
        workspaces: "fuzz -> target"
    - name: Emulate a nightly toolchain
      run: |
        echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}"
    - name: Run `cargo-fuzz build`
      # Force the correct target
      # https://github.com/rust-fuzz/cargo-fuzz/issues/398
      run: cargo fuzz build --target host-tuple

  fuzz-run:
    name: Fuzz (${{ matrix.target.name }})
    needs: detect
    if: needs.detect.outputs.targets != '[]'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    env:
      RUN_FOR: 60
    strategy:
      fail-fast: false
      # Stagger the jobs: all the targets at once hammer the GitHub cache API.
      max-parallel: 10
      matrix:
        target: ${{ fromJson(needs.detect.outputs.targets) }}

    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - name: Install `cargo-fuzz`
      uses: taiki-e/install-action@v2
      with:
        tool: cargo-fuzz
    - uses: Swatinem/rust-cache@v2
      with:
        workspaces: "fuzz -> target"
    - name: Emulate a nightly toolchain
      run: |
        echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}"
    - name: Run ${{ matrix.target.name }}
      shell: bash
      env:
        TARGET: ${{ matrix.target.name }}
        SHOULD_PASS: ${{ matrix.target.should_pass }}
      run: |
        set -o pipefail
        mkdir -p fuzz/stats

        STATS_FILE="fuzz/stats/${TARGET}.txt"
        # Force the correct target
        # https://github.com/rust-fuzz/cargo-fuzz/issues/398
        set +e
        cargo fuzz run --target host-tuple "$TARGET" -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE"
        FUZZ_STATUS=${PIPESTATUS[0]}
        set -e

        # Save should_pass value to file for summary job to use
        echo "$SHOULD_PASS" > "fuzz/stats/${TARGET}.should_pass"

        fuzz_stat() {
          if grep -q "$1" "$STATS_FILE"; then
            grep "$1" "$STATS_FILE" | awk '{print $2}'
          else
            echo "unknown"
          fi
        }
        if grep -q "SUMMARY: " "$STATS_FILE"; then
          STATUS=$(grep "SUMMARY: " "$STATS_FILE" | head -1)
        else
          STATUS="Completed"
        fi

        # Print stats to job output for immediate visibility
        echo "----------------------------------------"
        echo "FUZZING STATISTICS FOR $TARGET"
        echo "----------------------------------------"
        echo "Runs:           $(fuzz_stat stat::number_of_executed_units)"
        echo "Execution Rate: $(fuzz_stat stat::average_exec_per_sec) execs/sec"
        echo "New Units:      $(fuzz_stat stat::new_units_added)"
        echo "Expected:       $SHOULD_PASS"
        echo "Exit code:      $FUZZ_STATUS"
        echo "Status:         $STATUS"
        echo "----------------------------------------"

        # Add summary to GitHub step summary
        {
          echo "### Fuzzing Results for $TARGET"
          echo ""
          echo "| Metric | Value |"
          echo "|--------|-------|"
          echo "| Runs | $(fuzz_stat stat::number_of_executed_units) |"
          echo "| Execution Rate | $(fuzz_stat stat::average_exec_per_sec) execs/sec |"
          echo "| New Units | $(fuzz_stat stat::new_units_added) |"
          echo "| Should pass | $SHOULD_PASS |"
          echo "| Exit code | $FUZZ_STATUS |"
          echo "| Status | $STATUS |"
          echo ""
        } >> "$GITHUB_STEP_SUMMARY"

        if [ "$SHOULD_PASS" = "true" ] && [ "$FUZZ_STATUS" -ne 0 ]; then
          exit 1
        fi
    - name: Upload Stats
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: fuzz-stats-${{ matrix.target.name }}
        path: |
          fuzz/stats/*.txt
          fuzz/stats/*.should_pass
        retention-days: 5
  fuzz-summary:
    needs: [detect, fuzz-run]
    name: Fuzzing Summary
    runs-on: ubuntu-latest
    if: always() && needs.detect.outputs.targets != '[]'
    steps:
      - uses: actions/checkout@v7.0.1
        with:
          persist-credentials: false
      - name: Download all stats
        uses: actions/download-artifact@v8
        with:
          path: fuzz/stats-artifacts
          pattern: fuzz-stats-*
          merge-multiple: true
      - name: Prepare stats directory
        run: |
          mkdir -p fuzz/stats
          # Debug: List content of stats-artifacts directory
          echo "Contents of stats-artifacts directory:"
          find fuzz/stats-artifacts -type f | sort

          # Extract files from the artifact directories - handle nested directories
          find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \;
          find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \;

          # Debug information
          echo "Contents of stats directory after extraction:"
          ls -la fuzz/stats/
          echo "Contents of should_pass files (if any):"
          cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found"
      - name: Generate Summary
        run: |
          echo "# Fuzzing Summary" > fuzzing_summary.md
          echo "" >> fuzzing_summary.md
          echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md
          echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md

          TOTAL_RUNS=0
          TOTAL_NEW_UNITS=0

          for stat_file in fuzz/stats/*.txt; do
            TARGET=$(basename "$stat_file" .txt)
            SHOULD_PASS_FILE="${stat_file%.*}.should_pass"

            # Get expected status
            if [ -f "$SHOULD_PASS_FILE" ]; then
              EXPECTED=$(cat "$SHOULD_PASS_FILE")
            else
              EXPECTED="unknown"
            fi

            # Extract runs
            if grep -q "stat::number_of_executed_units" "$stat_file"; then
              RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}')
              TOTAL_RUNS=$((TOTAL_RUNS + RUNS))
            else
              RUNS="unknown"
            fi

            # Extract execution rate
            if grep -q "stat::average_exec_per_sec" "$stat_file"; then
              EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}')
            else
              EXEC_RATE="unknown"
            fi

            # Extract new units added
            if grep -q "stat::new_units_added" "$stat_file"; then
              NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}')
              if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then
                TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS))
              fi
            else
              NEW_UNITS="unknown"
            fi

            # Extract status
            if grep -q "SUMMARY: " "$stat_file"; then
              STATUS=$(grep "SUMMARY: " "$stat_file" | head -1)
            else
              STATUS="Completed"
            fi

            echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md
          done

          echo "" >> fuzzing_summary.md
          echo "## Overall Statistics" >> fuzzing_summary.md
          echo "" >> fuzzing_summary.md
          echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md
          echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md
          echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md

          # Add count by expected status
          echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md
          echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md

          # Write to GitHub step summary
          cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY
      - name: Show Summary
        run: |
          cat fuzzing_summary.md
      - name: Upload Summary
        uses: actions/upload-artifact@v7
        with:
          name: fuzzing-summary
          path: fuzzing_summary.md
          retention-days: 5