From ed8df069cb3da75e3fec0316e45be1e5b6730bd6 Mon Sep 17 00:00:00 2001 From: Alexander Karan Date: Sun, 22 Feb 2026 08:32:18 +0800 Subject: [PATCH] Possible Canary Version --- .github/workflows/generate-stats.yml | 130 +---------------- .github/workflows/measure-framework.yml | 137 ++++++++++++++++++ .github/workflows/preview-stats.yml | 126 ++++++++++++++++ .github/workflows/validate-stats.yml | 1 + packages/stats-generator/package.json | 1 + .../src/generate-preview-comment.ts | 70 +++++++++ 6 files changed, 343 insertions(+), 122 deletions(-) create mode 100644 .github/workflows/measure-framework.yml create mode 100644 .github/workflows/preview-stats.yml create mode 100644 packages/stats-generator/src/generate-preview-comment.ts diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index 5b9f148..b9fbffb 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -36,131 +36,17 @@ jobs: echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT echo "ssr=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssr"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - measure-install: + measure: needs: setup - if: needs.setup.outputs.install-matrix != '[]' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ${{ fromJson(needs.setup.outputs.install-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '24' - - - name: Install stats-generator dependencies - run: pnpm install --frozen-lockfile --filter @framework-tracker/stats-generator - - - name: Run install benchmark - run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "install") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:install ${{ matrix.framework.package }} $RUN_FREQUENCY - - - name: Upload install stats - uses: actions/upload-artifact@v4 - with: - name: install-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/install-stats.json - retention-days: 1 - if-no-files-found: error - - measure-build: - needs: setup - if: needs.setup.outputs.build-matrix != '[]' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ${{ fromJson(needs.setup.outputs.build-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile --ignore-workspace - - - name: Run build benchmark - run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "build") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:build ${{ matrix.framework.package }} $RUN_FREQUENCY - - - name: Upload build stats - uses: actions/upload-artifact@v4 - with: - name: build-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/build-stats.json - retention-days: 1 - if-no-files-found: error - - measure-ssr: - needs: setup - if: needs.setup.outputs.ssr-matrix != '[]' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ${{ fromJson(needs.setup.outputs.ssr-matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '24' - cache: 'pnpm' - - - name: Install workspace dependencies - run: pnpm install --frozen-lockfile - - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile --ignore-workspace - - - name: Build app - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm build - - - name: Run SSR benchmark - run: pnpm --filter @framework-tracker/stats-generator run:ssr ${{ matrix.framework.package }} - env: - RUNNER_LABEL: ubuntu-latest - - - name: Upload SSR stats - uses: actions/upload-artifact@v4 - with: - name: ssr-stats-${{ matrix.framework.name }} - path: packages/${{ matrix.framework.package }}/ci-stats.json - retention-days: 1 - if-no-files-found: error + uses: ./.github/workflows/measure-framework.yml + with: + install-matrix: ${{ needs.setup.outputs.install-matrix }} + build-matrix: ${{ needs.setup.outputs.build-matrix }} + ssr-matrix: ${{ needs.setup.outputs.ssr-matrix }} generate-stats: - needs: [setup, measure-install, measure-build, measure-ssr] - if: always() && needs.setup.result == 'success' && (needs.measure-install.result == 'success' || needs.measure-build.result == 'success' || needs.measure-ssr.result == 'success') + needs: [setup, measure] + if: always() && needs.setup.result == 'success' && needs.measure.result == 'success' runs-on: ubuntu-latest permissions: contents: write diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml new file mode 100644 index 0000000..01a6d45 --- /dev/null +++ b/.github/workflows/measure-framework.yml @@ -0,0 +1,137 @@ +name: Measure Framework + +on: + workflow_call: + inputs: + install-matrix: + description: 'JSON array of frameworks to measure install time' + type: string + required: true + build-matrix: + description: 'JSON array of frameworks to measure build time' + type: string + required: true + ssr-matrix: + description: 'JSON array of frameworks to measure SSR performance' + type: string + required: true + +jobs: + measure-install: + if: inputs.install-matrix != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ${{ fromJson(inputs.install-matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + + - name: Install stats-generator dependencies + run: pnpm install --frozen-lockfile --filter @framework-tracker/stats-generator + + - name: Run install benchmark + run: | + RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "install") | .runFrequency') + pnpm --filter @framework-tracker/stats-generator run:install ${{ matrix.framework.package }} $RUN_FREQUENCY + + - name: Upload install stats + uses: actions/upload-artifact@v4 + with: + name: install-stats-${{ matrix.framework.name }} + path: packages/${{ matrix.framework.package }}/install-stats.json + retention-days: 1 + if-no-files-found: error + + measure-build: + if: inputs.build-matrix != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ${{ fromJson(inputs.build-matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + + - name: Install workspace dependencies + run: pnpm install --frozen-lockfile + + - name: Install package dependencies + working-directory: ./packages/${{ matrix.framework.package }} + run: pnpm install --frozen-lockfile --ignore-workspace + + - name: Run build benchmark + run: | + RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "build") | .runFrequency') + pnpm --filter @framework-tracker/stats-generator run:build ${{ matrix.framework.package }} $RUN_FREQUENCY + + - name: Upload build stats + uses: actions/upload-artifact@v4 + with: + name: build-stats-${{ matrix.framework.name }} + path: packages/${{ matrix.framework.package }}/build-stats.json + retention-days: 1 + if-no-files-found: error + + measure-ssr: + if: inputs.ssr-matrix != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ${{ fromJson(inputs.ssr-matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + + - name: Install workspace dependencies + run: pnpm install --frozen-lockfile + + - name: Install package dependencies + working-directory: ./packages/${{ matrix.framework.package }} + run: pnpm install --frozen-lockfile --ignore-workspace + + - name: Build app + working-directory: ./packages/${{ matrix.framework.package }} + run: pnpm build + + - name: Run SSR benchmark + run: pnpm --filter @framework-tracker/stats-generator run:ssr ${{ matrix.framework.package }} + env: + RUNNER_LABEL: ubuntu-latest + + - name: Upload SSR stats + uses: actions/upload-artifact@v4 + with: + name: ssr-stats-${{ matrix.framework.name }} + path: packages/${{ matrix.framework.package }}/ci-stats.json + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/preview-stats.yml b/.github/workflows/preview-stats.yml new file mode 100644 index 0000000..65d28b1 --- /dev/null +++ b/.github/workflows/preview-stats.yml @@ -0,0 +1,126 @@ +name: Preview Stats + +on: + workflow_dispatch: + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + install-matrix: ${{ steps.detect.outputs.install }} + build-matrix: ${{ steps.detect.outputs.build }} + ssr-matrix: ${{ steps.detect.outputs.ssr }} + framework-names: ${{ steps.detect.outputs.framework-names }} + has-changes: ${{ steps.detect.outputs.has-changes }} + pr-number: ${{ steps.detect.outputs.pr-number }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect PR, changed frameworks, and build matrices + id: detect + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_INFO=$(gh pr view --json number,baseRefName 2>/dev/null) + if [[ -z "$PR_INFO" ]]; then + echo "::error::No open PR found for branch ${{ github.ref_name }}" + exit 1 + fi + PR_NUMBER=$(echo "$PR_INFO" | jq -r '.number') + BASE_REF=$(echo "$PR_INFO" | jq -r '.baseRefName') + echo "pr-number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "Detected PR #$PR_NUMBER targeting $BASE_REF" + + FRAMEWORKS=$(cat .github/frameworks.json) + + CHANGED_FILES=$(git diff --name-only origin/$BASE_REF...HEAD -- 'packages/starter-*/package.json' 'packages/app-*/package.json') + + if [[ -z "$CHANGED_FILES" ]]; then + echo "No framework package.json files changed, skipping" + echo "has-changes=false" >> $GITHUB_OUTPUT + echo "framework-names=[]" >> $GITHUB_OUTPUT + echo "install=[]" >> $GITHUB_OUTPUT + echo "build=[]" >> $GITHUB_OUTPUT + echo "ssr=[]" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Changed files:" + echo "$CHANGED_FILES" + + CHANGED_PACKAGES=$(echo "$CHANGED_FILES" | grep -oE 'packages/[^/]+' | sed 's|packages/||' | sort -u) + + echo "Changed packages:" + echo "$CHANGED_PACKAGES" + + NAMES_JSON=$(echo "$CHANGED_PACKAGES" | while read -r PKG; do + echo "$FRAMEWORKS" | jq -r --arg pkg "$PKG" '.[] | select(.starter.package == $pkg or .app.package == $pkg) | .name' + done | sort -u | jq -R . | jq -s -c .) + + echo "Detected frameworks: $NAMES_JSON" + echo "has-changes=true" >> $GITHUB_OUTPUT + echo "framework-names=$NAMES_JSON" >> $GITHUB_OUTPUT + + echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssr"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + + measure: + needs: setup + if: needs.setup.result == 'success' && needs.setup.outputs.has-changes == 'true' + uses: ./.github/workflows/measure-framework.yml + with: + install-matrix: ${{ needs.setup.outputs.install-matrix }} + build-matrix: ${{ needs.setup.outputs.build-matrix }} + ssr-matrix: ${{ needs.setup.outputs.ssr-matrix }} + + preview-comment: + needs: [setup, measure] + if: always() && needs.setup.result == 'success' && needs.setup.outputs.has-changes == 'true' && needs.measure.result == 'success' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: List downloaded artifacts + run: find artifacts -type f -name "*.json" 2>/dev/null || echo "No artifacts found" + + - name: Save CI stats + run: pnpm --filter @framework-tracker/stats-generator save:ci-stats $GITHUB_WORKSPACE/artifacts + env: + RUNNER_LABEL: ubuntu-latest + + - name: Collect stats for docs + run: pnpm collect:stats + + - name: Generate comment + run: | + FRAMEWORK_ARGS=$(echo '${{ needs.setup.outputs.framework-names }}' | jq -r '.[]' | tr '\n' ' ') + pnpm --filter @framework-tracker/stats-generator generate:preview-comment $FRAMEWORK_ARGS + + - name: Post preview comment + run: gh pr comment ${{ needs.setup.outputs.pr-number }} --body-file comment.md + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml index 867436a..8864fb1 100644 --- a/.github/workflows/validate-stats.yml +++ b/.github/workflows/validate-stats.yml @@ -5,6 +5,7 @@ on: paths: - 'packages/stats-generator/**' - '.github/workflows/generate-stats.yml' + - '.github/workflows/measure-framework.yml' - '.github/workflows/validate-stats.yml' - '.github/frameworks.json' - 'packages/app-*/**' diff --git a/packages/stats-generator/package.json b/packages/stats-generator/package.json index d02db6f..864d3ad 100644 --- a/packages/stats-generator/package.json +++ b/packages/stats-generator/package.json @@ -9,6 +9,7 @@ "run:install": "node src/run-install-benchmark.ts", "run:build": "node src/run-build-benchmark.ts", "save:ci-stats": "node src/save-ci-stats.ts", + "generate:preview-comment": "node src/generate-preview-comment.ts", "validate": "node src/validate-stats.ts", "sync:versions": "node src/sync-versions.ts", "lint": "eslint .", diff --git a/packages/stats-generator/src/generate-preview-comment.ts b/packages/stats-generator/src/generate-preview-comment.ts new file mode 100644 index 0000000..ca25d2e --- /dev/null +++ b/packages/stats-generator/src/generate-preview-comment.ts @@ -0,0 +1,70 @@ +import { execFileSync } from 'node:child_process' +import { writeFileSync } from 'node:fs' +import { join } from 'node:path' + +const MAX_DIFF_LINES = 500 +const COMMENT_MARKER = '' + +async function main() { + const frameworkNames = process.argv.slice(2) + + if (frameworkNames.length === 0) { + console.error( + 'Usage: generate:preview-comment [framework-name...]', + ) + process.exit(1) + } + + const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], { + encoding: 'utf-8', + }).trim() + + const fullDiff = execFileSync('git', ['diff', '--', 'packages/'], { + cwd: repoRoot, + encoding: 'utf-8', + }) + + const diffLines = fullDiff.trimEnd().split('\n').filter(Boolean) + const truncated = diffLines.length > MAX_DIFF_LINES + const diffSlice = diffLines.slice(0, MAX_DIFF_LINES).join('\n') + + const frameworkDisplay = frameworkNames.map((n) => `\`${n}\``).join(', ') + + const lines: string[] = [ + COMMENT_MARKER, + '', + `## Stats Preview: ${frameworkDisplay}`, + '', + `Stats were measured on this PR branch for ${frameworkDisplay}. Below is the diff against the base branch.`, + '', + '
', + 'View stats diff', + '', + ] + + if (!diffSlice) { + lines.push('No stats changes detected.') + } else { + lines.push('```diff') + lines.push(diffSlice) + lines.push('```') + if (truncated) { + lines.push('') + lines.push( + `_(Diff truncated at ${MAX_DIFF_LINES} lines — ${diffLines.length} total lines)_`, + ) + } + } + + lines.push('') + lines.push('
') + + const outputPath = join(repoRoot, 'comment.md') + writeFileSync(outputPath, lines.join('\n')) + console.info(`✓ Comment written to ${outputPath}`) +} + +main().catch((error) => { + console.error('Failed to generate preview comment:', error) + process.exit(1) +})