diff --git a/.changelog/v0.3.x.md b/.changelog/v0.3.21.md similarity index 98% rename from .changelog/v0.3.x.md rename to .changelog/v0.3.21.md index d0905b39..05806ac1 100644 --- a/.changelog/v0.3.x.md +++ b/.changelog/v0.3.21.md @@ -1,4 +1,4 @@ -# Release v0.3.x - SQLite Storage Layer & Migration Framework +# Release v0.3.21 - SQLite Storage Layer & Migration Framework Released: YYYY-MM-DD @@ -160,4 +160,4 @@ pm2 restart ecosystem.config.cjs ## 🔗 Full Changelog -**Full Diff**: https://github.com/atomantic/SparseTree/compare/v0.2.11...v0.3.x +**Full Diff**: https://github.com/atomantic/SparseTree/compare/v0.2.11...v0.3.21 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d794411b..b3b5f3f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,72 +1,166 @@ name: CI on: - pull_request: - branches: [main, dev] push: branches: [dev] + pull_request: + branches: [main] + +# Cancel duplicate runs for the same branch/PR +concurrency: + group: ci-${{ github.head_ref || github.ref }} + cancel-in-progress: true permissions: contents: write jobs: - test: + build: + name: Build runs-on: ubuntu-latest - # Skip if commit message contains [skip ci] if: "!contains(github.event.head_commit.message, '[skip ci]')" - strategy: - matrix: - node-version: [20.x] + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build all packages + run: npm run build + + - name: Cache build artifacts + uses: actions/cache/save@v4 + with: + path: | + server/dist + client/dist + shared/types + key: build-${{ github.sha }} + + unit-tests: + name: Unit Tests + runs-on: ubuntu-latest + needs: build + if: "!contains(github.event.head_commit.message, '[skip ci]')" steps: - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} + - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: ${{ matrix.node-version }} + node-version: '20' cache: 'npm' - - name: Install all dependencies - run: npm run install:all + - name: Install dependencies + run: npm ci + + - name: Restore build artifacts + uses: actions/cache/restore@v4 + with: + path: | + server/dist + client/dist + shared/types + key: build-${{ github.sha }} + + - name: Run unit tests + run: npm run test:unit + + - name: Upload coverage + uses: codecov/codecov-action@v4 + if: always() + with: + files: ./coverage/lcov.info + fail_ci_if_error: false + + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + needs: build + if: "!contains(github.event.head_commit.message, '[skip ci]')" - - name: Build shared types - run: npm run build -w shared + steps: + - uses: actions/checkout@v4 - - name: Build server - run: npm run build -w server + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' - - name: Build client - run: npm run build -w client + - name: Install dependencies + run: npm ci - lint: + - name: Restore build artifacts + uses: actions/cache/restore@v4 + with: + path: | + server/dist + client/dist + shared/types + key: build-${{ github.sha }} + + - name: Run integration tests + run: npm run test:integration + + scraper-tests: + name: Scraper Tests runs-on: ubuntu-latest + needs: build if: "!contains(github.event.head_commit.message, '[skip ci]')" steps: - uses: actions/checkout@v4 - - name: Use Node.js 20.x + - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: '20' cache: 'npm' - name: Install dependencies - run: npm run install:all + run: npm ci + + - name: Restore build artifacts + uses: actions/cache/restore@v4 + with: + path: | + server/dist + client/dist + shared/types + key: build-${{ github.sha }} - - name: Build shared types - run: npm run build -w shared + - name: Install Playwright browsers + run: npx playwright install chromium - - name: Check for TypeScript errors - run: npm run build -w server && npm run build -w client + - name: Run scraper tests + run: npm run test:scraper - bump-build: + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: scraper-test-results + path: test-results/ + retention-days: 7 + + bump-version: + name: Bump Version runs-on: ubuntu-latest - needs: [test, lint] - # Only run on push to dev (not PRs), and skip if already a version bump commit - if: github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '[skip ci]') + needs: [unit-tests, integration-tests, scraper-tests] + # Only on push to dev, not PRs, and only if all tests passed + if: | + github.event_name == 'push' && + github.ref == 'refs/heads/dev' && + !contains(github.event.head_commit.message, '[skip ci]') steps: - uses: actions/checkout@v4 @@ -80,25 +174,18 @@ jobs: - name: Bump patch version run: | - # Get current version CURRENT_VERSION=$(node -p "require('./package.json').version") - - # Split into parts MAJOR=$(echo $CURRENT_VERSION | cut -d. -f1) MINOR=$(echo $CURRENT_VERSION | cut -d. -f2) PATCH=$(echo $CURRENT_VERSION | cut -d. -f3) - - # Increment patch NEW_PATCH=$((PATCH + 1)) NEW_VERSION="$MAJOR.$MINOR.$NEW_PATCH" - # Update package.json files npm version $NEW_VERSION --no-git-tag-version cd shared && npm version $NEW_VERSION --no-git-tag-version && cd .. cd client && npm version $NEW_VERSION --no-git-tag-version && cd .. cd server && npm version $NEW_VERSION --no-git-tag-version && cd .. - # Commit and push git add package.json package-lock.json shared/package.json client/package.json server/package.json git commit -m "build: bump version to $NEW_VERSION [skip ci]" git push diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7fbdd61b..10086932 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -134,9 +134,12 @@ jobs: git commit -m "docs: archive changelog for v${CURRENT_VERSION} [skip ci]" git push origin dev + # Store the changelog commit SHA before switching branches + CHANGELOG_COMMIT=$(git rev-parse HEAD) + # Merge changelog back to main (without triggering CI) git checkout main - git cherry-pick HEAD~1 + git cherry-pick "$CHANGELOG_COMMIT" git push origin main git checkout dev fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 20b658cd..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Tests - -on: - push: - branches: [dev, main] - pull_request: - branches: [dev, main] - -jobs: - unit-tests: - name: Unit Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run unit tests - run: npm run test:unit - - - name: Upload coverage - uses: codecov/codecov-action@v4 - if: always() - with: - files: ./coverage/lcov.info - fail_ci_if_error: false - - integration-tests: - name: Integration Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run integration tests - run: npm run test:integration - - scraper-tests: - name: Scraper Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Install Playwright browsers - run: npx playwright install chromium - - - name: Run scraper tests - run: npm run test:scraper - - - name: Upload test results - uses: actions/upload-artifact@v4 - if: always() - with: - name: scraper-test-results - path: test-results/ - retention-days: 7 - - all-tests: - name: All Tests Summary - runs-on: ubuntu-latest - needs: [unit-tests, integration-tests, scraper-tests] - if: always() - - steps: - - name: Check test results - run: | - if [[ "${{ needs.unit-tests.result }}" != "success" ]] || \ - [[ "${{ needs.integration-tests.result }}" != "success" ]] || \ - [[ "${{ needs.scraper-tests.result }}" != "success" ]]; then - echo "One or more test jobs failed" - exit 1 - fi - echo "All tests passed!" diff --git a/client/package.json b/client/package.json index 8776c5a8..4e814fd5 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "@fsf/client", - "version": "0.3.21", + "version": "0.3.24", "type": "module", "scripts": { "dev": "vite --port 6373", diff --git a/package-lock.json b/package-lock.json index 1e93bbbf..18f8ea81 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "sparsetree", - "version": "0.3.21", + "version": "0.3.24", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sparsetree", - "version": "0.3.21", + "version": "0.3.24", "license": "ISC", "workspaces": [ "shared", @@ -59,7 +59,7 @@ }, "client": { "name": "@fsf/client", - "version": "0.3.21", + "version": "0.3.24", "dependencies": { "@fsf/shared": "*", "d3": "^7.9.0", @@ -420,13 +420,11 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "aix" ], - "peer": true, "engines": { "node": ">=18" } @@ -438,13 +436,11 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "android" ], - "peer": true, "engines": { "node": ">=18" } @@ -456,13 +452,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "android" ], - "peer": true, "engines": { "node": ">=18" } @@ -474,13 +468,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "android" ], - "peer": true, "engines": { "node": ">=18" } @@ -492,13 +484,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" ], - "peer": true, "engines": { "node": ">=18" } @@ -510,13 +500,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" ], - "peer": true, "engines": { "node": ">=18" } @@ -528,13 +516,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "freebsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -546,13 +532,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "freebsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -564,13 +548,11 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -582,13 +564,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -600,13 +580,11 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -618,13 +596,11 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -636,13 +612,11 @@ "cpu": [ "mips64el" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -654,13 +628,11 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -672,13 +644,11 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -690,13 +660,11 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -708,13 +676,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -726,13 +692,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "netbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -744,13 +708,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "netbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -762,13 +724,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "openbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -780,13 +740,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "openbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -798,13 +756,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "openharmony" ], - "peer": true, "engines": { "node": ">=18" } @@ -816,13 +772,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "sunos" ], - "peer": true, "engines": { "node": ">=18" } @@ -834,13 +788,11 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ], - "peer": true, "engines": { "node": ">=18" } @@ -852,13 +804,11 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ], - "peer": true, "engines": { "node": ">=18" } @@ -870,13 +820,11 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ], - "peer": true, "engines": { "node": ">=18" } @@ -3964,7 +3912,6 @@ "version": "0.27.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", "integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==", - "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -4360,7 +4307,6 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -4439,7 +4385,6 @@ "version": "4.13.0", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", - "dev": true, "license": "MIT", "dependencies": { "resolve-pkg-maps": "^1.0.0" @@ -6257,7 +6202,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" @@ -7308,7 +7252,6 @@ "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", - "dev": true, "license": "MIT", "dependencies": { "esbuild": "~0.27.0", @@ -8417,7 +8360,7 @@ }, "server": { "name": "@fsf/server", - "version": "0.3.21", + "version": "0.3.24", "dependencies": { "@fsf/shared": "*", "better-sqlite3": "^12.6.2", @@ -8426,6 +8369,7 @@ "playwright": "^1.57.0", "portos-ai-toolkit": "^0.1.0", "socket.io": "^4.8.3", + "tsx": "^4.19.2", "ulid": "^3.0.2", "zod": "^3.24.1" }, @@ -8434,13 +8378,12 @@ "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^22.10.2", - "tsx": "^4.19.2", "typescript": "^5.7.2" } }, "shared": { "name": "@fsf/shared", - "version": "0.3.21", + "version": "0.3.24", "devDependencies": { "typescript": "^5.7.2" } diff --git a/package.json b/package.json index c23b3667..11fd913c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sparsetree", - "version": "0.3.21", + "version": "0.3.24", "private": true, "description": "", "main": "index.js", @@ -20,7 +20,7 @@ "test": "vitest", "test:unit": "vitest run tests/unit", "test:integration": "vitest run tests/integration", - "test:scraper": "playwright test tests/scraper", + "test:scraper": "playwright test --config playwright.scraper.config.ts", "test:e2e": "playwright test tests/e2e", "test:ci": "vitest run --coverage", "test:watch": "vitest watch", diff --git a/playwright.config.ts b/playwright.config.ts index cff81e43..61cd5947 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,10 +1,11 @@ import { defineConfig, devices } from '@playwright/test'; /** - * Playwright configuration for scraper and E2E tests + * Playwright configuration for E2E tests + * For scraper tests, use playwright.scraper.config.ts */ export default defineConfig({ - testDir: './tests', + testDir: './tests/e2e', fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, @@ -19,17 +20,8 @@ export default defineConfig({ }, projects: [ - { - name: 'scraper-tests', - testMatch: 'tests/scraper/**/*.spec.ts', - use: { - ...devices['Desktop Chrome'], - headless: true, - }, - }, { name: 'e2e-tests', - testMatch: 'tests/e2e/**/*.spec.ts', use: { ...devices['Desktop Chrome'], headless: true, diff --git a/playwright.scraper.config.ts b/playwright.scraper.config.ts new file mode 100644 index 00000000..4e4bb0d6 --- /dev/null +++ b/playwright.scraper.config.ts @@ -0,0 +1,31 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Playwright configuration for scraper tests only + * These tests use their own mock servers, no webServer needed + */ +export default defineConfig({ + testDir: './tests/scraper', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: 'html', + timeout: 30000, + + use: { + trace: 'on-first-retry', + screenshot: 'only-on-failure', + }, + + projects: [ + { + name: 'scraper-tests', + use: { + ...devices['Desktop Chrome'], + headless: true, + }, + }, + ], + // No webServer - scraper tests use mock servers +}); diff --git a/server/package.json b/server/package.json index f478a906..997b8de3 100644 --- a/server/package.json +++ b/server/package.json @@ -1,12 +1,12 @@ { "name": "@fsf/server", - "version": "0.3.21", + "version": "0.3.24", "type": "module", "main": "dist/index.js", "scripts": { "dev": "tsx watch src/index.ts", "build": "tsc && cp src/db/schema.sql dist/db/", - "start": "node dist/index.js" + "start": "tsx src/index.ts" }, "dependencies": { "@fsf/shared": "*", @@ -16,6 +16,7 @@ "playwright": "^1.57.0", "portos-ai-toolkit": "^0.1.0", "socket.io": "^4.8.3", + "tsx": "^4.19.2", "ulid": "^3.0.2", "zod": "^3.24.1" }, @@ -24,7 +25,6 @@ "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^22.10.2", - "tsx": "^4.19.2", "typescript": "^5.7.2" } } diff --git a/shared/package.json b/shared/package.json index 8d00fca3..36645dd5 100644 --- a/shared/package.json +++ b/shared/package.json @@ -1,6 +1,6 @@ { "name": "@fsf/shared", - "version": "0.3.21", + "version": "0.3.24", "type": "module", "main": "types/index.js", "types": "types/index.d.ts",