diff --git a/apps/web/src/lib/db-schema.ts b/apps/web/src/lib/db-schema.ts index 0829589..c92ce73 100644 --- a/apps/web/src/lib/db-schema.ts +++ b/apps/web/src/lib/db-schema.ts @@ -1,6 +1,16 @@ import type { ForageRepository, ImportEvent, RepositoryAnalysis } from "@forage/shared"; import Dexie, { type EntityTable } from "dexie"; +export const forageDatabaseName = "forage"; +export const forageDatabaseVersion = 3; + +export const forageDatabaseStores = { + repositories: "github_id, &full_name, primary_language, starred_at", + importEvents: "id", + metadata: "id", + analysisResults: "repository_id, &repository_full_name, analysis_version", +} as const; + export const localLibraryProfileKey = "local-library-profile"; export const localOperationLockKey = "local-operation-lock"; @@ -25,18 +35,17 @@ export interface LocalOperationLock { type MetadataRecord = LocalLibraryProfile | LocalOperationLock; -interface ForageDatabase extends Dexie { +export interface ForageDatabase extends Dexie { repositories: EntityTable; importEvents: EntityTable; metadata: EntityTable; analysisResults: EntityTable; } -export const db = new Dexie("forage") as ForageDatabase; +export function createForageDatabase(name = forageDatabaseName): ForageDatabase { + const database = new Dexie(name) as ForageDatabase; + database.version(forageDatabaseVersion).stores(forageDatabaseStores); + return database; +} -db.version(3).stores({ - repositories: "github_id, &full_name, primary_language, starred_at", - importEvents: "id", - metadata: "id", - analysisResults: "repository_id, &repository_full_name, analysis_version", -}); +export const db = createForageDatabase(); diff --git a/apps/web/src/lib/db.test.ts b/apps/web/src/lib/db.test.ts index 14fdf43..11444c9 100644 --- a/apps/web/src/lib/db.test.ts +++ b/apps/web/src/lib/db.test.ts @@ -1,5 +1,6 @@ import { analyzeRepository } from "@forage/analysis"; import type { ForageRepository, ImportEvent } from "@forage/shared"; +import Dexie from "dexie"; import { beforeEach, describe, expect, it } from "vitest"; import { acquireLocalOperationLock, @@ -16,6 +17,12 @@ import { saveLocalLibraryProfile, saveRepositories, } from "./db"; +import { + createForageDatabase, + forageDatabaseStores, + forageDatabaseVersion, + localLibraryProfileKey, +} from "./db-schema"; describe("local data store", () => { beforeEach(async () => { @@ -146,6 +153,69 @@ describe("local data store", () => { }); }); +describe("local data schema", () => { + it("keeps the current schema contract explicit", () => { + expect(forageDatabaseVersion).toBe(3); + expect(forageDatabaseStores).toEqual({ + repositories: "github_id, &full_name, primary_language, starred_at", + importEvents: "id", + metadata: "id", + analysisResults: "repository_id, &repository_full_name, analysis_version", + }); + }); + + it("upgrades v2 local data without dropping existing records", async () => { + const databaseName = `forage-migration-${crypto.randomUUID()}`; + const repository = createRepository(10, "forage/migration"); + const event = createImportEvent("event-migration", "2026-06-06T12:00:00.000Z"); + const legacyDatabase = new Dexie(databaseName); + + legacyDatabase.version(2).stores({ + repositories: forageDatabaseStores.repositories, + importEvents: forageDatabaseStores.importEvents, + metadata: forageDatabaseStores.metadata, + }); + + try { + await legacyDatabase.open(); + await legacyDatabase.table("repositories").put(repository); + await legacyDatabase.table("importEvents").put(event); + await legacyDatabase.table("metadata").put({ + id: localLibraryProfileKey, + github_login: "dills122", + github_user_id: 123, + repository_count: 1, + updated_at: "2026-06-06T12:00:00.000Z", + }); + legacyDatabase.close(); + + const upgradedDatabase = createForageDatabase(databaseName); + await upgradedDatabase.open(); + + expect(await upgradedDatabase.repositories.toArray()).toMatchObject([ + { github_id: 10, full_name: "forage/migration" }, + ]); + expect(await upgradedDatabase.importEvents.toArray()).toMatchObject([ + { id: "event-migration", status: "completed" }, + ]); + expect(await upgradedDatabase.metadata.get(localLibraryProfileKey)).toMatchObject({ + github_login: "dills122", + repository_count: 1, + }); + + await upgradedDatabase.analysisResults.put(analyzeRepository(repository)); + expect(await upgradedDatabase.analysisResults.toArray()).toMatchObject([ + { repository_id: 10, repository_full_name: "forage/migration" }, + ]); + + upgradedDatabase.close(); + } finally { + legacyDatabase.close(); + await Dexie.delete(databaseName); + } + }); +}); + function createRepository(githubId: number, fullName: string): ForageRepository { const importedAt = "2026-06-06T12:00:00.000Z"; const [owner, repoName] = fullName.split("/"); diff --git a/apps/web/src/lib/import-pipeline.ts b/apps/web/src/lib/import-pipeline.ts index cda3e70..c081a09 100644 --- a/apps/web/src/lib/import-pipeline.ts +++ b/apps/web/src/lib/import-pipeline.ts @@ -51,7 +51,7 @@ export async function runRepositoryImportPipeline( const currentPage: number = page; onProgress({ importRun, phase: "importing", page: currentPage, observedFieldNames }); const result = await runImportRequestWithRetry( - () => api.getStarredPage(currentPage, 100, input.signal), + (signal) => api.getStarredPage(currentPage, 100, signal), { signal: input.signal }, ); diff --git a/apps/web/src/lib/import-retry.test.ts b/apps/web/src/lib/import-retry.test.ts index c459014..49fc5f4 100644 --- a/apps/web/src/lib/import-retry.test.ts +++ b/apps/web/src/lib/import-retry.test.ts @@ -28,18 +28,31 @@ describe("import retry helpers", () => { ).resolves.toBe("ok"); expect(request).toHaveBeenCalledTimes(2); - expect(sleep).toHaveBeenCalledWith(750, expect.any(AbortSignal)); + expect(request).toHaveBeenCalledWith(expect.any(AbortSignal)); + expect(sleep).toHaveBeenCalledWith(500, expect.any(AbortSignal)); }); it("does not retry auth, validation, or rate-limit failures", () => { expect(shouldRetryImportRequest(new WorkerApiError("auth", 401, null), 1)).toBe(false); expect(shouldRetryImportRequest(new WorkerApiError("rate", 429, null, 60), 1)).toBe(false); - expect(shouldRetryImportRequest(new WorkerApiError("temporary", 502, null), 3)).toBe(false); + expect(shouldRetryImportRequest(new WorkerApiError("temporary", 502, null), 2)).toBe(false); + }); + + it("retries page request timeouts without treating user cancellation as retryable", () => { + expect( + shouldRetryImportRequest( + new DOMException("Import page request timed out.", "TimeoutError"), + 1, + ), + ).toBe(true); + expect(shouldRetryImportRequest(new DOMException("Import cancelled.", "AbortError"), 1)).toBe( + false, + ); }); it("uses retry-after metadata before exponential fallback", () => { expect(getImportRetryDelayMs(new WorkerApiError("temporary", 503, null, 2), 1)).toBe(2_000); - expect(getImportRetryDelayMs(new WorkerApiError("temporary", 503, null), 2)).toBe(1_500); + expect(getImportRetryDelayMs(new WorkerApiError("temporary", 503, null), 2)).toBe(1_000); }); it("estimates rate-limit retry timing from GitHub reset metadata", () => { diff --git a/apps/web/src/lib/import-retry.ts b/apps/web/src/lib/import-retry.ts index 3706d99..837cebe 100644 --- a/apps/web/src/lib/import-retry.ts +++ b/apps/web/src/lib/import-retry.ts @@ -1,9 +1,10 @@ import { WorkerApiError } from "./api"; export const importRetryPolicy = { - maxAttempts: 3, - baseDelayMs: 750, + maxAttempts: 2, + baseDelayMs: 500, maxDelayMs: 5_000, + requestTimeoutMs: 5_000, retryableStatuses: new Set([408, 500, 502, 503, 504]), }; @@ -13,17 +14,20 @@ interface RetryOptions { } export async function runImportRequestWithRetry( - request: () => Promise, + request: (signal: AbortSignal) => Promise, { signal, sleep = sleepWithAbort }: RetryOptions, ) { for (let attempt = 1; ; attempt += 1) { throwIfAborted(signal); + const requestSignal = createImportRequestSignal(signal); try { - return await request(); + return await request(requestSignal.signal); } catch (error) { if (!shouldRetryImportRequest(error, attempt)) throw error; await sleep(getImportRetryDelayMs(error, attempt), signal); + } finally { + requestSignal.dispose(); } } } @@ -33,6 +37,7 @@ export function shouldRetryImportRequest(error: unknown, attempt: number) { if (error instanceof WorkerApiError) { return importRetryPolicy.retryableStatuses.has(error.status); } + if (isImportRequestTimeout(error)) return true; return error instanceof TypeError; } @@ -67,6 +72,45 @@ function sleepWithAbort(delayMs: number, signal: AbortSignal) { }); } +function createImportRequestSignal(parentSignal: AbortSignal) { + const controller = new AbortController(); + let disposed = false; + const timeout = globalThis.setTimeout(() => { + controller.abort(new DOMException("Import page request timed out.", "TimeoutError")); + }, importRetryPolicy.requestTimeoutMs); + const abortFromParent = () => { + controller.abort(new DOMException("Import cancelled.", "AbortError")); + }; + + if (parentSignal.aborted) { + abortFromParent(); + } else { + parentSignal.addEventListener("abort", abortFromParent, { once: true }); + } + + controller.signal.addEventListener( + "abort", + () => { + globalThis.clearTimeout(timeout); + }, + { once: true }, + ); + + return { + signal: controller.signal, + dispose: () => { + if (disposed) return; + disposed = true; + globalThis.clearTimeout(timeout); + parentSignal.removeEventListener("abort", abortFromParent); + }, + }; +} + +function isImportRequestTimeout(error: unknown) { + return error instanceof DOMException && error.name === "TimeoutError"; +} + function throwIfAborted(signal: AbortSignal) { if (signal.aborted) { throw new DOMException("Import cancelled.", "AbortError"); diff --git a/apps/worker/src/crypto.ts b/apps/worker/src/crypto.ts index 3a95e2d..b567805 100644 --- a/apps/worker/src/crypto.ts +++ b/apps/worker/src/crypto.ts @@ -36,7 +36,9 @@ export async function operationalHash(value: string, env: Env) { } export function createPkceVerifier() { - return createId(); + const bytes = new Uint8Array(32); + crypto.getRandomValues(bytes); + return base64UrlEncode(bytes); } export async function createPkceChallenge(verifier: string) { diff --git a/apps/worker/test/index.test.mjs b/apps/worker/test/index.test.mjs index 8f4f39a..accb5d8 100644 --- a/apps/worker/test/index.test.mjs +++ b/apps/worker/test/index.test.mjs @@ -41,7 +41,7 @@ test("GitHub auth uses state and PKCE verifier during token exchange", async () assert.equal(callbackResponse.headers.get("location"), "https://forage.test"); assert.equal(tokenExchangeBodies.length, 1); assert.equal(tokenExchangeBodies[0].code, "test-code"); - assert.match(tokenExchangeBodies[0].code_verifier, /^[A-Za-z0-9_-]{32,}$/); + assert.match(tokenExchangeBodies[0].code_verifier, /^[A-Za-z0-9._~-]{43,128}$/); } finally { restoreFetch(); } diff --git a/apps/worker/wrangler.toml b/apps/worker/wrangler.toml index 6d0098b..dd23a4a 100644 --- a/apps/worker/wrangler.toml +++ b/apps/worker/wrangler.toml @@ -29,22 +29,36 @@ new_sqlite_classes = ["AuthCoordinator"] # binding = "OAUTH_STATE_KV" # id = "" +[env.staging] +workers_dev = false + [env.staging.vars] ENVIRONMENT = "staging" GITHUB_API_VERSION = "2022-11-28" -GITHUB_REDIRECT_URI = "https://api-staging.forage.example.com/auth/github/callback" -WEB_ORIGIN = "https://staging.forage.example.com" +GITHUB_REDIRECT_URI = "https://api-staging.forage.shrimpworks.dev/auth/github/callback" +WEB_ORIGIN = "https://forage-staging.shrimpworks.dev" [[env.staging.durable_objects.bindings]] name = "AUTH_COORDINATOR" class_name = "AuthCoordinator" +[[env.staging.kv_namespaces]] +binding = "SETTINGS_KV" +id = "fe90c0a2a9334383b42653d5b5d370aa" + +[env.production] +workers_dev = false + [env.production.vars] ENVIRONMENT = "production" GITHUB_API_VERSION = "2022-11-28" -GITHUB_REDIRECT_URI = "https://api.forage.example.com/auth/github/callback" -WEB_ORIGIN = "https://forage.example.com" +GITHUB_REDIRECT_URI = "https://api.forage.shrimpworks.dev/auth/github/callback" +WEB_ORIGIN = "https://forage.shrimpworks.dev" [[env.production.durable_objects.bindings]] name = "AUTH_COORDINATOR" class_name = "AuthCoordinator" + +[[env.production.kv_namespaces]] +binding = "SETTINGS_KV" +id = "ee63c81acc454a78a20904ff7438bbe9" diff --git a/docs/13-storage-schema.md b/docs/13-storage-schema.md index 56122fc..9d1f126 100644 --- a/docs/13-storage-schema.md +++ b/docs/13-storage-schema.md @@ -6,6 +6,11 @@ Initial browser schema implemented Primary storage: IndexedDB in the user's browser. +Current browser database: +- Name: `forage` +- Version: `3` +- Store contract: exported from `apps/web/src/lib/db-schema.ts` + Local data classes: - Repository metadata - Import events @@ -32,6 +37,8 @@ Required schema properties: Migration posture: Use forward migrations for local IndexedDB data when practical. If a future schema change is too large or risky to migrate safely, Forage may require a local reset and GitHub re-import, but that should be exceptional. +The current test suite includes a fixture-backed upgrade check from the previous v2 local schema into the current v3 schema. Any future browser database version bump should add a matching fixture test that proves existing repository, import event, profile, and analysis records are either preserved or intentionally migrated. + Backup and restore: JSON export should be the first supported full-state backup format. Restore should validate schema version, show incompatible version errors clearly, and avoid silently merging incompatible data. @@ -53,7 +60,7 @@ Current IndexedDB stores: - Indexes: `repository_full_name`, `analysis_version` - `metadata` - Key: `id` - - Current record: `local-library-profile` + - Current records: `local-library-profile`, `local-operation-lock` Current server settings behavior: - `GET /api/settings` returns the authenticated session's settings. diff --git a/docs/21-hosting-ui-setup.md b/docs/21-hosting-ui-setup.md index 4cbfce2..4e5a5fa 100644 --- a/docs/21-hosting-ui-setup.md +++ b/docs/21-hosting-ui-setup.md @@ -15,7 +15,7 @@ Replace the placeholder domains below before production: - Production web: `https://forage.example.com` - Production API: `https://api.forage.example.com` -- Staging web: `https://staging.forage.example.com` +- Staging web: `https://forage-staging.example.com` - Staging API: `https://api-staging.forage.example.com` ## GitHub App UI @@ -95,7 +95,7 @@ Staging Worker variables: - `ENVIRONMENT=staging` - `GITHUB_API_VERSION=2022-11-28` - `GITHUB_REDIRECT_URI=https://api-staging.forage.example.com/auth/github/callback` -- `WEB_ORIGIN=https://staging.forage.example.com` +- `WEB_ORIGIN=https://forage-staging.example.com` Production Worker secrets: - `GITHUB_CLIENT_ID` @@ -173,7 +173,7 @@ FORAGE_WORKER_ORIGIN=https://api.forage.example.com \ pnpm smoke:hosted ``` -For staging, use the staging web and API origins. This script verifies Worker health, CORS, preflight headers, Pages security headers, CSP Worker origin, and basic app HTML. +For staging, use the staging web and API origins and set `FORAGE_SMOKE_EXPECT_PRODUCTION=false`. This script verifies Worker health, CORS, preflight headers, Pages security headers, CSP Worker origin, and basic app HTML. Verify Worker health: - `GET https://api.forage.example.com/api/health` diff --git a/docs/22-infrastructure-as-code.md b/docs/22-infrastructure-as-code.md index baa3dd4..ddae638 100644 --- a/docs/22-infrastructure-as-code.md +++ b/docs/22-infrastructure-as-code.md @@ -1,7 +1,7 @@ # Infrastructure As Code Status: -Initial OpenTofu scaffold +Cloudflare staging path proven Forage uses OpenTofu-compatible Terraform configuration for Cloudflare resources under `infra/opentofu`. @@ -19,6 +19,7 @@ Current OpenTofu scope: - Cloudflare Pages project for `apps/web`. - Pages production and preview build environment variables. - Pages custom domains. +- DNS CNAME records for Pages custom domains. - Cloudflare KV namespaces for `SETTINGS_KV`. - Optional Worker custom domains after the Worker service exists. - Outputs for GitHub App homepage/callback URL values. @@ -57,6 +58,10 @@ Set `manage_worker_custom_domains = false` until the Worker service has been dep After apply, copy the `settings_kv_namespaces` output into the Worker binding configuration as `SETTINGS_KV` for each environment. +Worker custom domains intentionally attach to the Cloudflare Worker service environment named `production`. Wrangler environment deploys create separate Worker service names, such as `forage-worker-staging`, and each of those services exposes its deployed code under Cloudflare's service-level `production` environment. + +Pages custom domain DNS is managed by OpenTofu when `manage_pages_domains = true`. Production points at `.pages.dev`; non-production environments point at `..pages.dev`, so the Pages branch name must match the environment key. + Use [Deployment Automation](./23-deployment-automation.md) for the GitHub Actions workflow, repository secrets, GitHub environment variables, and first deployment order. ## Sources Checked diff --git a/docs/23-deployment-automation.md b/docs/23-deployment-automation.md index fcb6e78..0c23899 100644 --- a/docs/23-deployment-automation.md +++ b/docs/23-deployment-automation.md @@ -1,7 +1,7 @@ # Deployment Automation Status: -Manual GitHub Actions workflow scaffolded +Manual GitHub Actions workflow scaffolded and staging-hosting path proven Forage deploys through `.github/workflows/deploy.yml`. The workflow is manual-only for now so staging and production deploys remain operator-controlled while the Cloudflare hosting path is still being proven. @@ -76,9 +76,14 @@ Hosted smoke: ```sh FORAGE_WEB_ORIGIN=https://staging.forage.example.com \ FORAGE_WORKER_ORIGIN=https://api-staging.forage.example.com \ +FORAGE_SMOKE_EXPECT_PRODUCTION=false \ pnpm smoke:hosted ``` +Set `FORAGE_SMOKE_EXPECT_PRODUCTION=false` for staging because staging Worker config intentionally exposes non-secret setup diagnostics. Leave it unset for production so the smoke check verifies that production config hides those diagnostics. + +When using Cloudflare Pages custom branch domains, the staging branch name should match the OpenTofu environment key. For example, the `staging` environment maps to `staging..pages.dev` and can be connected to a custom hostname such as `forage-staging.example.com`. + ## Quality Gates The main Check workflow installs OpenTofu and runs `npm run infra:fmt:check`. The root `npm run check` also includes the same infra formatting check so local and CI gates stay aligned. diff --git a/infra/opentofu/.terraform.lock.hcl b/infra/opentofu/.terraform.lock.hcl index 73b5e87..d715e1e 100644 --- a/infra/opentofu/.terraform.lock.hcl +++ b/infra/opentofu/.terraform.lock.hcl @@ -1,7 +1,7 @@ -# This file is maintained automatically by "terraform init". +# This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/cloudflare/cloudflare" { +provider "registry.opentofu.org/cloudflare/cloudflare" { version = "5.9.0" constraints = "5.9.0" hashes = [ diff --git a/infra/opentofu/main.tf b/infra/opentofu/main.tf index a986858..8373e44 100644 --- a/infra/opentofu/main.tf +++ b/infra/opentofu/main.tf @@ -114,6 +114,23 @@ resource "cloudflare_pages_domain" "web" { depends_on = [cloudflare_pages_project.web] } +resource "cloudflare_dns_record" "pages_web" { + for_each = { + for name, environment in local.environment_config : name => environment + if var.manage_pages_domains && environment.manage_pages_domain + } + + zone_id = var.cloudflare_zone_id + name = each.value.web_hostname + type = "CNAME" + content = each.key == "production" ? "${var.pages_project_name}.pages.dev" : "${each.key}.${var.pages_project_name}.pages.dev" + proxied = true + ttl = 1 + comment = "Forage ${each.key} Pages custom domain" + + depends_on = [cloudflare_pages_domain.web] +} + resource "cloudflare_workers_custom_domain" "api" { for_each = { for name, environment in local.environment_config : name => environment @@ -121,9 +138,8 @@ resource "cloudflare_workers_custom_domain" "api" { } account_id = var.cloudflare_account_id - environment = each.key + environment = "production" hostname = each.value.api_hostname service = each.value.worker_service_name zone_id = var.cloudflare_zone_id - zone_name = var.cloudflare_zone_name } diff --git a/infra/opentofu/terraform.tfvars.example b/infra/opentofu/terraform.tfvars.example index 192e23c..444db8f 100644 --- a/infra/opentofu/terraform.tfvars.example +++ b/infra/opentofu/terraform.tfvars.example @@ -2,7 +2,6 @@ cloudflare_account_id = "replace-with-account-id" cloudflare_zone_id = "replace-with-zone-id" -cloudflare_zone_name = "example.com" pages_project_name = "forage-web" worker_service_name = "forage-worker" diff --git a/infra/opentofu/variables.tf b/infra/opentofu/variables.tf index d1f3722..cd85a01 100644 --- a/infra/opentofu/variables.tf +++ b/infra/opentofu/variables.tf @@ -8,11 +8,6 @@ variable "cloudflare_zone_id" { type = string } -variable "cloudflare_zone_name" { - description = "Cloudflare zone name for Forage custom domains." - type = string -} - variable "project_slug" { description = "Short stable project slug used in managed Cloudflare resource names." type = string diff --git a/scripts/check-hosted-smoke.mjs b/scripts/check-hosted-smoke.mjs index b40bd4c..25bc36a 100644 --- a/scripts/check-hosted-smoke.mjs +++ b/scripts/check-hosted-smoke.mjs @@ -8,6 +8,7 @@ const workerOrigin = normalizeOrigin( const untrustedOrigin = normalizeOrigin( process.env.FORAGE_UNTRUSTED_ORIGIN ?? "https://forage-smoke.invalid", ); +const expectProductionConfig = parseBooleanEnv(process.env.FORAGE_SMOKE_EXPECT_PRODUCTION, true); const failures = []; @@ -63,7 +64,10 @@ assertHeader( if (allowedConfig.payload?.stores_repository_data !== false) { failures.push("Worker config must report stores_repository_data: false."); } -if (Object.hasOwn(allowedConfig.payload ?? {}, "has_github_client_secret")) { +if ( + expectProductionConfig && + Object.hasOwn(allowedConfig.payload ?? {}, "has_github_client_secret") +) { failures.push("Production Worker config exposes has_github_client_secret diagnostics."); } @@ -164,3 +168,8 @@ function assertHeader(response, name, expected, label, { optional = false } = {} function normalizeOrigin(origin) { return origin.trim().replace(/\/+$/, ""); } + +function parseBooleanEnv(value, fallback) { + if (value === undefined) return fallback; + return !["0", "false", "no"].includes(value.trim().toLowerCase()); +}