Harden hosted staging import flow

apps/web/src/lib/db-schema.ts

@@ -1,6 +1,16 @@
 import type { ForageRepository, ImportEvent, RepositoryAnalysis } from "@forage/shared";
 import Dexie, { type EntityTable } from "dexie";
 
+export const forageDatabaseName = "forage";
+export const forageDatabaseVersion = 3;
+
+export const forageDatabaseStores = {
+  repositories: "github_id, &full_name, primary_language, starred_at",
+  importEvents: "id",
+  metadata: "id",
+  analysisResults: "repository_id, &repository_full_name, analysis_version",
+} as const;
+
 export const localLibraryProfileKey = "local-library-profile";
 export const localOperationLockKey = "local-operation-lock";
 
@@ -25,18 +35,17 @@ export interface LocalOperationLock {
 
 type MetadataRecord = LocalLibraryProfile | LocalOperationLock;
 
-interface ForageDatabase extends Dexie {
+export interface ForageDatabase extends Dexie {
   repositories: EntityTable<ForageRepository, "github_id">;
   importEvents: EntityTable<ImportEvent, "id">;
   metadata: EntityTable<MetadataRecord, "id">;
   analysisResults: EntityTable<RepositoryAnalysis, "repository_id">;
 }
 
-export const db = new Dexie("forage") as ForageDatabase;
+export function createForageDatabase(name = forageDatabaseName): ForageDatabase {
+  const database = new Dexie(name) as ForageDatabase;
+  database.version(forageDatabaseVersion).stores(forageDatabaseStores);
+  return database;
+}
 
-db.version(3).stores({
-  repositories: "github_id, &full_name, primary_language, starred_at",
-  importEvents: "id",
-  metadata: "id",
-  analysisResults: "repository_id, &repository_full_name, analysis_version",
-});
+export const db = createForageDatabase();

apps/web/src/lib/db.test.ts

@@ -1,5 +1,6 @@
 import { analyzeRepository } from "@forage/analysis";
 import type { ForageRepository, ImportEvent } from "@forage/shared";
+import Dexie from "dexie";
 import { beforeEach, describe, expect, it } from "vitest";
 import {
   acquireLocalOperationLock,
@@ -16,6 +17,12 @@ import {
   saveLocalLibraryProfile,
   saveRepositories,
 } from "./db";
+import {
+  createForageDatabase,
+  forageDatabaseStores,
+  forageDatabaseVersion,
+  localLibraryProfileKey,
+} from "./db-schema";
 
 describe("local data store", () => {
   beforeEach(async () => {
@@ -146,6 +153,69 @@ describe("local data store", () => {
   });
 });
 
+describe("local data schema", () => {
+  it("keeps the current schema contract explicit", () => {
+    expect(forageDatabaseVersion).toBe(3);
+    expect(forageDatabaseStores).toEqual({
+      repositories: "github_id, &full_name, primary_language, starred_at",
+      importEvents: "id",
+      metadata: "id",
+      analysisResults: "repository_id, &repository_full_name, analysis_version",
+    });
+  });
+
+  it("upgrades v2 local data without dropping existing records", async () => {
+    const databaseName = `forage-migration-${crypto.randomUUID()}`;
+    const repository = createRepository(10, "forage/migration");
+    const event = createImportEvent("event-migration", "2026-06-06T12:00:00.000Z");
+    const legacyDatabase = new Dexie(databaseName);
+
+    legacyDatabase.version(2).stores({
+      repositories: forageDatabaseStores.repositories,
+      importEvents: forageDatabaseStores.importEvents,
+      metadata: forageDatabaseStores.metadata,
+    });
+
+    try {
+      await legacyDatabase.open();
+      await legacyDatabase.table("repositories").put(repository);
+      await legacyDatabase.table("importEvents").put(event);
+      await legacyDatabase.table("metadata").put({
+        id: localLibraryProfileKey,
+        github_login: "dills122",
+        github_user_id: 123,
+        repository_count: 1,
+        updated_at: "2026-06-06T12:00:00.000Z",
+      });
+      legacyDatabase.close();
+
+      const upgradedDatabase = createForageDatabase(databaseName);
+      await upgradedDatabase.open();
+
+      expect(await upgradedDatabase.repositories.toArray()).toMatchObject([
+        { github_id: 10, full_name: "forage/migration" },
+      ]);
+      expect(await upgradedDatabase.importEvents.toArray()).toMatchObject([
+        { id: "event-migration", status: "completed" },
+      ]);
+      expect(await upgradedDatabase.metadata.get(localLibraryProfileKey)).toMatchObject({
+        github_login: "dills122",
+        repository_count: 1,
+      });
+
+      await upgradedDatabase.analysisResults.put(analyzeRepository(repository));
+      expect(await upgradedDatabase.analysisResults.toArray()).toMatchObject([
+        { repository_id: 10, repository_full_name: "forage/migration" },
+      ]);
+
+      upgradedDatabase.close();
+    } finally {
+      legacyDatabase.close();
+      await Dexie.delete(databaseName);
+    }
+  });
+});
+
 function createRepository(githubId: number, fullName: string): ForageRepository {
   const importedAt = "2026-06-06T12:00:00.000Z";
   const [owner, repoName] = fullName.split("/");

apps/web/src/lib/import-pipeline.ts

@@ -51,7 +51,7 @@ export async function runRepositoryImportPipeline(
       const currentPage: number = page;
       onProgress({ importRun, phase: "importing", page: currentPage, observedFieldNames });
       const result = await runImportRequestWithRetry<StarredPageResponse>(
-        () => api.getStarredPage(currentPage, 100, input.signal),
+        (signal) => api.getStarredPage(currentPage, 100, signal),
         { signal: input.signal },
       );
 

apps/web/src/lib/import-retry.test.ts

@@ -28,18 +28,31 @@ describe("import retry helpers", () => {
     ).resolves.toBe("ok");
 
     expect(request).toHaveBeenCalledTimes(2);
-    expect(sleep).toHaveBeenCalledWith(750, expect.any(AbortSignal));
+    expect(request).toHaveBeenCalledWith(expect.any(AbortSignal));
+    expect(sleep).toHaveBeenCalledWith(500, expect.any(AbortSignal));
   });
 
   it("does not retry auth, validation, or rate-limit failures", () => {
     expect(shouldRetryImportRequest(new WorkerApiError("auth", 401, null), 1)).toBe(false);
     expect(shouldRetryImportRequest(new WorkerApiError("rate", 429, null, 60), 1)).toBe(false);
-    expect(shouldRetryImportRequest(new WorkerApiError("temporary", 502, null), 3)).toBe(false);
+    expect(shouldRetryImportRequest(new WorkerApiError("temporary", 502, null), 2)).toBe(false);
+  });
+
+  it("retries page request timeouts without treating user cancellation as retryable", () => {
+    expect(
+      shouldRetryImportRequest(
+        new DOMException("Import page request timed out.", "TimeoutError"),
+        1,
+      ),
+    ).toBe(true);
+    expect(shouldRetryImportRequest(new DOMException("Import cancelled.", "AbortError"), 1)).toBe(
+      false,
+    );
   });
 
   it("uses retry-after metadata before exponential fallback", () => {
     expect(getImportRetryDelayMs(new WorkerApiError("temporary", 503, null, 2), 1)).toBe(2_000);
-    expect(getImportRetryDelayMs(new WorkerApiError("temporary", 503, null), 2)).toBe(1_500);
+    expect(getImportRetryDelayMs(new WorkerApiError("temporary", 503, null), 2)).toBe(1_000);
   });
 
   it("estimates rate-limit retry timing from GitHub reset metadata", () => {

apps/web/src/lib/import-retry.ts

@@ -1,9 +1,10 @@
 import { WorkerApiError } from "./api";
 
 export const importRetryPolicy = {
-  maxAttempts: 3,
-  baseDelayMs: 750,
+  maxAttempts: 2,
+  baseDelayMs: 500,
   maxDelayMs: 5_000,
+  requestTimeoutMs: 5_000,
   retryableStatuses: new Set([408, 500, 502, 503, 504]),
 };
 
@@ -13,17 +14,20 @@ interface RetryOptions {
 }
 
 export async function runImportRequestWithRetry<T>(
-  request: () => Promise<T>,
+  request: (signal: AbortSignal) => Promise<T>,
   { signal, sleep = sleepWithAbort }: RetryOptions,
 ) {
   for (let attempt = 1; ; attempt += 1) {
     throwIfAborted(signal);
+    const requestSignal = createImportRequestSignal(signal);
 
     try {
-      return await request();
+      return await request(requestSignal.signal);
     } catch (error) {
       if (!shouldRetryImportRequest(error, attempt)) throw error;
       await sleep(getImportRetryDelayMs(error, attempt), signal);
+    } finally {
+      requestSignal.dispose();
     }
   }
 }
@@ -33,6 +37,7 @@ export function shouldRetryImportRequest(error: unknown, attempt: number) {
   if (error instanceof WorkerApiError) {
     return importRetryPolicy.retryableStatuses.has(error.status);
   }
+  if (isImportRequestTimeout(error)) return true;
   return error instanceof TypeError;
 }
 
@@ -67,6 +72,45 @@ function sleepWithAbort(delayMs: number, signal: AbortSignal) {
   });
 }
 
+function createImportRequestSignal(parentSignal: AbortSignal) {
+  const controller = new AbortController();
+  let disposed = false;
+  const timeout = globalThis.setTimeout(() => {
+    controller.abort(new DOMException("Import page request timed out.", "TimeoutError"));
+  }, importRetryPolicy.requestTimeoutMs);
+  const abortFromParent = () => {
+    controller.abort(new DOMException("Import cancelled.", "AbortError"));
+  };
+
+  if (parentSignal.aborted) {
+    abortFromParent();
+  } else {
+    parentSignal.addEventListener("abort", abortFromParent, { once: true });
+  }
+
+  controller.signal.addEventListener(
+    "abort",
+    () => {
+      globalThis.clearTimeout(timeout);
+    },
+    { once: true },
+  );
+
+  return {
+    signal: controller.signal,
+    dispose: () => {
+      if (disposed) return;
+      disposed = true;
+      globalThis.clearTimeout(timeout);
+      parentSignal.removeEventListener("abort", abortFromParent);
+    },
+  };
+}
+
+function isImportRequestTimeout(error: unknown) {
+  return error instanceof DOMException && error.name === "TimeoutError";
+}
+
 function throwIfAborted(signal: AbortSignal) {
   if (signal.aborted) {
     throw new DOMException("Import cancelled.", "AbortError");

apps/worker/src/crypto.ts

@@ -36,7 +36,9 @@ export async function operationalHash(value: string, env: Env) {
 }
 
 export function createPkceVerifier() {
-  return createId();
+  const bytes = new Uint8Array(32);
+  crypto.getRandomValues(bytes);
+  return base64UrlEncode(bytes);
 }
 
 export async function createPkceChallenge(verifier: string) {

apps/worker/test/index.test.mjs

@@ -41,7 +41,7 @@ test("GitHub auth uses state and PKCE verifier during token exchange", async ()
     assert.equal(callbackResponse.headers.get("location"), "https://forage.test");
     assert.equal(tokenExchangeBodies.length, 1);
     assert.equal(tokenExchangeBodies[0].code, "test-code");
-    assert.match(tokenExchangeBodies[0].code_verifier, /^[A-Za-z0-9_-]{32,}$/);
+    assert.match(tokenExchangeBodies[0].code_verifier, /^[A-Za-z0-9._~-]{43,128}$/);
   } finally {
     restoreFetch();
   }

apps/worker/wrangler.toml

@@ -29,22 +29,36 @@ new_sqlite_classes = ["AuthCoordinator"]
 # binding = "OAUTH_STATE_KV"
 # id = "<OAUTH_STATE_KV_NAMESPACE_ID>"
 
+[env.staging]
+workers_dev = false
+
 [env.staging.vars]
 ENVIRONMENT = "staging"
 GITHUB_API_VERSION = "2022-11-28"
-GITHUB_REDIRECT_URI = "https://api-staging.forage.example.com/auth/github/callback"
-WEB_ORIGIN = "https://staging.forage.example.com"
+GITHUB_REDIRECT_URI = "https://api-staging.forage.shrimpworks.dev/auth/github/callback"
+WEB_ORIGIN = "https://forage-staging.shrimpworks.dev"
 
 [[env.staging.durable_objects.bindings]]
 name = "AUTH_COORDINATOR"
 class_name = "AuthCoordinator"
 
+[[env.staging.kv_namespaces]]
+binding = "SETTINGS_KV"
+id = "fe90c0a2a9334383b42653d5b5d370aa"
+
+[env.production]
+workers_dev = false
+
 [env.production.vars]
 ENVIRONMENT = "production"
 GITHUB_API_VERSION = "2022-11-28"
-GITHUB_REDIRECT_URI = "https://api.forage.example.com/auth/github/callback"
-WEB_ORIGIN = "https://forage.example.com"
+GITHUB_REDIRECT_URI = "https://api.forage.shrimpworks.dev/auth/github/callback"
+WEB_ORIGIN = "https://forage.shrimpworks.dev"
 
 [[env.production.durable_objects.bindings]]
 name = "AUTH_COORDINATOR"
 class_name = "AuthCoordinator"
+
+[[env.production.kv_namespaces]]
+binding = "SETTINGS_KV"
+id = "ee63c81acc454a78a20904ff7438bbe9"

docs/13-storage-schema.md

@@ -6,6 +6,11 @@ Initial browser schema implemented
 Primary storage:
 IndexedDB in the user's browser.
 
+Current browser database:
+- Name: `forage`
+- Version: `3`
+- Store contract: exported from `apps/web/src/lib/db-schema.ts`
+
 Local data classes:
 - Repository metadata
 - Import events
@@ -32,6 +37,8 @@ Required schema properties:
 Migration posture:
 Use forward migrations for local IndexedDB data when practical. If a future schema change is too large or risky to migrate safely, Forage may require a local reset and GitHub re-import, but that should be exceptional.
 
+The current test suite includes a fixture-backed upgrade check from the previous v2 local schema into the current v3 schema. Any future browser database version bump should add a matching fixture test that proves existing repository, import event, profile, and analysis records are either preserved or intentionally migrated.
+
 Backup and restore:
 JSON export should be the first supported full-state backup format. Restore should validate schema version, show incompatible version errors clearly, and avoid silently merging incompatible data.
 
@@ -53,7 +60,7 @@ Current IndexedDB stores:
   - Indexes: `repository_full_name`, `analysis_version`
 - `metadata`
   - Key: `id`
-  - Current record: `local-library-profile`
+  - Current records: `local-library-profile`, `local-operation-lock`
 
 Current server settings behavior:
 - `GET /api/settings` returns the authenticated session's settings.

docs/21-hosting-ui-setup.md

@@ -15,7 +15,7 @@ Replace the placeholder domains below before production:
 
 - Production web: `https://forage.example.com`
 - Production API: `https://api.forage.example.com`
-- Staging web: `https://staging.forage.example.com`
+- Staging web: `https://forage-staging.example.com`
 - Staging API: `https://api-staging.forage.example.com`
 
 ## GitHub App UI
@@ -95,7 +95,7 @@ Staging Worker variables:
 - `ENVIRONMENT=staging`
 - `GITHUB_API_VERSION=2022-11-28`
 - `GITHUB_REDIRECT_URI=https://api-staging.forage.example.com/auth/github/callback`
-- `WEB_ORIGIN=https://staging.forage.example.com`
+- `WEB_ORIGIN=https://forage-staging.example.com`
 
 Production Worker secrets:
 - `GITHUB_CLIENT_ID`
@@ -173,7 +173,7 @@ FORAGE_WORKER_ORIGIN=https://api.forage.example.com \
 pnpm smoke:hosted
 ```
 
-For staging, use the staging web and API origins. This script verifies Worker health, CORS, preflight headers, Pages security headers, CSP Worker origin, and basic app HTML.
+For staging, use the staging web and API origins and set `FORAGE_SMOKE_EXPECT_PRODUCTION=false`. This script verifies Worker health, CORS, preflight headers, Pages security headers, CSP Worker origin, and basic app HTML.
 
 Verify Worker health:
 - `GET https://api.forage.example.com/api/health`