From 15c05955b5255c80160638697e633b3bafbc859f Mon Sep 17 00:00:00 2001 From: Dylan Steele Date: Sat, 6 Jun 2026 23:07:10 -0400 Subject: [PATCH 1/2] Refactor web app orchestration --- apps/web/src/components/ForageApp.svelte | 315 +++-------------------- apps/web/src/lib/app-refresh.test.ts | 97 +++++++ apps/web/src/lib/app-refresh.ts | 123 +++++++++ apps/web/src/lib/app-state.test.ts | 106 ++++++++ apps/web/src/lib/app-state.ts | 168 ++++++++++++ apps/web/src/lib/export-data.ts | 57 ++++ apps/web/src/lib/theme.ts | 21 ++ 7 files changed, 603 insertions(+), 284 deletions(-) create mode 100644 apps/web/src/lib/app-refresh.test.ts create mode 100644 apps/web/src/lib/app-refresh.ts create mode 100644 apps/web/src/lib/app-state.test.ts create mode 100644 apps/web/src/lib/app-state.ts create mode 100644 apps/web/src/lib/export-data.ts create mode 100644 apps/web/src/lib/theme.ts diff --git a/apps/web/src/components/ForageApp.svelte b/apps/web/src/components/ForageApp.svelte index 067614b..5f8afae 100644 --- a/apps/web/src/components/ForageApp.svelte +++ b/apps/web/src/components/ForageApp.svelte @@ -1,12 +1,4 @@ @@ -526,8 +264,17 @@ {#if state.repositoryCount > 0}
- {getLocalLibraryNoticeTitle()} - {getLocalLibraryNoticeBody()} + + {getLocalLibraryNoticeTitle(state.localLibraryConflict, state.authenticated)} + + + {getLocalLibraryNoticeBody({ + authenticated: state.authenticated, + localLibraryConflict: state.localLibraryConflict, + localLibraryOwner: state.localLibraryOwner, + sessionLogin: state.sessionUser?.login ?? null, + })} +
{/if} diff --git a/apps/web/src/lib/app-refresh.test.ts b/apps/web/src/lib/app-refresh.test.ts new file mode 100644 index 0000000..87ffd4f --- /dev/null +++ b/apps/web/src/lib/app-refresh.test.ts @@ -0,0 +1,97 @@ +import type { ForageRepository } from "@forage/shared"; +import { describe, expect, it } from "vitest"; +import { createAppStateSnapshotPatch } from "./app-refresh"; + +describe("app refresh helpers", () => { + it("turns local and remote snapshots into an app state patch", () => { + const olderRepository = createRepository(1, "forage/older", { + primary_language: "Go", + starred_at: "2026-01-01T00:00:00.000Z", + }); + const newerRepository = createRepository(2, "forage/newer", { + primary_language: "TypeScript", + starred_at: "2026-02-01T00:00:00.000Z", + }); + + const patch = createAppStateSnapshotPatch({ + repositories: [olderRepository, newerRepository], + importEvents: [{ status: "completed", repositories: 2 }], + analysisResults: [], + localLibraryProfile: { + id: "local-library-profile", + github_login: "dills122", + github_user_id: 123, + repository_count: 2, + updated_at: "2026-06-06T12:00:00.000Z", + }, + config: { github_configured: true }, + session: { + authenticated: true, + user: { login: "dills122", id: 123 }, + }, + settingsResponse: { + settings: { + analytics_enabled: true, + updated_at: "2026-06-06T12:00:00.000Z", + }, + settings_store: "durable_object", + stores_repository_data: false, + }, + }); + + expect(patch.configStatus).toBe("Ready"); + expect(patch.sessionStatus).toBe("Authenticated"); + expect(patch.latestImport).toBe("completed (2)"); + expect(patch.repositoryCount).toBe(2); + expect(patch.repositories?.map((repository) => repository.full_name)).toEqual([ + "forage/newer", + "forage/older", + ]); + expect(patch.user).toBe("dills122"); + expect(patch.localLibraryConflict).toBe(false); + expect(patch.settingsStatus).toBe( + "On. Anonymous product analytics only; repository data stays in this browser.", + ); + }); +}); + +function createRepository( + githubId: number, + fullName: string, + overrides: Partial = {}, +): ForageRepository { + const importedAt = "2026-06-06T12:00:00.000Z"; + const [owner, repoName] = fullName.split("/"); + + return { + github_id: githubId, + node_id: `node-${githubId}`, + repo_name: repoName ?? "repo", + owner: owner ?? "forage", + full_name: fullName, + url: `https://github.com/${fullName}`, + description: "Repository fixture", + homepage: null, + topics: ["testing"], + primary_language: "TypeScript", + license: "MIT", + stars: 10, + forks: 1, + watchers: 10, + open_issues: 0, + archived: false, + disabled: false, + fork: false, + private: false, + default_branch: "main", + owner_avatar_url: "https://example.com/avatar.png", + created_at: "2025-01-01T00:00:00.000Z", + updated_at: importedAt, + pushed_at: importedAt, + starred_at: importedAt, + imported_at: importedAt, + source_api_version: "2022-11-28", + schema_version: 1, + ...overrides, + }; +} diff --git a/apps/web/src/lib/app-refresh.ts b/apps/web/src/lib/app-refresh.ts new file mode 100644 index 0000000..887867c --- /dev/null +++ b/apps/web/src/lib/app-refresh.ts @@ -0,0 +1,123 @@ +import type { RepositoryAnalysis } from "@forage/shared"; +import type { SessionResponse, SettingsResponse, WorkerConfig } from "./api"; +import { + type AppState, + defaultSettings, + getConfigStatus, + getLocalLibraryOwner, + getLocalLibraryStatus, + getSettingsStatus, + getUserDisplay, + hasLocalLibraryConflict, + type WorkerConfigStatusInput, +} from "./app-state"; +import { + getAllAnalysisResults, + getAllRepositories, + getImportEvents, + getLocalLibraryProfile, + type LocalLibraryProfile, +} from "./db"; +import { createCurrentAnalysisMap, getTopLanguage, sortRepositoriesByStarredAt } from "./library"; + +export interface AppRefreshApi { + getConfig: () => Promise; + getSession: () => Promise; + getSettings: () => Promise; +} + +export async function loadAppStateSnapshot(api: AppRefreshApi): Promise> { + try { + const [repositories, events, analysisResults, localLibraryProfile] = await Promise.all([ + getAllRepositories(), + getImportEvents(), + getAllAnalysisResults(), + getLocalLibraryProfile(), + ]); + const configPromise: Promise = api + .getConfig() + .catch((error: Error) => ({ + error: error.message, + has_github_client_id: false, + has_github_client_secret: false, + })); + const sessionPromise = api.getSession().catch( + (error: Error): SessionResponse => ({ + authenticated: false, + error: error.message, + }), + ); + const [config, session] = await Promise.all([configPromise, sessionPromise]); + const settingsResponse = session.authenticated + ? await api.getSettings().catch(() => null) + : null; + + return createAppStateSnapshotPatch({ + repositories, + importEvents: events, + analysisResults, + localLibraryProfile, + config, + session, + settingsResponse, + }); + } catch (error) { + return { + configStatus: error instanceof Error ? error.message : "Worker unavailable", + sessionStatus: "Unavailable", + settings: defaultSettings(), + settingsStatus: "Settings unavailable.", + }; + } +} + +export function createAppStateSnapshotPatch(input: { + repositories: AppState["repositories"]; + importEvents: Array<{ status: string; repositories: number }>; + analysisResults: RepositoryAnalysis[]; + localLibraryProfile: LocalLibraryProfile | null; + config: WorkerConfigStatusInput; + session: SessionResponse; + settingsResponse: SettingsResponse | null; +}): Partial { + const localLibraryOwner = getLocalLibraryOwner( + input.localLibraryProfile, + input.repositories.length, + ); + const localLibraryConflict = hasLocalLibraryConflict( + input.localLibraryProfile, + input.session.user, + ); + + return { + configStatus: getConfigStatus(input.config), + authenticated: input.session.authenticated, + sessionUser: input.session.user ?? null, + sessionStatus: input.session.authenticated + ? "Authenticated" + : input.session.error || "Disconnected", + repositoryCount: input.repositories.length, + latestImport: input.importEvents[0] + ? `${input.importEvents[0].status} (${input.importEvents[0].repositories})` + : "-", + repositories: sortRepositoriesByStarredAt(input.repositories), + analysisByRepositoryId: createCurrentAnalysisMap(input.analysisResults), + topLanguage: getTopLanguage(input.repositories), + localLibraryProfile: input.localLibraryProfile, + localLibraryOwner, + localLibraryConflict, + user: getUserDisplay(input.session, input.localLibraryProfile, input.repositories.length), + localLibraryStatus: getLocalLibraryStatus( + input.repositories.length, + input.session.authenticated, + localLibraryConflict, + localLibraryOwner, + input.session.user?.login ?? null, + ), + settings: input.settingsResponse?.settings ?? defaultSettings(), + settingsStatus: getSettingsStatus( + input.session.authenticated, + input.settingsResponse?.settings, + ), + }; +} diff --git a/apps/web/src/lib/app-state.test.ts b/apps/web/src/lib/app-state.test.ts new file mode 100644 index 0000000..283b9d0 --- /dev/null +++ b/apps/web/src/lib/app-state.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import type { SessionResponse } from "./api"; +import { + createInitialState, + defaultSettings, + getConfigStatus, + getLocalLibraryNoticeBody, + getLocalLibraryNoticeTitle, + getLocalLibraryOwner, + getLocalLibraryStatus, + getSelectedRepository, + getSettingsStatus, + getUserDisplay, + hasLocalLibraryConflict, +} from "./app-state"; +import type { LocalLibraryProfile } from "./db"; + +describe("app state helpers", () => { + it("creates a stable initial state", () => { + const state = createInitialState("http://127.0.0.1:8787"); + + expect(state.workerOrigin).toBe("http://127.0.0.1:8787"); + expect(state.configStatus).toBe("Checking"); + expect(state.settings).toEqual(defaultSettings()); + expect(state.repositories).toEqual([]); + expect(state.analysisByRepositoryId.size).toBe(0); + }); + + it("formats worker config, settings, and local library status", () => { + expect(getConfigStatus({ github_configured: true })).toBe("Ready"); + expect( + getConfigStatus({ + has_github_client_id: true, + has_github_client_secret: false, + }), + ).toBe("Missing GitHub env"); + expect(getConfigStatus({ error: "Worker unavailable" })).toBe("Worker unavailable"); + expect(getSettingsStatus(false, defaultSettings())).toBe( + "Connect GitHub to manage this setting.", + ); + expect(getSettingsStatus(true, defaultSettings())).toBe( + "Off. Repository data stays in this browser.", + ); + expect(getSettingsStatus(true, { analytics_enabled: true, updated_at: null })).toBe( + "On. Anonymous product analytics only; repository data stays in this browser.", + ); + expect(getLocalLibraryStatus(0, false, false, "-", null)).toBe( + "No repository data stored locally.", + ); + expect(getLocalLibraryStatus(12, true, false, "dills122", "dills122")).toBe( + "12 repositories stored locally and ready to refresh.", + ); + expect(getLocalLibraryStatus(12, false, false, "dills122", null)).toBe( + "12 repositories stored locally; GitHub is disconnected.", + ); + expect(getLocalLibraryStatus(12, true, true, "personal", "work")).toBe( + "12 repositories stored locally for personal; current GitHub session is work.", + ); + }); + + it("handles connected, disconnected, and conflicting owners", () => { + const profile: LocalLibraryProfile = { + id: "local-library-profile", + github_login: "dills122", + github_user_id: 123, + repository_count: 720, + updated_at: "2026-06-06T12:00:00.000Z", + }; + const authenticatedSession: SessionResponse = { + authenticated: true, + user: { login: "dills122", id: 123 }, + }; + const disconnectedSession: SessionResponse = { authenticated: false }; + + expect(getLocalLibraryOwner(null, 0)).toBe("-"); + expect(getLocalLibraryOwner(profile, 720)).toBe("dills122"); + expect(getUserDisplay(authenticatedSession, profile, 720)).toBe("dills122"); + expect(getUserDisplay(disconnectedSession, profile, 720)).toBe("dills122 (not connected)"); + expect(hasLocalLibraryConflict(profile, authenticatedSession.user)).toBe(false); + expect(hasLocalLibraryConflict(profile, { login: "work-user", id: 456 })).toBe(true); + expect(getLocalLibraryNoticeTitle(true, true)).toBe("Different GitHub account connected"); + expect(getLocalLibraryNoticeTitle(false, true)).toBe("Local library synced to this browser"); + expect( + getLocalLibraryNoticeBody({ + authenticated: false, + localLibraryConflict: false, + localLibraryOwner: "dills122", + sessionLogin: null, + }), + ).toBe( + "This browser has local data for dills122. Connect GitHub when you want to refresh imports.", + ); + }); + + it("selects the requested repository or falls back to the first visible repository", () => { + const repositories = [ + { github_id: 1, full_name: "forage/one" }, + { github_id: 2, full_name: "forage/two" }, + ] as Parameters[0]; + + expect(getSelectedRepository([], null)).toBeNull(); + expect(getSelectedRepository(repositories, null)?.full_name).toBe("forage/one"); + expect(getSelectedRepository(repositories, 2)?.full_name).toBe("forage/two"); + expect(getSelectedRepository(repositories, 999)?.full_name).toBe("forage/one"); + }); +}); diff --git a/apps/web/src/lib/app-state.ts b/apps/web/src/lib/app-state.ts new file mode 100644 index 0000000..cfa647b --- /dev/null +++ b/apps/web/src/lib/app-state.ts @@ -0,0 +1,168 @@ +import type { ImportRunState } from "@forage/core"; +import type { ApplicationSettings, ForageRepository, RepositoryAnalysis } from "@forage/shared"; +import type { SessionResponse, WorkerConfig } from "./api"; +import type { LocalLibraryProfile } from "./db"; +import type { LibrarySortMode } from "./library"; + +export interface AppState { + workerOrigin: string; + configStatus: string; + sessionStatus: string; + user: string; + repositoryCount: number; + latestImport: string; + progress: string; + localLibraryStatus: string; + authenticated: boolean; + sessionUser: SessionResponse["user"] | null; + localLibraryProfile: LocalLibraryProfile | null; + localLibraryOwner: string; + localLibraryConflict: boolean; + settings: ApplicationSettings; + settingsStatus: string; + repositories: ForageRepository[]; + analysisByRepositoryId: Map; + topLanguage: string; + searchQuery: string; + languageFilter: string; + categoryFilter: string; + sortMode: LibrarySortMode; + selectedRepositoryId: number | null; + importRun: ImportRunState | null; + observedFields: string; +} + +export type WorkerConfigStatusInput = Pick< + WorkerConfig, + "github_configured" | "has_github_client_id" | "has_github_client_secret" +> & { + error?: string; +}; + +export function createInitialState(workerOrigin: string): AppState { + return { + workerOrigin, + configStatus: "Checking", + sessionStatus: "Checking", + user: "-", + repositoryCount: 0, + latestImport: "-", + progress: "Ready.", + localLibraryStatus: "No repository data stored locally.", + authenticated: false, + sessionUser: null, + localLibraryProfile: null, + localLibraryOwner: "-", + localLibraryConflict: false, + settings: defaultSettings(), + settingsStatus: "Connect GitHub to manage this setting.", + repositories: [], + analysisByRepositoryId: new Map(), + topLanguage: "-", + searchQuery: "", + languageFilter: "", + categoryFilter: "", + sortMode: "starred_at_desc", + selectedRepositoryId: null, + importRun: null, + observedFields: "Run an import to capture raw GitHub fields.", + }; +} + +export function defaultSettings(): ApplicationSettings { + return { + analytics_enabled: false, + updated_at: null, + }; +} + +export function getConfigStatus(config: WorkerConfigStatusInput) { + if (config.error) return config.error; + const configured = + config.github_configured ?? + Boolean(config.has_github_client_id && config.has_github_client_secret); + return configured ? "Ready" : "Missing GitHub env"; +} + +export function getLocalLibraryStatus( + repositoryCount: number, + authenticated: boolean, + localLibraryConflict: boolean, + localLibraryOwner: string, + sessionLogin: string | null, +) { + if (repositoryCount === 0) return "No repository data stored locally."; + if (localLibraryConflict) { + return `${repositoryCount} repositories stored locally for ${localLibraryOwner}; current GitHub session is ${sessionLogin}.`; + } + if (authenticated) return `${repositoryCount} repositories stored locally and ready to refresh.`; + return `${repositoryCount} repositories stored locally; GitHub is disconnected.`; +} + +export function getLocalLibraryOwner(profile: LocalLibraryProfile | null, repositoryCount: number) { + if (repositoryCount === 0) return "-"; + return profile?.github_login ? profile.github_login : "Unknown local owner"; +} + +export function getSettingsStatus( + authenticated: boolean, + settings: ApplicationSettings | undefined, +) { + if (!authenticated) return "Connect GitHub to manage this setting."; + if (!settings?.analytics_enabled) return "Off. Repository data stays in this browser."; + return "On. Anonymous product analytics only; repository data stays in this browser."; +} + +export function getUserDisplay( + session: SessionResponse, + profile: LocalLibraryProfile | null, + repositoryCount: number, +) { + if (session.authenticated) return session.user?.login ?? "-"; + if (repositoryCount > 0 && profile?.github_login) + return `${profile.github_login} (not connected)`; + if (repositoryCount > 0) return "Not connected"; + return "-"; +} + +export function hasLocalLibraryConflict( + profile: LocalLibraryProfile | null, + sessionUser: SessionResponse["user"] | undefined, +) { + if (!profile?.github_login || !sessionUser?.login) return false; + return profile.github_login.toLowerCase() !== sessionUser.login.toLowerCase(); +} + +export function getLocalLibraryNoticeTitle(localLibraryConflict: boolean, authenticated: boolean) { + if (localLibraryConflict) return "Different GitHub account connected"; + if (authenticated) return "Local library synced to this browser"; + return "Local library available"; +} + +export function getLocalLibraryNoticeBody(input: { + authenticated: boolean; + localLibraryConflict: boolean; + localLibraryOwner: string; + sessionLogin: string | null; +}) { + if (input.localLibraryConflict) { + return `This browser has local data for ${input.localLibraryOwner}, but the current GitHub session is ${input.sessionLogin}. Export or reset local data before importing another account.`; + } + if (input.authenticated) { + return `This browser has local data for ${input.localLibraryOwner}. Refresh imports when you want to update it.`; + } + return `This browser has local data for ${input.localLibraryOwner}. Connect GitHub when you want to refresh imports.`; +} + +export function getSelectedRepository( + repositories: ForageRepository[], + selectedRepositoryId: number | null, +) { + if (repositories.length === 0) return null; + if (selectedRepositoryId === null) return repositories[0] ?? null; + return ( + repositories.find((repository) => repository.github_id === selectedRepositoryId) ?? + repositories[0] ?? + null + ); +} diff --git a/apps/web/src/lib/export-data.ts b/apps/web/src/lib/export-data.ts new file mode 100644 index 0000000..d2e3525 --- /dev/null +++ b/apps/web/src/lib/export-data.ts @@ -0,0 +1,57 @@ +import { analyzeRepository } from "@forage/analysis"; +import { + createForageExport, + serializeForageExportJson, + serializeRepositoryAnalysisCsv, +} from "@forage/reporting"; +import { + getAllAnalysisResults, + getAllRepositories, + getImportEvents, + getLocalLibraryProfile, +} from "./db"; +import { createCurrentAnalysisMap } from "./library"; + +export type ExportFormat = "json" | "csv"; + +export async function exportLocalLibrary(format: ExportFormat) { + const [repositories, events, analysisResults, localLibraryProfile] = await Promise.all([ + getAllRepositories(), + getImportEvents(), + getAllAnalysisResults(), + getLocalLibraryProfile(), + ]); + const analysisByRepositoryId = createCurrentAnalysisMap(analysisResults); + const payload = createForageExport({ + repositories, + analysis_results: repositories.map( + (repository) => + analysisByRepositoryId.get(repository.github_id) ?? analyzeRepository(repository), + ), + latest_import_event: events[0] ?? null, + local_library_profile: localLibraryProfile, + }); + const isJson = format === "json"; + const contents = isJson + ? serializeForageExportJson(payload) + : serializeRepositoryAnalysisCsv(payload); + const mimeType = isJson ? "application/json" : "text/csv"; + const extension = isJson ? "json" : "csv"; + downloadText( + contents, + mimeType, + `forage-export-${new Date().toISOString().slice(0, 10)}.${extension}`, + ); +} + +export function downloadText(contents: string, mimeType: string, filename: string) { + const blob = new Blob([contents], { + type: mimeType, + }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = filename; + link.click(); + URL.revokeObjectURL(url); +} diff --git a/apps/web/src/lib/theme.ts b/apps/web/src/lib/theme.ts new file mode 100644 index 0000000..4a292f8 --- /dev/null +++ b/apps/web/src/lib/theme.ts @@ -0,0 +1,21 @@ +export type Theme = "light" | "dark"; + +const themeStorageKey = "forage-theme"; + +export function getCurrentTheme() { + return document.documentElement.dataset.theme === "dark" ? "dark" : "light"; +} + +export function applyTheme(theme: Theme) { + document.documentElement.dataset.theme = theme; + return { + isDarkTheme: theme === "dark", + themeLabel: theme === "dark" ? "Dark" : "Light", + }; +} + +export function toggleStoredTheme() { + const nextTheme: Theme = getCurrentTheme() === "dark" ? "light" : "dark"; + localStorage.setItem(themeStorageKey, nextTheme); + return applyTheme(nextTheme); +} From b80e5009b2a6bf4ce209b3b0a5ef285ccdea5502 Mon Sep 17 00:00:00 2001 From: Dylan Steele Date: Sat, 6 Jun 2026 23:11:13 -0400 Subject: [PATCH 2/2] Extract repository import pipeline --- apps/web/src/lib/import-messages.test.ts | 67 ++++++++++++ apps/web/src/lib/import-messages.ts | 33 ++++++ apps/web/src/lib/import-pipeline.ts | 107 +++++++++++++++++++ apps/web/src/workers/import.worker.ts | 130 ++++------------------- 4 files changed, 227 insertions(+), 110 deletions(-) create mode 100644 apps/web/src/lib/import-messages.test.ts create mode 100644 apps/web/src/lib/import-messages.ts create mode 100644 apps/web/src/lib/import-pipeline.ts diff --git a/apps/web/src/lib/import-messages.test.ts b/apps/web/src/lib/import-messages.test.ts new file mode 100644 index 0000000..16a72d6 --- /dev/null +++ b/apps/web/src/lib/import-messages.test.ts @@ -0,0 +1,67 @@ +import { createImportRunState, recordImportPage } from "@forage/core"; +import { describe, expect, it } from "vitest"; +import { + getImportProgressText, + getImportTerminalText, + sortObservedFieldNames, +} from "./import-messages"; + +const rateLimit = { + limit: null, + remaining: null, + reset: null, + resource: null, + used: null, +}; + +describe("import worker messages", () => { + it("formats progress text by phase", () => { + const importRun = recordImportPage(createImportRunState(), { + page: 1, + repositories: 100, + rate_limit: rateLimit, + }); + + expect(getImportProgressText(importRun, "importing", 2)).toBe( + "Importing page 2; 100 repositories stored so far...", + ); + expect(getImportProgressText(importRun, "analyzing", 2)).toBe( + "Analyzing page 2 in browser worker after importing 100 repositories...", + ); + }); + + it("formats terminal import states", () => { + const runningImport = createImportRunState(); + const importedPage = recordImportPage(runningImport, { + page: 1, + repositories: 100, + rate_limit: rateLimit, + }); + + expect(getImportTerminalText({ ...importedPage, status: "completed" })).toBe( + "Imported 100 repositories across 1 page(s).", + ); + expect(getImportTerminalText({ ...importedPage, status: "cancelled" })).toBe( + "Import cancelled after 1 page(s) and 100 repositories.", + ); + expect(getImportTerminalText({ ...importedPage, status: "rate_limited" })).toBe( + "Import paused by GitHub rate limits after 1 page(s). Try again later.", + ); + expect( + getImportTerminalText({ + ...importedPage, + status: "failed", + errors: ["GitHub request failed."], + }), + ).toBe("GitHub request failed."); + expect(getImportTerminalText(runningImport)).toBe("Import running."); + }); + + it("sorts observed field names", () => { + expect(sortObservedFieldNames(new Set(["updated_at", "full_name", "id"]))).toEqual([ + "full_name", + "id", + "updated_at", + ]); + }); +}); diff --git a/apps/web/src/lib/import-messages.ts b/apps/web/src/lib/import-messages.ts new file mode 100644 index 0000000..bcbdddc --- /dev/null +++ b/apps/web/src/lib/import-messages.ts @@ -0,0 +1,33 @@ +import type { ImportRunState } from "@forage/core"; +import type { ImportWorkerPhase } from "./import-worker"; + +export function getImportProgressText( + importRun: ImportRunState, + phase: ImportWorkerPhase, + page: number, +) { + if (phase === "analyzing") { + return `Analyzing page ${page} in browser worker after importing ${importRun.repositories} repositories...`; + } + return `Importing page ${page}; ${importRun.repositories} repositories stored so far...`; +} + +export function getImportTerminalText(importRun: ImportRunState) { + if (importRun.status === "completed") { + return `Imported ${importRun.repositories} repositories across ${importRun.pages} page(s).`; + } + if (importRun.status === "cancelled") { + return `Import cancelled after ${importRun.pages} page(s) and ${importRun.repositories} repositories.`; + } + if (importRun.status === "rate_limited") { + return `Import paused by GitHub rate limits after ${importRun.pages} page(s). Try again later.`; + } + if (importRun.status === "failed") { + return importRun.errors[0] || "Import failed."; + } + return "Import running."; +} + +export function sortObservedFieldNames(observedFieldNames: Iterable) { + return Array.from(observedFieldNames).sort(); +} diff --git a/apps/web/src/lib/import-pipeline.ts b/apps/web/src/lib/import-pipeline.ts new file mode 100644 index 0000000..ae102a2 --- /dev/null +++ b/apps/web/src/lib/import-pipeline.ts @@ -0,0 +1,107 @@ +import { analyzeRepositories } from "@forage/analysis"; +import { + cancelImport, + completeImport, + createImportRunState, + failImport, + type ImportRunState, + rateLimitImport, + recordImportPage, +} from "@forage/core"; +import { WorkerApi, WorkerApiError } from "./api"; +import { + reconcileImportedRepositories, + saveAnalysisResults, + saveLocalLibraryProfile, + saveRepositories, +} from "./db"; +import type { ImportWorkerPhase, StartRepositoryImportInput } from "./import-worker"; + +export interface ImportPipelineInput extends StartRepositoryImportInput { + signal: AbortSignal; + isCancelRequested: () => boolean; +} + +export interface ImportPipelineProgress { + importRun: ImportRunState; + phase: ImportWorkerPhase; + page: number; + observedFieldNames: Set; +} + +export interface ImportPipelineResult { + importRun: ImportRunState; + observedFieldNames: Set; +} + +export async function runRepositoryImportPipeline( + input: ImportPipelineInput, + onProgress: (progress: ImportPipelineProgress) => void, +): Promise { + let importRun = createImportRunState(); + const observedFieldNames = new Set(); + const importedRepositoryIds = new Set(); + const api = new WorkerApi(input.workerOrigin); + + try { + let page: number | null = 1; + + while (page) { + onProgress({ importRun, phase: "importing", page, observedFieldNames }); + const result = await api.getStarredPage(page, 100, input.signal); + + await saveRepositories(result.repositories); + for (const repository of result.repositories) { + importedRepositoryIds.add(repository.github_id); + } + onProgress({ importRun, phase: "analyzing", page, observedFieldNames }); + await saveAnalysisResults(analyzeRepositories(result.repositories)); + + importRun = recordImportPage(importRun, { + page, + repositories: result.repositories.length, + rate_limit: result.rate_limit, + }); + for (const fieldName of result.raw_field_names) observedFieldNames.add(fieldName); + + page = result.next_page; + } + + const completedAt = new Date().toISOString(); + await reconcileImportedRepositories(importedRepositoryIds); + await saveLocalLibraryProfile({ + github_login: input.sessionUser?.login ?? null, + github_user_id: input.sessionUser?.id ?? null, + repository_count: importRun.repositories, + updated_at: completedAt, + }); + importRun = completeImport(importRun, completedAt); + } catch (error) { + if (importRun.status === "running") { + const errorMessage = error instanceof Error ? error.message : "Import failed."; + if (input.isCancelRequested() || isAbortError(error)) { + importRun = cancelImport(importRun); + } else if (isRateLimitError(error)) { + importRun = rateLimitImport(importRun, errorMessage, error.rateLimit); + } else { + importRun = failImport(importRun, errorMessage); + } + } + } + + return { + importRun, + observedFieldNames, + }; +} + +function isAbortError(error: unknown) { + return error instanceof DOMException && error.name === "AbortError"; +} + +function isRateLimitError(error: unknown): error is WorkerApiError { + return ( + error instanceof WorkerApiError && + (error.status === 429 || (error.status === 403 && error.rateLimit?.remaining === "0")) + ); +} diff --git a/apps/web/src/workers/import.worker.ts b/apps/web/src/workers/import.worker.ts index beec41e..0d43964 100644 --- a/apps/web/src/workers/import.worker.ts +++ b/apps/web/src/workers/import.worker.ts @@ -1,26 +1,14 @@ -import { analyzeRepositories } from "@forage/analysis"; +import { type ImportRunState, importRunStateToEvent } from "@forage/core"; +import { saveImportEvent } from "../lib/db"; import { - cancelImport, - completeImport, - createImportRunState, - failImport, - type ImportRunState, - importRunStateToEvent, - rateLimitImport, - recordImportPage, -} from "@forage/core"; -import { WorkerApi, WorkerApiError } from "../lib/api"; -import { - reconcileImportedRepositories, - saveAnalysisResults, - saveImportEvent, - saveLocalLibraryProfile, - saveRepositories, -} from "../lib/db"; + getImportProgressText, + getImportTerminalText, + sortObservedFieldNames, +} from "../lib/import-messages"; +import { type ImportPipelineProgress, runRepositoryImportPipeline } from "../lib/import-pipeline"; import type { ImportWorkerCompleteMessage, ImportWorkerErrorMessage, - ImportWorkerPhase, ImportWorkerProgressMessage, ImportWorkerRequest, ImportWorkerStartMessage, @@ -53,56 +41,15 @@ async function runImport(message: ImportWorkerStartMessage) { activeImportController = new AbortController(); cancelRequested = false; - let importRun = createImportRunState(); - const observedFieldNames = new Set(); - const importedRepositoryIds = new Set(); - const api = new WorkerApi(message.workerOrigin); - - try { - let page: number | null = 1; - - while (page) { - postProgress(message.id, importRun, "importing", page, observedFieldNames); - const result = await api.getStarredPage(page, 100, activeImportController.signal); - - await saveRepositories(result.repositories); - for (const repository of result.repositories) { - importedRepositoryIds.add(repository.github_id); - } - postProgress(message.id, importRun, "analyzing", page, observedFieldNames); - await saveAnalysisResults(analyzeRepositories(result.repositories)); - - importRun = recordImportPage(importRun, { - page, - repositories: result.repositories.length, - rate_limit: result.rate_limit, - }); - for (const fieldName of result.raw_field_names) observedFieldNames.add(fieldName); - - page = result.next_page; - } - - const completedAt = new Date().toISOString(); - await reconcileImportedRepositories(importedRepositoryIds); - await saveLocalLibraryProfile({ - github_login: message.sessionUser?.login ?? null, - github_user_id: message.sessionUser?.id ?? null, - repository_count: importRun.repositories, - updated_at: completedAt, - }); - importRun = completeImport(importRun, completedAt); - } catch (error) { - if (importRun.status === "running") { - const errorMessage = error instanceof Error ? error.message : "Import failed."; - if (cancelRequested || isAbortError(error)) { - importRun = cancelImport(importRun); - } else if (isRateLimitError(error)) { - importRun = rateLimitImport(importRun, errorMessage, error.rateLimit); - } else { - importRun = failImport(importRun, errorMessage); - } - } - } + const { importRun, observedFieldNames } = await runRepositoryImportPipeline( + { + workerOrigin: message.workerOrigin, + sessionUser: message.sessionUser, + signal: activeImportController.signal, + isCancelRequested: () => cancelRequested, + }, + (progress) => postProgress(message.id, progress), + ); try { await saveImportEvent(importRunStateToEvent(message.id, importRun)); @@ -122,10 +69,7 @@ async function runImport(message: ImportWorkerStartMessage) { function postProgress( id: string, - importRun: ImportRunState, - phase: ImportWorkerPhase, - page: number, - observedFieldNames: Set, + { importRun, phase, page, observedFieldNames }: ImportPipelineProgress, ) { self.postMessage({ id, @@ -134,7 +78,7 @@ function postProgress( phase, page, message: getImportProgressText(importRun, phase, page), - observedFieldNames: Array.from(observedFieldNames).sort(), + observedFieldNames: sortObservedFieldNames(observedFieldNames), } satisfies ImportWorkerProgressMessage); } @@ -144,7 +88,7 @@ function postComplete(id: string, importRun: ImportRunState, observedFieldNames: type: "import:complete", importRun, message: getImportTerminalText(importRun), - observedFieldNames: Array.from(observedFieldNames).sort(), + observedFieldNames: sortObservedFieldNames(observedFieldNames), } satisfies ImportWorkerCompleteMessage); } @@ -159,40 +103,6 @@ function postError( type: "import:error", importRun, message, - observedFieldNames: Array.from(observedFieldNames).sort(), + observedFieldNames: sortObservedFieldNames(observedFieldNames), } satisfies ImportWorkerErrorMessage); } - -function getImportProgressText(importRun: ImportRunState, phase: ImportWorkerPhase, page: number) { - if (phase === "analyzing") { - return `Analyzing page ${page} in browser worker after importing ${importRun.repositories} repositories...`; - } - return `Importing page ${page}; ${importRun.repositories} repositories stored so far...`; -} - -function getImportTerminalText(importRun: ImportRunState) { - if (importRun.status === "completed") { - return `Imported ${importRun.repositories} repositories across ${importRun.pages} page(s).`; - } - if (importRun.status === "cancelled") { - return `Import cancelled after ${importRun.pages} page(s) and ${importRun.repositories} repositories.`; - } - if (importRun.status === "rate_limited") { - return `Import paused by GitHub rate limits after ${importRun.pages} page(s). Try again later.`; - } - if (importRun.status === "failed") { - return importRun.errors[0] || "Import failed."; - } - return "Import running."; -} - -function isAbortError(error: unknown) { - return error instanceof DOMException && error.name === "AbortError"; -} - -function isRateLimitError(error: unknown): error is WorkerApiError { - return ( - error instanceof WorkerApiError && - (error.status === 429 || (error.status === 403 && error.rateLimit?.remaining === "0")) - ); -}