From 2d94149793f86aebd53634c940fcf46f2b01c00d Mon Sep 17 00:00:00 2001 From: Chase J <54216608+chajac@users.noreply.github.com> Date: Mon, 14 Sep 2026 03:33:16 +0100 Subject: [PATCH 1/2] feat(flows): record environment variable reads during pull --- .changeset/flow-env-var-names.md | 9 + src/core/envVarAnalysis/missing.test.ts | 67 ++++++ src/core/envVarAnalysis/missing.ts | 31 +++ src/core/messages/flows.test.ts | 68 ++++++ src/core/messages/flows.ts | 77 +------ src/core/messages/flowsPull.ts | 102 +++++++++ src/core/runtimeEnvVars.test.ts | 27 +++ src/core/runtimeEnvVars.ts | 46 ++++ src/domains/doctor/checks/fileAssets.ts | 24 +- src/domains/flows/pull/bundle.test.ts | 2 + src/domains/flows/pull/bundle.ts | 24 +- src/domains/flows/pull/bundleEnvVars.test.ts | 217 +++++++++++++++++++ src/domains/flows/pull/bundleOrder.test.ts | 1 + src/domains/flows/pull/bundleTags.test.ts | 1 + src/domains/flows/pull/collectFlowEnvVars.ts | 44 ++++ src/domains/flows/pull/handler.test.ts | 6 +- src/domains/flows/pull/previousPull.ts | 33 +++ src/domains/flows/pull/pullSafety.test.ts | 6 +- src/domains/flows/pull/reportPull.ts | 17 +- src/domains/flows/pull/safety.test.ts | 50 ++--- src/domains/flows/pull/stage.test.ts | 9 + src/domains/flows/pull/stage.ts | 56 ++--- src/domains/flows/readCachedTags.test.ts | 39 +++- src/domains/flows/resolveTags.test.ts | 13 +- src/domains/runner/run.manifestStamp.test.ts | 3 +- src/shell/manifest/io.test.ts | 39 +++- src/shell/manifest/io.ts | 6 + src/shell/manifest/lookup.test.ts | 11 +- src/shell/manifest/manifest.testUtils.ts | 15 ++ src/shell/manifest/types.ts | 6 + 30 files changed, 858 insertions(+), 191 deletions(-) create mode 100644 .changeset/flow-env-var-names.md create mode 100644 src/core/envVarAnalysis/missing.test.ts create mode 100644 src/core/envVarAnalysis/missing.ts create mode 100644 src/core/messages/flowsPull.ts create mode 100644 src/core/runtimeEnvVars.test.ts create mode 100644 src/core/runtimeEnvVars.ts create mode 100644 src/domains/flows/pull/bundleEnvVars.test.ts create mode 100644 src/domains/flows/pull/collectFlowEnvVars.ts create mode 100644 src/domains/flows/pull/previousPull.ts create mode 100644 src/shell/manifest/manifest.testUtils.ts diff --git a/.changeset/flow-env-var-names.md b/.changeset/flow-env-var-names.md new file mode 100644 index 000000000..06e8b9499 --- /dev/null +++ b/.changeset/flow-env-var-names.md @@ -0,0 +1,9 @@ +--- +"@qawolf/cli": minor +--- + +`qawolf flows pull` now records environment variable names found by static analysis of each flow. It follows module initialization and reachable calls, including literal names passed through helpers such as `requireEnv("NAME")`. Dormant function bodies and unused methods do not add reads merely because their module was imported. + +The pull result warns about variables the flows read that the environment does not set, grouping them by name and reporting how many flows read each one. Reads may be optional. Runtime and operating-system settings are excluded from these warnings. + +Dynamic keys and unresolved local calls can hide additional reads. Affected flows are counted in the pull result and marked with `envVarsMayBeIncomplete` in the manifest. Recorded names are a static approximation, not a complete list of required variables, and local edits do not refresh them. Older manifests leave the analysis fields absent until the next pull. diff --git a/src/core/envVarAnalysis/missing.test.ts b/src/core/envVarAnalysis/missing.test.ts new file mode 100644 index 000000000..d5a83ae1e --- /dev/null +++ b/src/core/envVarAnalysis/missing.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "bun:test"; + +import { findMissingEnvVars } from "./missing.js"; + +describe("findMissingEnvVars", () => { + const byFlow = (entries: Record) => + new Map( + Object.entries(entries).map(([path, names]) => [ + path, + { names, mayBeIncomplete: false }, + ]), + ); + + it("reports a variable no flow's environment defines", () => { + expect( + findMissingEnvVars({ + byFlow: byFlow({ "a.flow.ts": ["PRESENT", "ABSENT"] }), + definedNames: new Set(["PRESENT"]), + }), + ).toEqual([{ name: "ABSENT", flowCount: 1 }]); + }); + + it("counts how many flows read each missing variable", () => { + expect( + findMissingEnvVars({ + byFlow: byFlow({ + "a.flow.ts": ["SHARED"], + "b.flow.ts": ["SHARED"], + "c.flow.ts": ["RARE"], + }), + definedNames: new Set(), + }), + ).toEqual([ + { name: "SHARED", flowCount: 2 }, + { name: "RARE", flowCount: 1 }, + ]); + }); + + it("breaks ties on count by name", () => { + expect( + findMissingEnvVars({ + byFlow: byFlow({ "a.flow.ts": ["ZED", "ALPHA"] }), + definedNames: new Set(), + }).map((m) => m.name), + ).toEqual(["ALPHA", "ZED"]); + }); + + it("ignores runner- and OS-provided variables", () => { + expect( + findMissingEnvVars({ + byFlow: byFlow({ + "a.flow.ts": ["QAWOLF_EXAMPLE_ID", "TEAM_STORAGE_DIR", "HOME"], + }), + definedNames: new Set(), + }), + ).toEqual([]); + }); + + it("reports nothing when the environment defines everything", () => { + expect( + findMissingEnvVars({ + byFlow: byFlow({ "a.flow.ts": ["ALPHA", "BETA"] }), + definedNames: new Set(["ALPHA", "BETA"]), + }), + ).toEqual([]); + }); +}); diff --git a/src/core/envVarAnalysis/missing.ts b/src/core/envVarAnalysis/missing.ts new file mode 100644 index 000000000..8dc00e46a --- /dev/null +++ b/src/core/envVarAnalysis/missing.ts @@ -0,0 +1,31 @@ +import { isRuntimeProvidedEnvVar } from "~/core/runtimeEnvVars.js"; + +import type { FlowEnvVars } from "./types.js"; + +/** One variable flows read that the environment does not define. */ +export type MissingEnvVar = { + name: string; + flowCount: number; +}; + +/** Aggregates missing variables across flows, excluding runner and OS variables. */ +export function findMissingEnvVars(args: { + byFlow: ReadonlyMap; + definedNames: ReadonlySet; +}): MissingEnvVar[] { + const flowCountByName = new Map(); + + for (const { names } of args.byFlow.values()) { + for (const name of names) { + if (args.definedNames.has(name)) continue; + if (isRuntimeProvidedEnvVar(name)) continue; + flowCountByName.set(name, (flowCountByName.get(name) ?? 0) + 1); + } + } + + // Most-used first, so the line that matters most is the one that survives + // any truncation downstream. + return [...flowCountByName] + .map(([name, flowCount]) => ({ name, flowCount })) + .sort((a, b) => b.flowCount - a.flowCount || a.name.localeCompare(b.name)); +} diff --git a/src/core/messages/flows.test.ts b/src/core/messages/flows.test.ts index 1bcee8da0..f5d007b52 100644 --- a/src/core/messages/flows.test.ts +++ b/src/core/messages/flows.test.ts @@ -44,3 +44,71 @@ describe("flowsMessages.pull.summary", () => { ); }); }); + +describe("flowsMessages.pull.summary incomplete flows", () => { + const base = { + envDir: "/tmp/env", + flowCount: 1, + envVarCount: 0, + flowsWithTeamStorageRefs: [], + assetDownloadedCount: 0, + assetReusedCount: 0, + assetSkippedCount: 0, + }; + + it("says nothing when every key is knowable", () => { + expect( + flowsMessages.pull.summary({ ...base, incompleteFlowCount: 0 }, "/tmp/a"), + ).toBe("Pulled 1 flow into /tmp/env"); + }); + + // Said once as a count, not marked on each of the flows it covers. + it("counts the flows whose list is a floor", () => { + expect( + flowsMessages.pull.summary({ ...base, incompleteFlowCount: 3 }, "/tmp/a"), + ).toBe( + [ + "Pulled 1 flow into /tmp/env", + "3 flows may read more variables than listed; static analysis could not resolve every read.", + ].join("\n"), + ); + }); +}); + +describe("flowsMessages.pull.missingEnvVars", () => { + it("names one variable and how many flows read it", () => { + expect( + flowsMessages.pull.missingEnvVars([{ name: "LOGIN_PW", flowCount: 1 }]), + ).toBe( + [ + "1 environment variable is read by flows but not set in this environment (some reads may be optional):", + " - LOGIN_PW (read by 1 flow)", + ].join("\n"), + ); + }); + + it("pluralizes across several variables and flows", () => { + expect( + flowsMessages.pull.missingEnvVars([ + { name: "SHARED", flowCount: 12 }, + { name: "RARE", flowCount: 1 }, + ]), + ).toBe( + [ + "2 environment variables are read by flows but not set in this environment (some reads may be optional):", + " - SHARED (read by 12 flows)", + " - RARE (read by 1 flow)", + ].join("\n"), + ); + }); + + it("truncates a long list", () => { + const missing = Array.from({ length: 9 }, (_, i) => ({ + name: `VAR_${String(i)}`, + flowCount: 1, + })); + expect(flowsMessages.pull.missingEnvVars(missing)).toContain( + " ... and 4 more", + ); + }); +}); diff --git a/src/core/messages/flows.ts b/src/core/messages/flows.ts index 3069578d7..da7ff2617 100644 --- a/src/core/messages/flows.ts +++ b/src/core/messages/flows.ts @@ -1,14 +1,6 @@ import { pluralize } from "~/core/pluralize.js"; -type PullSummaryInput = { - readonly envDir: string; - readonly flowCount: number; - readonly envVarCount: number; - readonly flowsWithTeamStorageRefs: readonly string[]; - readonly assetDownloadedCount?: number | undefined; - readonly assetReusedCount?: number | undefined; - readonly assetSkippedCount?: number | undefined; -}; +import { flowsPullMessages } from "./flowsPull.js"; export const flowsMessages = { title: "Flows", @@ -61,72 +53,7 @@ export const flowsMessages = { requiresEnv: "An environment is required. Pass --env or set QAWOLF_ENVIRONMENT.", }, - pull: { - requiresEnv: - "An environment is required. Pass --env or set QAWOLF_ENVIRONMENT.", - downloadingBundle: "Downloading flows bundle", - fetchingEnvVars: "Fetching environment variables", - fetchingTags: "Fetching flow tags", - downloadComplete: "Downloaded flows bundle and environment variables", - needsYesError: "Re-run with --yes to overwrite locally-modified files", - aborted: "Aborted; no changes.", - extractingBundle: "Extracting bundle", - downloadingTeamStorageAssets: "Downloading team-storage assets", - downloadingTeamStorageAssetsProgress: (current: number, total: number) => - `Downloading team-storage assets (${String(current)}/${String(total)})`, - teamStorageRequiresTeam: - "Team storage needs a team. Pull an environment to name its team, choose a workspace with 'qawolf auth switch', or use a team API key.", - summary: (result: PullSummaryInput, assetsAbs: string) => { - const flows = pluralize(result.flowCount, "flow"); - const envVars = - result.envVarCount === 0 - ? "" - : ` and ${pluralize(result.envVarCount, "environment variable")}`; - const lines = [`Pulled ${flows}${envVars} into ${result.envDir}`]; - if (result.flowsWithTeamStorageRefs.length > 0) { - const refs = pluralize(result.flowsWithTeamStorageRefs.length, "flow"); - lines.push(`Team-storage assets referenced by ${refs}:`); - for (const path of result.flowsWithTeamStorageRefs) { - lines.push(` - ${path}`); - } - } - const downloaded = result.assetDownloadedCount ?? 0; - const reused = result.assetReusedCount ?? 0; - const skipped = result.assetSkippedCount ?? 0; - if (downloaded > 0 || reused > 0 || skipped > 0) { - let assetSummary = `Downloaded ${pluralize( - downloaded, - "team-storage asset", - )}`; - if (reused > 0) { - assetSummary += ` and reused ${pluralize( - reused, - "team-storage asset", - )}`; - } - assetSummary += ` into ${assetsAbs}`; - if (skipped > 0) { - assetSummary += ` (${pluralize( - skipped, - "unsafe or unsupported asset", - )} skipped)`; - } - lines.push(assetSummary); - } - return lines.join("\n"); - }, - symlinkRejected: (path: string) => `symlink entry rejected: ${path}`, - unknownEntrySize: (path: string) => - `entry with unknown size rejected: ${path}`, - entryTooLarge: (path: string, size: number, maxBytes: number) => - `entry exceeds max size (${path}): ${String(size)} > ${String(maxBytes)}`, - localModsWouldOverwrite: ( - count: number, - envDir: string, - fileList: string, - ) => - `${count} locally-modified file(s) under ${envDir} would be overwritten:\n${fileList}`, - }, + pull: flowsPullMessages, ensureDeps: { multiPackagePattern: (count: number, listed: string) => `Pattern matches flows from ${count} packages — narrow it to a single package:\n${listed}\n\nHint: pass a pattern scoped to one package, e.g \`qawolf flows run '.qawolf//**'\`.`, diff --git a/src/core/messages/flowsPull.ts b/src/core/messages/flowsPull.ts new file mode 100644 index 000000000..e57ff37ae --- /dev/null +++ b/src/core/messages/flowsPull.ts @@ -0,0 +1,102 @@ +import type { MissingEnvVar } from "~/core/envVarAnalysis/missing.js"; +import { pluralize } from "~/core/pluralize.js"; + +type PullSummaryInput = { + readonly envDir: string; + readonly flowCount: number; + readonly envVarCount: number; + readonly flowsWithTeamStorageRefs: readonly string[]; + readonly incompleteFlowCount?: number | undefined; + readonly assetDownloadedCount?: number | undefined; + readonly assetReusedCount?: number | undefined; + readonly assetSkippedCount?: number | undefined; +}; + +// Enough to act on without turning a spinner's stop message into a wall of +// text; the full set is always in the JSON output. +const maxListedNames = 5; + +export const flowsPullMessages = { + requiresEnv: + "An environment is required. Pass --env or set QAWOLF_ENVIRONMENT.", + downloadingBundle: "Downloading flows bundle", + fetchingEnvVars: "Fetching environment variables", + fetchingTags: "Fetching flow tags", + downloadComplete: "Downloaded flows bundle and environment variables", + needsYesError: "Re-run with --yes to overwrite locally-modified files", + aborted: "Aborted; no changes.", + extractingBundle: "Extracting bundle", + downloadingTeamStorageAssets: "Downloading team-storage assets", + downloadingTeamStorageAssetsProgress: (current: number, total: number) => + `Downloading team-storage assets (${String(current)}/${String(total)})`, + teamStorageRequiresTeam: + "Team storage needs a team. Pull an environment to name its team, choose a workspace with 'qawolf auth switch', or use a team API key.", + summary: (result: PullSummaryInput, assetsAbs: string) => { + const flows = pluralize(result.flowCount, "flow"); + const envVars = + result.envVarCount === 0 + ? "" + : ` and ${pluralize(result.envVarCount, "environment variable")}`; + const lines = [`Pulled ${flows}${envVars} into ${result.envDir}`]; + if (result.flowsWithTeamStorageRefs.length > 0) { + const refs = pluralize(result.flowsWithTeamStorageRefs.length, "flow"); + lines.push(`Team-storage assets referenced by ${refs}:`); + for (const path of result.flowsWithTeamStorageRefs) { + lines.push(` - ${path}`); + } + } + // Shared dynamic helpers can affect every flow; report their impact once. + const incomplete = result.incompleteFlowCount ?? 0; + if (incomplete > 0) { + lines.push( + `${pluralize(incomplete, "flow")} may read more variables than listed; static analysis could not resolve every read.`, + ); + } + const downloaded = result.assetDownloadedCount ?? 0; + const reused = result.assetReusedCount ?? 0; + const skipped = result.assetSkippedCount ?? 0; + if (downloaded > 0 || reused > 0 || skipped > 0) { + let assetSummary = `Downloaded ${pluralize( + downloaded, + "team-storage asset", + )}`; + if (reused > 0) { + assetSummary += ` and reused ${pluralize( + reused, + "team-storage asset", + )}`; + } + assetSummary += ` into ${assetsAbs}`; + if (skipped > 0) { + assetSummary += ` (${pluralize( + skipped, + "unsafe or unsupported asset", + )} skipped)`; + } + lines.push(assetSummary); + } + return lines.join("\n"); + }, + missingEnvVars: (missing: readonly MissingEnvVar[]) => { + // A read does not prove a variable is required by the flow. + const lines = [ + `${pluralize(missing.length, "environment variable")} ${ + missing.length === 1 ? "is" : "are" + } read by flows but not set in this environment (some reads may be optional):`, + ]; + for (const { name, flowCount } of missing.slice(0, maxListedNames)) { + lines.push(` - ${name} (read by ${pluralize(flowCount, "flow")})`); + } + if (missing.length > maxListedNames) { + lines.push(` ... and ${String(missing.length - maxListedNames)} more`); + } + return lines.join("\n"); + }, + symlinkRejected: (path: string) => `symlink entry rejected: ${path}`, + unknownEntrySize: (path: string) => + `entry with unknown size rejected: ${path}`, + entryTooLarge: (path: string, size: number, maxBytes: number) => + `entry exceeds max size (${path}): ${String(size)} > ${String(maxBytes)}`, + localModsWouldOverwrite: (count: number, envDir: string, fileList: string) => + `${count} locally-modified file(s) under ${envDir} would be overwritten:\n${fileList}`, +} as const; diff --git a/src/core/runtimeEnvVars.test.ts b/src/core/runtimeEnvVars.test.ts new file mode 100644 index 000000000..8064d0822 --- /dev/null +++ b/src/core/runtimeEnvVars.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "bun:test"; + +import { isRuntimeProvidedEnvVar } from "./runtimeEnvVars.js"; + +describe("isRuntimeProvidedEnvVar", () => { + it("treats runner and OS variables as provided", () => { + for (const name of [ + "QAWOLF_EXAMPLE_ID", + "QAWOLF_EXAMPLE_DIR", + "TEAM_STORAGE_DIR", + "RUN_INPUT_PATH", + "RUN_EXAMPLE_DIR", + "PW_EXAMPLE_OPTION", + "PLAYWRIGHT_EXAMPLE_OPTION", + "HOME", + "CI", + ]) { + expect(isRuntimeProvidedEnvVar(name)).toBe(true); + } + }); + + it("treats ordinary environment variables as not provided", () => { + for (const name of ["EXAMPLE_PASSWORD", "EXAMPLE_EMAIL", "EXAMPLE_URL"]) { + expect(isRuntimeProvidedEnvVar(name)).toBe(false); + } + }); +}); diff --git a/src/core/runtimeEnvVars.ts b/src/core/runtimeEnvVars.ts new file mode 100644 index 000000000..599699878 --- /dev/null +++ b/src/core/runtimeEnvVars.ts @@ -0,0 +1,46 @@ +export type FileAssetCategory = "file-asset" | "mobile-input"; + +// Doctor checks file-backed inputs; missing-variable warnings exclude them. +export const fileAssetVarPatterns: readonly { + readonly pattern: string; + readonly category: FileAssetCategory; +}[] = [ + { pattern: "TEAM_STORAGE_DIR", category: "file-asset" }, + { pattern: "QAWOLF_*_DIR", category: "file-asset" }, + { pattern: "RUN_*_DIR", category: "mobile-input" }, + { pattern: "RUN_INPUT_PATH", category: "mobile-input" }, +]; + +/** A pattern's `*` as a regular expression: one or more word characters. */ +export const expandEnvVarPattern = (pattern: string): string => + pattern.replace(/\*/g, "\\w+"); + +// Runtime settings are intentionally excluded from missing-variable warnings; +// their absence from a pulled .env does not imply a missing flow credential. +const runtimeProvidedNames = new Set([ + // OS paths and conventional process controls; not guaranteed to be set. + "HOME", + "PATH", + "PWD", + "TMPDIR", + "TEMP", + "TMP", + "NODE_ENV", + "CI", +]); + +const runtimeProvidedPatterns = [ + /^QAWOLF_/, + /^PW_/, + /^PLAYWRIGHT_/, + ...fileAssetVarPatterns.map( + ({ pattern }) => new RegExp(`^${expandEnvVarPattern(pattern)}$`), + ), +]; + +export function isRuntimeProvidedEnvVar(name: string): boolean { + return ( + runtimeProvidedNames.has(name) || + runtimeProvidedPatterns.some((pattern) => pattern.test(name)) + ); +} diff --git a/src/domains/doctor/checks/fileAssets.ts b/src/domains/doctor/checks/fileAssets.ts index 4dca7d105..956ee5bf3 100644 --- a/src/domains/doctor/checks/fileAssets.ts +++ b/src/domains/doctor/checks/fileAssets.ts @@ -1,32 +1,22 @@ import { relative } from "node:path"; import { doctorMessages } from "~/core/messages/index.js"; +import { + expandEnvVarPattern, + type FileAssetCategory, + fileAssetVarPatterns, +} from "~/core/runtimeEnvVars.js"; import type { CheckResult } from "~/domains/doctor/types.js"; import { errorMessage } from "~/core/errors.js"; -type FileAssetCategory = "file-asset" | "mobile-input"; - -const fileAssetVarPatterns: readonly { - readonly pattern: string; - readonly category: FileAssetCategory; -}[] = [ - { pattern: "TEAM_STORAGE_DIR", category: "file-asset" }, - { pattern: "QAWOLF_*_DIR", category: "file-asset" }, - { pattern: "RUN_*_DIR", category: "mobile-input" }, - { pattern: "RUN_INPUT_PATH", category: "mobile-input" }, -]; - -const expandPattern = (pattern: string): string => - pattern.replace(/\*/g, "\\w+"); - const fileAssetVarRe = new RegExp( - `\\b(?:${fileAssetVarPatterns.map(({ pattern }) => expandPattern(pattern)).join("|")})\\b`, + `\\b(?:${fileAssetVarPatterns.map(({ pattern }) => expandEnvVarPattern(pattern)).join("|")})\\b`, "g", ); const compiledByCategory = fileAssetVarPatterns.map( ({ pattern, category }) => ({ - re: new RegExp(`^${expandPattern(pattern)}$`), + re: new RegExp(`^${expandEnvVarPattern(pattern)}$`), category, }), ); diff --git a/src/domains/flows/pull/bundle.test.ts b/src/domains/flows/pull/bundle.test.ts index b5edeaf1b..af85b2738 100644 --- a/src/domains/flows/pull/bundle.test.ts +++ b/src/domains/flows/pull/bundle.test.ts @@ -116,6 +116,7 @@ describe("buildManifest", () => { wrapperName: string | undefined; qawolfCommittedAt: string | undefined; tags: undefined; + envVarsByFlow: undefined; } => ({ envId: "env-x", bundleDir: workDir, @@ -125,6 +126,7 @@ describe("buildManifest", () => { wrapperName: undefined, qawolfCommittedAt: undefined, tags: undefined, + envVarsByFlow: undefined, }); it("walks .flow.ts and .flow.js files, ignores other extensions", async () => { diff --git a/src/domains/flows/pull/bundle.ts b/src/domains/flows/pull/bundle.ts index 67e91b7c1..5c45aa716 100644 --- a/src/domains/flows/pull/bundle.ts +++ b/src/domains/flows/pull/bundle.ts @@ -1,9 +1,9 @@ import { join, relative } from "node:path"; +import type { FlowEnvVars } from "~/core/envVarAnalysis/types.js"; import { isFlowFile } from "~/core/flowMeta.js"; -import { walkFiles } from "~/shell/walkFiles.js"; - import { toPosix } from "~/core/repoRelativePath.js"; +import { walkFiles } from "~/shell/walkFiles.js"; import { hashFile } from "~/shell/manifest/io.js"; import type { Manifest } from "~/shell/manifest/types.js"; @@ -41,6 +41,13 @@ export async function flattenSingleWrapper( return innerName; } +// Flow files under `root`, relative to it and sorted, so the manifest lists +// them in the same order on every pull. +async function flowPathsIn(root: string, fs: Fs): Promise { + const found = await walkFiles(root, isFlowFile, fs); + return found.map((path) => toPosix(relative(root, path))).sort(); +} + // GitHub's tarball archives wrap content in `--/`, where // the trailing 40 hex chars are the commit SHA. Defensive: returns undefined // when the wrapper name doesn't match — keeps manifest writes infallible. @@ -72,6 +79,9 @@ export async function buildManifest( wrapperName: string | undefined; qawolfCommittedAt: string | undefined; tags: FetchedTags | undefined; + // A flow missing from the map records no env vars rather than an empty + // set: unknown, not none. + envVarsByFlow: ReadonlyMap | undefined; }, fs: Fs = makeDefaultFs(), ): Promise { @@ -86,6 +96,9 @@ export async function buildManifest( // Left unset when the fetch did not cover this file — unknown, not // untagged. tags: args.tags?.byPath.get(toPosix(rel)), + envVars: args.envVarsByFlow?.get(toPosix(rel))?.names, + envVarsMayBeIncomplete: args.envVarsByFlow?.get(toPosix(rel)) + ?.mayBeIncomplete, })), ); @@ -103,13 +116,6 @@ export async function buildManifest( }; } -// Flow files under `root`, relative to it and sorted, so the manifest lists -// them in the same order on every pull. -async function flowPathsIn(root: string, fs: Fs): Promise { - const found = await walkFiles(root, isFlowFile, fs); - return found.map((path) => toPosix(relative(root, path))).sort(); -} - // Samples the mtime of any flow file in the bundle. GitHub-archive bundles // share one mtime across all entries (preserved by extract.ts). Returns // undefined when the bundle has no flow files. Sample BEFORE any local diff --git a/src/domains/flows/pull/bundleEnvVars.test.ts b/src/domains/flows/pull/bundleEnvVars.test.ts new file mode 100644 index 000000000..3276a56bf --- /dev/null +++ b/src/domains/flows/pull/bundleEnvVars.test.ts @@ -0,0 +1,217 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; + +import { readManifest } from "~/shell/manifest/io.js"; +import { buildManifest } from "./bundle.js"; +import { buildBundle } from "./pull.fixtures.js"; +import { stageBundle } from "./stage.js"; + +let workDir = ""; + +beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), "qawolf-bundle-env-vars-")); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); +}); + +const baseArgs = () => ({ + envVarsByFlow: undefined, + envId: "env-x", + envSlug: undefined, + envName: undefined, + bundleDir: workDir, + cliFlowsVersion: "0.4.0", + now: new Date("2026-05-10T12:00:00.000Z"), + envVarsFetchedAt: undefined, + wrapperName: undefined, + qawolfCommittedAt: undefined, + tags: undefined, +}); + +const entryFor = ( + manifest: Awaited>, + path: string, +) => manifest.flows.find((f) => f.path === path); + +async function stage(names: string[]): Promise { + for (const name of names) { + const p = join(workDir, name); + await mkdir(dirname(p), { recursive: true }); + await writeFile(p, "// flow", "utf8"); + } +} + +describe("buildManifest env vars", () => { + it("leaves both fields unset when no scan was supplied", async () => { + await stage(["src/flows/a.flow.ts"]); + + const manifest = await buildManifest(baseArgs()); + + const entry = entryFor(manifest, "src/flows/a.flow.ts"); + expect(entry?.envVars).toBeUndefined(); + expect(entry?.envVarsMayBeIncomplete).toBeUndefined(); + }); + + it("records the scanned names per flow", async () => { + await stage(["src/flows/a.flow.ts", "src/flows/b.flow.ts"]); + + const manifest = await buildManifest({ + ...baseArgs(), + envVarsByFlow: new Map([ + [ + "src/flows/a.flow.ts", + { names: ["ALPHA", "BETA"], mayBeIncomplete: false }, + ], + ["src/flows/b.flow.ts", { names: [], mayBeIncomplete: true }], + ]), + }); + + expect(entryFor(manifest, "src/flows/a.flow.ts")?.envVars).toEqual([ + "ALPHA", + "BETA", + ]); + expect( + entryFor(manifest, "src/flows/b.flow.ts")?.envVarsMayBeIncomplete, + ).toBe(true); + }); + + it("leaves a flow the scan did not cover unset", async () => { + await stage(["src/flows/a.flow.ts", "src/flows/b.flow.ts"]); + + const manifest = await buildManifest({ + ...baseArgs(), + envVarsByFlow: new Map([ + ["src/flows/a.flow.ts", { names: ["ALPHA"], mayBeIncomplete: false }], + ]), + }); + + expect(entryFor(manifest, "src/flows/b.flow.ts")?.envVars).toBeUndefined(); + }); +}); + +describe("stageBundle env vars", () => { + const stageArgs = (destDir: string, archive: string) => ({ + tmpArchive: archive, + destAbs: destDir, + assetsAbs: join(destDir, "..", "assets"), + envId: "env-abc", + envSlug: undefined, + envName: undefined, + cliFlowsVersion: "0.4.0", + now: new Date("2026-05-10T12:00:00.000Z"), + envVarsFetchedAt: new Date("2026-05-10T12:00:00.000Z"), + tags: undefined, + }); + + it("writes scanned env vars into the manifest, following imports", async () => { + const archive = join(workDir, "bundle.tar.gz"); + await buildBundle(archive, { + flows: [ + { + name: "src/flows/a.flow.ts", + data: `import { x } from "../pages/login.js";\nexport default async () => [x(), process.env.LOGIN_USER];`, + }, + { + name: "src/pages/login.ts", + data: `export const x = () => process.env.LOGIN_PW;`, + }, + ], + }); + const destDir = join(workDir, "env"); + + await stageBundle({ + ...stageArgs(destDir, archive), + envVars: { LOGIN_USER: "u", LOGIN_PW: "p" }, + }); + + const manifest = await readManifest(destDir); + if (typeof manifest === "string") throw new Error(manifest); + const entry = manifest.flows.find((f) => f.path === "src/flows/a.flow.ts"); + expect(entry?.envVars).toEqual(["LOGIN_PW", "LOGIN_USER"]); + expect(entry?.envVarsMayBeIncomplete).toBe(false); + }); + + it("reports variables the environment does not define, counted by flow", async () => { + const archive = join(workDir, "bundle.tar.gz"); + await buildBundle(archive, { + flows: [ + { + name: "src/flows/a.flow.ts", + data: `export default async () => process.env.MISSING_ONE;`, + }, + { + name: "src/flows/b.flow.ts", + data: `export default async () => [process.env.MISSING_ONE, process.env.PRESENT];`, + }, + ], + }); + const destDir = join(workDir, "env"); + + const result = await stageBundle({ + ...stageArgs(destDir, archive), + envVars: { PRESENT: "yes" }, + }); + + expect(result.missingEnvVars).toEqual([ + { name: "MISSING_ONE", flowCount: 2 }, + ]); + expect(result.incompleteFlowCount).toBe(0); + }); + + it("does not report runner-provided variables as missing", async () => { + const archive = join(workDir, "bundle.tar.gz"); + await buildBundle(archive, { + flows: [ + { + name: "src/flows/a.flow.ts", + data: `export default async () => [process.env.QAWOLF_EXAMPLE_ID, process.env.HOME];`, + }, + ], + }); + const destDir = join(workDir, "env"); + + const result = await stageBundle({ + ...stageArgs(destDir, archive), + envVars: {}, + }); + + expect(result.missingEnvVars).toEqual([]); + }); + + // Unlike tags, which survive a failed fetch, these are rebuilt from the + // bundle every time — a variable a flow no longer reads must disappear. + it("does not carry stale env vars forward from a previous pull", async () => { + const destDir = join(workDir, "env"); + const first = join(workDir, "first.tar.gz"); + await buildBundle(first, { + flows: [ + { + name: "src/flows/a.flow.ts", + data: `export default async () => process.env.OLD_ONE;`, + }, + ], + }); + await stageBundle({ ...stageArgs(destDir, first), envVars: {} }); + + const second = join(workDir, "second.tar.gz"); + await buildBundle(second, { + flows: [ + { + name: "src/flows/a.flow.ts", + data: `export default async () => process.env.NEW_ONE;`, + }, + ], + }); + await stageBundle({ ...stageArgs(destDir, second), envVars: {} }); + + const manifest = await readManifest(destDir); + if (typeof manifest === "string") throw new Error(manifest); + expect( + manifest.flows.find((f) => f.path === "src/flows/a.flow.ts")?.envVars, + ).toEqual(["NEW_ONE"]); + }); +}); diff --git a/src/domains/flows/pull/bundleOrder.test.ts b/src/domains/flows/pull/bundleOrder.test.ts index 5db90dc0a..586e16add 100644 --- a/src/domains/flows/pull/bundleOrder.test.ts +++ b/src/domains/flows/pull/bundleOrder.test.ts @@ -19,6 +19,7 @@ it("sorts manifest paths after normalizing separators", async () => { wrapperName: undefined, qawolfCommittedAt: undefined, tags: undefined, + envVarsByFlow: undefined, }, { ...fs, diff --git a/src/domains/flows/pull/bundleTags.test.ts b/src/domains/flows/pull/bundleTags.test.ts index 6290b5769..84a9e6883 100644 --- a/src/domains/flows/pull/bundleTags.test.ts +++ b/src/domains/flows/pull/bundleTags.test.ts @@ -27,6 +27,7 @@ async function stage(names: string[]): Promise { } const baseArgs = () => ({ + envVarsByFlow: undefined, envId: "env-x", envSlug: undefined, envName: undefined, diff --git a/src/domains/flows/pull/collectFlowEnvVars.ts b/src/domains/flows/pull/collectFlowEnvVars.ts new file mode 100644 index 000000000..cf287a481 --- /dev/null +++ b/src/domains/flows/pull/collectFlowEnvVars.ts @@ -0,0 +1,44 @@ +import { relative } from "node:path"; + +import { findEnvAccessors } from "~/core/envVarAnalysis/accessors.js"; +import { collectEnvVarsByFlow } from "~/core/envVarAnalysis/callGraph.js"; +import type { FlowEnvVars } from "~/core/envVarAnalysis/types.js"; +import { isSourceFile } from "~/core/flowMeta.js"; +import { toPosix } from "~/core/repoRelativePath.js"; +import { createFlowProgram } from "~/shell/flowProgram.js"; +import { makeDefaultFs } from "~/shell/fs.js"; +import { walkFiles } from "~/shell/walkFiles.js"; + +type CollectFlowEnvVarsResult = { + /** Keyed by bundle-relative posix path, matching the manifest's flow paths. */ + readonly byFlow: ReadonlyMap; +}; + +// Uses real disk because the compiler loads sources and tsconfig itself. +// Run after applyTeamStorageRewrite so introduced TEAM_STORAGE_DIR reads count. +export async function collectFlowEnvVars( + bundleDir: string, +): Promise { + const sourcePaths = await walkFiles(bundleDir, isSourceFile, makeDefaultFs()); + if (sourcePaths.length === 0) return { byFlow: new Map() }; + + const program = await createFlowProgram({ bundleDir, sourcePaths }); + const accessors = findEnvAccessors( + program.compiler, + program.checker, + program.sourceFiles, + ); + const byAbsolutePath = collectEnvVarsByFlow({ + compiler: program.compiler, + checker: program.checker, + accessors, + isLocalFile: program.isLocalFile, + flowFiles: program.flowFiles, + }); + + const byFlow = new Map(); + for (const [fileName, value] of byAbsolutePath) { + byFlow.set(toPosix(relative(bundleDir, fileName)), value); + } + return { byFlow }; +} diff --git a/src/domains/flows/pull/handler.test.ts b/src/domains/flows/pull/handler.test.ts index 49eea8841..f7112a64b 100644 --- a/src/domains/flows/pull/handler.test.ts +++ b/src/domains/flows/pull/handler.test.ts @@ -91,7 +91,7 @@ function makeCtx( } describe("handleFlowsPull json mode output", () => { - it("emits env, envDir, assetsDir, fetchedAt, flowCount, envVarCount, flowsWithTeamStorageRefs, manifestPath", async () => { + it("emits env, envDir, assetsDir, fetchedAt, flowCount, envVarCount, flowsWithTeamStorageRefs, missingEnvVars, incompleteFlowCount, manifestPath", async () => { await buildBundle(bundleArchive, { flows: [ { name: "login.flow.ts", data: "// login\n" }, @@ -118,7 +118,9 @@ describe("handleFlowsPull json mode output", () => { "fetchedAt", "flowCount", "flowsWithTeamStorageRefs", + "incompleteFlowCount", "manifestPath", + "missingEnvVars", ]); expect(payload).toEqual({ assetsDir: expect.stringContaining(join(workDir, "assets")), @@ -133,6 +135,8 @@ describe("handleFlowsPull json mode output", () => { flowCount: 2, envVarCount: 2, flowsWithTeamStorageRefs: [], + missingEnvVars: [], + incompleteFlowCount: 0, manifestPath: join(destDir, manifestFilename), }); expect(JSON.parse(JSON.stringify(payload))).toEqual(payload); diff --git a/src/domains/flows/pull/previousPull.ts b/src/domains/flows/pull/previousPull.ts new file mode 100644 index 000000000..561cab1af --- /dev/null +++ b/src/domains/flows/pull/previousPull.ts @@ -0,0 +1,33 @@ +import { toPosix } from "~/core/repoRelativePath.js"; +import type { Fs } from "~/shell/fs.js"; +import { readManifest } from "~/shell/manifest/io.js"; + +import type { FetchedTags } from "./bundle.js"; + +// A failed tag fetch must not erase cached tags or break offline tag queries. +export async function carriedFromPreviousPull( + envDir: string, + fs: Fs, +): Promise<{ + tags: FetchedTags | undefined; +}> { + const previous = await readManifest(envDir, fs); + if (typeof previous === "string") { + return { tags: undefined }; + } + + const tagsByPath = new Map(); + for (const flow of previous.flows) { + // A manifest written by an older CLI on win32 may hold `\` paths; the new + // manifest looks entries up by posix path, so normalize or the carried + // values never match and vanish silently. + const path = toPosix(flow.path); + if (flow.tags !== undefined) tagsByPath.set(path, [...flow.tags]); + } + return { + tags: + previous.tagsFetchedAt === undefined + ? undefined + : { fetchedAt: new Date(previous.tagsFetchedAt), byPath: tagsByPath }, + }; +} diff --git a/src/domains/flows/pull/pullSafety.test.ts b/src/domains/flows/pull/pullSafety.test.ts index 4066521cb..72ae423b6 100644 --- a/src/domains/flows/pull/pullSafety.test.ts +++ b/src/domains/flows/pull/pullSafety.test.ts @@ -8,6 +8,7 @@ import type { Manifest } from "~/shell/manifest/types.js"; import { buildBundle } from "./pull.fixtures.js"; import { checkSafety } from "./pull.js"; import { stageBundle } from "./stage.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; let workDir = ""; let bundleArchive = ""; @@ -38,11 +39,10 @@ describe("safety + staging integration", () => { qawolfCommittedAt: undefined, tagsFetchedAt: undefined, flows: [ - { + makeManifestFlow({ path: "a.flow.ts", contentHash: await hashFile(join(destDir, "a.flow.ts")), - tags: undefined, - }, + }), ], }; await writeManifest(destDir, manifest); diff --git a/src/domains/flows/pull/reportPull.ts b/src/domains/flows/pull/reportPull.ts index 4a7b52a2d..635bf2cef 100644 --- a/src/domains/flows/pull/reportPull.ts +++ b/src/domains/flows/pull/reportPull.ts @@ -1,5 +1,8 @@ import { join } from "node:path"; +import { flowsMessages } from "~/core/messages/index.js"; + +import type { MissingEnvVar } from "~/core/envVarAnalysis/missing.js"; import { manifestFilename } from "~/shell/manifest/io.js"; import type { UI } from "~/shell/ui/index.js"; @@ -8,6 +11,8 @@ type StageResult = { readonly flowCount: number; readonly envVarCount: number; readonly flowsWithTeamStorageRefs: string[]; + readonly missingEnvVars: MissingEnvVar[]; + readonly incompleteFlowCount: number; }; type AssetResult = { @@ -17,8 +22,10 @@ type AssetResult = { }; /** - * Emits the machine-readable pull result. Human and agent modes already got - * the progress summary, so only JSON mode has anything left to say. + * Reports what the pull found once the spinner has stopped: a warning, in every + * mode, for variables the flows read that this environment does not set; then + * the machine-readable result in JSON mode. Human and agent modes already got + * the progress summary. */ export function reportPullResult( ui: UI, @@ -30,6 +37,10 @@ export function reportPullResult( readonly assets: AssetResult; }, ): void { + // Here rather than in the progress steps, so the spinner cannot overwrite it. + if (args.stage.missingEnvVars.length > 0) { + ui.warn(flowsMessages.pull.missingEnvVars(args.stage.missingEnvVars)); + } if (ui.mode !== "json") return; ui.output( { @@ -40,6 +51,8 @@ export function reportPullResult( flowCount: args.stage.flowCount, envVarCount: args.stage.envVarCount, flowsWithTeamStorageRefs: args.stage.flowsWithTeamStorageRefs, + missingEnvVars: args.stage.missingEnvVars, + incompleteFlowCount: args.stage.incompleteFlowCount, assetDownloadedCount: args.assets.downloadedCount, assetReusedCount: args.assets.reusedCount, assetSkippedCount: args.assets.skippedCount, diff --git a/src/domains/flows/pull/safety.test.ts b/src/domains/flows/pull/safety.test.ts index 9c845b7d4..b2807cb90 100644 --- a/src/domains/flows/pull/safety.test.ts +++ b/src/domains/flows/pull/safety.test.ts @@ -8,6 +8,7 @@ import { detectLocalModifications, promptOverwriteIfModified, } from "./safety.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; let workDir = ""; @@ -19,9 +20,7 @@ afterEach(async () => { await rm(workDir, { recursive: true, force: true }); }); -const baseManifest = ( - flows: { path: string; contentHash: string; tags: undefined }[], -): Manifest => ({ +const baseManifest = (flows: Manifest["flows"]): Manifest => ({ envId: "env-abc", envSlug: undefined, envName: undefined, @@ -34,6 +33,11 @@ const baseManifest = ( flows, }); +const flowEntry = ( + path: string, + contentHash: string, +): Manifest["flows"][number] => makeManifestFlow({ path, contentHash }); + // sha256("hello") = 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 const helloHash = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"; @@ -41,26 +45,20 @@ const helloHash = describe("detectLocalModifications", () => { it("returns [] when every file matches its manifest hash", async () => { await writeFile(join(workDir, "a.flow.ts"), "hello", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); expect(await detectLocalModifications(workDir, manifest)).toEqual([]); }); it("flags a file whose hash differs as 'modified'", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); expect(await detectLocalModifications(workDir, manifest)).toEqual([ { path: "a.flow.ts", reason: "modified" }, ]); }); it("flags a missing file as 'missing-from-disk'", async () => { - const manifest = baseManifest([ - { path: "gone.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("gone.flow.ts", helloHash)]); expect(await detectLocalModifications(workDir, manifest)).toEqual([ { path: "gone.flow.ts", reason: "missing-from-disk" }, ]); @@ -73,9 +71,7 @@ describe("detectLocalModifications", () => { }); it("rejects a manifest containing an absolute path entry", async () => { - const manifest = baseManifest([ - { path: "/etc/passwd", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("/etc/passwd", helloHash)]); let caught: unknown; try { await detectLocalModifications(workDir, manifest); @@ -87,9 +83,7 @@ describe("detectLocalModifications", () => { }); it("rejects a manifest entry that escapes the env directory", async () => { - const manifest = baseManifest([ - { path: "../escape.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("../escape.flow.ts", helloHash)]); let caught: unknown; try { await detectLocalModifications(workDir, manifest); @@ -120,9 +114,7 @@ describe("promptOverwriteIfModified", () => { it("proceeds without prompt when there are no modifications", async () => { await writeFile(join(workDir, "a.flow.ts"), "hello", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(false); const log = makeLog(); @@ -139,9 +131,7 @@ describe("promptOverwriteIfModified", () => { }); it("proceeds without prompt when only missing-from-disk entries exist", async () => { - const manifest = baseManifest([ - { path: "gone.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("gone.flow.ts", helloHash)]); const confirm = makeFakeConfirm(false); const log = makeLog(); @@ -159,9 +149,7 @@ describe("promptOverwriteIfModified", () => { it("proceeds without prompt and logs a notice when yes is true and mods exist", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(true); const log = makeLog(); @@ -181,9 +169,7 @@ describe("promptOverwriteIfModified", () => { it("prompts via confirm and proceeds when the user accepts", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(true); const log = makeLog(); @@ -202,9 +188,7 @@ describe("promptOverwriteIfModified", () => { it("aborts when the user declines the prompt", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(false); const log = makeLog(); diff --git a/src/domains/flows/pull/stage.test.ts b/src/domains/flows/pull/stage.test.ts index a354e65b4..91031c96b 100644 --- a/src/domains/flows/pull/stage.test.ts +++ b/src/domains/flows/pull/stage.test.ts @@ -51,6 +51,8 @@ describe("stageBundle", () => { flowCount: 2, envVarCount: 1, flowsWithTeamStorageRefs: [], + missingEnvVars: [], + incompleteFlowCount: 0, }); expect(await readFile(join(destDir, "checkout.flow.ts"), "utf8")).toBe( "// checkout\n", @@ -219,5 +221,12 @@ describe("stageBundle", () => { // The written .env overrides the API's TEAM_STORAGE_DIR with assetsAbs. const env = parseDotenv(await readFile(join(destDir, ".env"), "utf8")); expect(env["TEAM_STORAGE_DIR"]).toBe(assetsDir); + + // Analysis must see the env read introduced by the rewrite. + const manifest = await readManifest(destDir); + if (typeof manifest === "string") throw new Error(manifest); + expect( + manifest.flows.find((flow) => flow.path === "upload.flow.ts")?.envVars, + ).toEqual(["TEAM_STORAGE_DIR"]); }); }); diff --git a/src/domains/flows/pull/stage.ts b/src/domains/flows/pull/stage.ts index b617fdd97..6470b9688 100644 --- a/src/domains/flows/pull/stage.ts +++ b/src/domains/flows/pull/stage.ts @@ -1,13 +1,18 @@ -import { toPosix } from "~/core/repoRelativePath.js"; import { makeDefaultFs, type Fs } from "~/shell/fs.js"; -import { readManifest, writeManifest } from "~/shell/manifest/io.js"; +import { writeManifest } from "~/shell/manifest/io.js"; import { buildManifest, flattenSingleWrapper, sampleQawolfCommittedAt, type FetchedTags, } from "./bundle.js"; +import { + findMissingEnvVars, + type MissingEnvVar, +} from "~/core/envVarAnalysis/missing.js"; import { applyTeamStorageRewrite } from "./applyTeamStorageRewrite.js"; +import { carriedFromPreviousPull } from "./previousPull.js"; +import { collectFlowEnvVars } from "./collectFlowEnvVars.js"; import { writeEnvFile } from "./envVars.js"; import { extractTarGz } from "./extract.js"; import { @@ -35,6 +40,11 @@ type StageBundleResult = { flowCount: number; envVarCount: number; flowsWithTeamStorageRefs: string[]; + // Variables the flows read that this environment does not define, most-read + // first. Aggregated by name rather than by flow: one missing variable in a + // shared helper reaches nearly every flow, so a per-flow list is unreadable. + missingEnvVars: MissingEnvVar[]; + incompleteFlowCount: number; }; export async function stageBundle( @@ -67,10 +77,20 @@ export async function stageBundle( TEAM_STORAGE_DIR: args.assetsAbs, }; await writeEnvFile(tmpDir, effectiveEnvVars, fs); + const { byFlow } = await collectFlowEnvVars(tmpDir); + const missingEnvVars = findMissingEnvVars({ + byFlow, + definedNames: new Set(Object.keys(effectiveEnvVars)), + }); + // A failed tag fetch must not erase cached tags. + const carried = + args.tags === undefined + ? await carriedFromPreviousPull(args.destAbs, fs) + : undefined; const manifest = await buildManifest( { envId: args.envId, - tags: args.tags ?? (await carriedTags(args.destAbs, fs)), + tags: args.tags ?? carried?.tags, envSlug: args.envSlug, envName: args.envName, bundleDir: tmpDir, @@ -79,6 +99,7 @@ export async function stageBundle( envVarsFetchedAt: args.envVarsFetchedAt, wrapperName, qawolfCommittedAt, + envVarsByFlow: byFlow, }, fs, ); @@ -103,34 +124,13 @@ export async function stageBundle( flowCount: manifest.flows.length, envVarCount: Object.keys(effectiveEnvVars).length, flowsWithTeamStorageRefs, + missingEnvVars, + incompleteFlowCount: [...byFlow.values()].filter( + (entry) => entry.mayBeIncomplete, + ).length, }; } catch (err) { await removeTempDir(tmpDir, registry, fs).catch(() => {}); throw err; } } - -/** - * Tags kept from the previous pull of this environment. - * - * A pull rebuilds the manifest from the bundle, so a failed tag fetch would - * otherwise erase tags that were cached successfully earlier. Stale tags are - * reported as stale; losing them silently would break every offline query. - */ -async function carriedTags( - envDir: string, - fs: Fs, -): Promise { - const previous = await readManifest(envDir, fs); - if (typeof previous === "string") return undefined; - if (previous.tagsFetchedAt === undefined) return undefined; - - const byPath = new Map(); - for (const flow of previous.flows) { - // A manifest written by an older CLI on win32 may hold `\` paths; the new - // manifest looks entries up by posix path, so normalize or the carried - // tags never match and vanish silently. - if (flow.tags !== undefined) byPath.set(toPosix(flow.path), [...flow.tags]); - } - return { fetchedAt: new Date(previous.tagsFetchedAt), byPath }; -} diff --git a/src/domains/flows/readCachedTags.test.ts b/src/domains/flows/readCachedTags.test.ts index 6b26ba41e..d3a8a7ad1 100644 --- a/src/domains/flows/readCachedTags.test.ts +++ b/src/domains/flows/readCachedTags.test.ts @@ -6,6 +6,7 @@ import { manifestFilename } from "~/shell/manifest/io.js"; import type { Manifest } from "~/shell/manifest/types.js"; import { readCachedTags } from "./readCachedTags.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; const envDir = "/proj/.qawolf/staging"; const flowA = `${envDir}/src/flows/a.flow.ts`; @@ -43,8 +44,16 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["auth"] }, - { path: "src/flows/b.flow.ts", contentHash: "h2", tags: [] }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["auth"], + }), + makeManifestFlow({ + path: "src/flows/b.flow.ts", + contentHash: "h2", + tags: [], + }), ], }), ); @@ -62,7 +71,7 @@ describe("readCachedTags", () => { manifest({ tagsFetchedAt: undefined, flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: undefined }, + makeManifestFlow({ path: "src/flows/a.flow.ts", contentHash: "h1" }), ], }), ); @@ -78,8 +87,12 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["auth"] }, - { path: "src/flows/b.flow.ts", contentHash: "h2", tags: undefined }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["auth"], + }), + makeManifestFlow({ path: "src/flows/b.flow.ts", contentHash: "h2" }), ], }), ); @@ -94,8 +107,16 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["auth"] }, - { path: "src/flows/b.flow.ts", contentHash: "h2", tags: ["smoke"] }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["auth"], + }), + makeManifestFlow({ + path: "src/flows/b.flow.ts", + contentHash: "h2", + tags: ["smoke"], + }), ], }), ); @@ -120,11 +141,11 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { + makeManifestFlow({ path: "src\\flows\\a.flow.ts", contentHash: "h1", tags: ["auth"], - }, + }), ], }), ); diff --git a/src/domains/flows/resolveTags.test.ts b/src/domains/flows/resolveTags.test.ts index e53aaaaab..4ef217cb7 100644 --- a/src/domains/flows/resolveTags.test.ts +++ b/src/domains/flows/resolveTags.test.ts @@ -13,6 +13,7 @@ import { } from "~/shell/platform/createPlatformClient.testUtils.js"; import { resolveTags } from "./resolveTags.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; afterEach(() => { mock.restore(); @@ -52,7 +53,11 @@ async function fsWithManifest(over: Partial): Promise { qawolfCommittedAt: undefined, tagsFetchedAt: "2026-05-01T12:00:00.000Z", flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["cached-tag"] }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["cached-tag"], + }), ], ...over, }; @@ -146,7 +151,7 @@ describe("resolveTags fallback", () => { envDir, await fsWithManifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: undefined }, + makeManifestFlow({ path: "src/flows/a.flow.ts", contentHash: "h1" }), ], }), ); @@ -164,11 +169,11 @@ describe("resolveTags fallback", () => { envDir, await fsWithManifest({ flows: [ - { + makeManifestFlow({ path: "src\\flows\\a.flow.ts", contentHash: "h1", tags: ["cached-tag"], - }, + }), ], }), ); diff --git a/src/domains/runner/run.manifestStamp.test.ts b/src/domains/runner/run.manifestStamp.test.ts index 1ac2ea43c..cc56ed57e 100644 --- a/src/domains/runner/run.manifestStamp.test.ts +++ b/src/domains/runner/run.manifestStamp.test.ts @@ -9,6 +9,7 @@ import type { Manifest } from "~/shell/manifest/types.js"; import { defaultFlags, makeDeps, passResult } from "./run.fixtures.js"; import { dispatchFlow } from "./dispatchFlow.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; let workDir = ""; let envDir = ""; @@ -35,7 +36,7 @@ const sampleManifest = (): Manifest => ({ tagsFetchedAt: undefined, envVarsFetchedAt: undefined, flows: [ - { path: "login.flow.ts", contentHash: "hash-login", tags: undefined }, + makeManifestFlow({ path: "login.flow.ts", contentHash: "hash-login" }), ], }); diff --git a/src/shell/manifest/io.test.ts b/src/shell/manifest/io.test.ts index aa8701cc0..a5567833e 100644 --- a/src/shell/manifest/io.test.ts +++ b/src/shell/manifest/io.test.ts @@ -12,6 +12,7 @@ import { writeManifest, } from "./io.js"; import type { Manifest } from "./types.js"; +import { makeManifestFlow } from "./manifest.testUtils.js"; const envDir = "/qawolf/manifest-test"; @@ -26,7 +27,7 @@ const sample: Manifest = { tagsFetchedAt: undefined, envVarsFetchedAt: "2026-05-10T12:30:00.000Z", flows: [ - { path: "src/checkout.flow.ts", contentHash: "deadbeef", tags: undefined }, + makeManifestFlow({ path: "src/checkout.flow.ts", contentHash: "deadbeef" }), ], }; @@ -64,6 +65,31 @@ describe("readManifest", () => { expect(result).toBe("malformed"); }); + // Same guarantee for the newer per-flow env var fields: an env pulled by an + // older CLI must still list and run. + it("parses a manifest written before per-flow env vars existed", async () => { + const memFs = makeMemoryFs(); + await memFs.mkdir(envDir, { recursive: true }); + await memFs.writeFile( + join(envDir, manifestFilename), + JSON.stringify({ + envId: "env-abc", + fetchedAt: "2026-05-10T12:00:00.000Z", + cliFlowsVersion: "0.1.0", + tagsFetchedAt: "2026-05-10T12:00:00.000Z", + flows: [ + { path: "src/checkout.flow.ts", contentHash: "deadbeef", tags: [] }, + ], + }), + ); + + const result = await readManifest(envDir, memFs); + + if (typeof result === "string") throw new Error(result); + expect(result.flows[0]?.envVars).toBeUndefined(); + expect(result.flows[0]?.envVarsMayBeIncomplete).toBeUndefined(); + }); + // Manifests written before tags existed must keep parsing: a hard failure // here would break `flows run` against any env pulled by an older CLI. it("parses a manifest written before tags existed", async () => { @@ -79,7 +105,6 @@ describe("readManifest", () => { { path: "src/checkout.flow.ts", contentHash: "deadbeef", - tags: undefined, }, ], }), @@ -99,12 +124,16 @@ describe("readManifest", () => { ...sample, tagsFetchedAt: "2026-05-10T12:45:00.000Z", flows: [ - { + makeManifestFlow({ path: "src/checkout.flow.ts", contentHash: "deadbeef", tags: ["smoke", "auth"], - }, - { path: "src/untagged.flow.ts", contentHash: "cafe", tags: [] }, + }), + makeManifestFlow({ + path: "src/untagged.flow.ts", + contentHash: "cafe", + tags: [], + }), ], }; diff --git a/src/shell/manifest/io.ts b/src/shell/manifest/io.ts index ae8e21944..586d3c775 100644 --- a/src/shell/manifest/io.ts +++ b/src/shell/manifest/io.ts @@ -15,6 +15,10 @@ const flowEntrySchema = z.object({ // Absent on manifests written before tags existed, and on flows the tag // fetch did not return. Both optional so an older manifest still parses. tags: z.array(z.string()).optional(), + // Absent on manifests written before per-flow env vars existed. Optional so + // an older manifest still parses. + envVars: z.array(z.string()).optional(), + envVarsMayBeIncomplete: z.boolean().optional(), }); const manifestSchema = z.object({ @@ -70,6 +74,8 @@ export async function readManifest( path: flow.path, contentHash: flow.contentHash, tags: flow.tags, + envVars: flow.envVars, + envVarsMayBeIncomplete: flow.envVarsMayBeIncomplete, })), }; } diff --git a/src/shell/manifest/lookup.test.ts b/src/shell/manifest/lookup.test.ts index fbbdfe1a1..52186cfd9 100644 --- a/src/shell/manifest/lookup.test.ts +++ b/src/shell/manifest/lookup.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import { manifestFilename, writeManifest } from "./io.js"; import { findFlowStamp } from "./lookup.js"; import type { Manifest } from "./types.js"; +import { makeManifestFlow } from "./manifest.testUtils.js"; let workDir = ""; @@ -44,12 +45,14 @@ const sampleManifest: Manifest = { tagsFetchedAt: undefined, envVarsFetchedAt: undefined, flows: [ - { + makeManifestFlow({ path: join("src", "login.flow.ts"), contentHash: "hash-login", - tags: undefined, - }, - { path: "checkout.flow.ts", contentHash: "hash-checkout", tags: undefined }, + }), + makeManifestFlow({ + path: "checkout.flow.ts", + contentHash: "hash-checkout", + }), ], }; diff --git a/src/shell/manifest/manifest.testUtils.ts b/src/shell/manifest/manifest.testUtils.ts new file mode 100644 index 000000000..d38fb6309 --- /dev/null +++ b/src/shell/manifest/manifest.testUtils.ts @@ -0,0 +1,15 @@ +import type { Manifest } from "./types.js"; + +type ManifestFlow = Manifest["flows"][number]; + +/** A flow entry with every optional field unset, then `over`. */ +export const makeManifestFlow = ( + over: Partial = {}, +): ManifestFlow => ({ + path: "src/flows/a.flow.ts", + contentHash: "hash", + tags: undefined, + envVars: undefined, + envVarsMayBeIncomplete: undefined, + ...over, +}); diff --git a/src/shell/manifest/types.ts b/src/shell/manifest/types.ts index bc1c0c7f9..2ded4f2d9 100644 --- a/src/shell/manifest/types.ts +++ b/src/shell/manifest/types.ts @@ -5,6 +5,12 @@ type ManifestFlowEntry = { // absent both on pre-tags manifests and on flows the tag fetch skipped — // `Manifest.tagsFetchedAt` distinguishes those from a genuinely untagged flow. tags: string[] | undefined; + // Reads found in module initialization and reachable calls at pull time. + // Undefined on older manifests. Recomputed on every pull; carrying this + // forward would resurrect names removed by a source rewrite. + envVars: string[] | undefined; + // Unknown keys or unresolved local calls can hide additional reads. + envVarsMayBeIncomplete: boolean | undefined; }; // Identifies a flow run against a pulled env: derived by walking the From 0311daaf5653a4d3363f4b18a305bcca458aad2a Mon Sep 17 00:00:00 2001 From: Chase J <54216608+chajac@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:51:33 +0100 Subject: [PATCH 2/2] fix(flows): preserve the supplied analysis filesystem --- src/domains/flows/pull/collectFlowEnvVars.ts | 8 +-- src/domains/flows/pull/stage.ts | 2 +- src/domains/flows/pull/stageMemory.test.ts | 58 ++++++++++++++++++ src/shell/flowProgram.test.ts | 63 +++++++++++++++++++- src/shell/flowProgram.ts | 35 +++++++++-- 5 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 src/domains/flows/pull/stageMemory.test.ts diff --git a/src/domains/flows/pull/collectFlowEnvVars.ts b/src/domains/flows/pull/collectFlowEnvVars.ts index cf287a481..7004e711d 100644 --- a/src/domains/flows/pull/collectFlowEnvVars.ts +++ b/src/domains/flows/pull/collectFlowEnvVars.ts @@ -6,7 +6,7 @@ import type { FlowEnvVars } from "~/core/envVarAnalysis/types.js"; import { isSourceFile } from "~/core/flowMeta.js"; import { toPosix } from "~/core/repoRelativePath.js"; import { createFlowProgram } from "~/shell/flowProgram.js"; -import { makeDefaultFs } from "~/shell/fs.js"; +import { makeDefaultFs, type Fs } from "~/shell/fs.js"; import { walkFiles } from "~/shell/walkFiles.js"; type CollectFlowEnvVarsResult = { @@ -14,15 +14,15 @@ type CollectFlowEnvVarsResult = { readonly byFlow: ReadonlyMap; }; -// Uses real disk because the compiler loads sources and tsconfig itself. // Run after applyTeamStorageRewrite so introduced TEAM_STORAGE_DIR reads count. export async function collectFlowEnvVars( bundleDir: string, + fs: Fs = makeDefaultFs(), ): Promise { - const sourcePaths = await walkFiles(bundleDir, isSourceFile, makeDefaultFs()); + const sourcePaths = await walkFiles(bundleDir, isSourceFile, fs); if (sourcePaths.length === 0) return { byFlow: new Map() }; - const program = await createFlowProgram({ bundleDir, sourcePaths }); + const program = await createFlowProgram({ bundleDir, sourcePaths, fs }); const accessors = findEnvAccessors( program.compiler, program.checker, diff --git a/src/domains/flows/pull/stage.ts b/src/domains/flows/pull/stage.ts index 6470b9688..b7807705a 100644 --- a/src/domains/flows/pull/stage.ts +++ b/src/domains/flows/pull/stage.ts @@ -77,7 +77,7 @@ export async function stageBundle( TEAM_STORAGE_DIR: args.assetsAbs, }; await writeEnvFile(tmpDir, effectiveEnvVars, fs); - const { byFlow } = await collectFlowEnvVars(tmpDir); + const { byFlow } = await collectFlowEnvVars(tmpDir, fs); const missingEnvVars = findMissingEnvVars({ byFlow, definedNames: new Set(Object.keys(effectiveEnvVars)), diff --git a/src/domains/flows/pull/stageMemory.test.ts b/src/domains/flows/pull/stageMemory.test.ts new file mode 100644 index 000000000..902e52107 --- /dev/null +++ b/src/domains/flows/pull/stageMemory.test.ts @@ -0,0 +1,58 @@ +import { expect, it } from "bun:test"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { makeMemoryFs } from "~/shell/fs.testUtils.js"; +import { readManifest } from "~/shell/manifest/io.js"; + +import { buildBundle } from "./pull.fixtures.js"; +import { stageBundle } from "./stage.js"; + +it("stages and analyzes a bundle using the supplied filesystem", async () => { + const workDir = await mkdtemp(join(tmpdir(), "stage-memory-")); + try { + const archive = join(workDir, "bundle.tar.gz"); + await buildBundle(archive, { + flows: [ + { + name: "src/a.flow.ts", + data: 'import { read } from "./read.js"; export default () => read();', + }, + { + name: "src/read.ts", + data: "export function read() { return process.env.TOKEN; }", + }, + ], + }); + const fs = makeMemoryFs(); + await fs.mkdir("/work", { recursive: true }); + await fs.writeFile("/work/bundle.tar.gz", await readFile(archive)); + const result = await stageBundle( + { + tmpArchive: "/work/bundle.tar.gz", + destAbs: "/work/env", + assetsAbs: "/work/assets", + envId: "env", + envSlug: undefined, + envName: undefined, + cliFlowsVersion: "1.0.0", + now: new Date(0), + envVars: {}, + envVarsFetchedAt: new Date(0), + tags: undefined, + }, + fs, + ); + expect(result.flowCount).toBe(1); + expect(result.missingEnvVars).toEqual([{ name: "TOKEN", flowCount: 1 }]); + expect(result.incompleteFlowCount).toBe(0); + const manifest = await readManifest("/work/env", fs); + if (typeof manifest === "string") throw new Error(manifest); + expect(manifest.flows[0]?.envVars).toEqual(["TOKEN"]); + expect(manifest.flows[0]?.envVarsMayBeIncomplete).toBe(false); + expect(await fs.pathExists("/work/env/src/a.flow.ts")).toBe(true); + } finally { + await rm(workDir, { recursive: true, force: true }); + } +}); diff --git a/src/shell/flowProgram.test.ts b/src/shell/flowProgram.test.ts index c37bd26c6..4cfee1073 100644 --- a/src/shell/flowProgram.test.ts +++ b/src/shell/flowProgram.test.ts @@ -1,7 +1,11 @@ import { expect, it } from "bun:test"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; + +import { toPosix } from "~/core/repoRelativePath.js"; +import { makeDefaultFs } from "./fs.js"; +import { makeMemoryFs } from "./fs.testUtils.js"; import { createFlowProgram } from "./flowProgram.js"; @@ -23,3 +27,60 @@ it("recognizes native Windows paths after TypeScript normalizes source file name await rm(bundleDir, { recursive: true, force: true }); } }); + +it.each(["memory", "disk"])( + "resolves inherited aliases and relative imports using %s files", + async (storage) => { + const bundleDir = await mkdtemp(join(tmpdir(), "flow-program-")); + const fs = storage === "memory" ? makeMemoryFs() : makeDefaultFs(); + const files = { + "tsconfig.json": JSON.stringify({ extends: "./base.json" }), + "base.json": JSON.stringify({ + compilerOptions: { + module: "ESNext", + moduleResolution: "Bundler", + baseUrl: ".", + paths: { "@helpers/*": ["src/lib/*"] }, + }, + }), + "src/a.flow.ts": + 'import { read } from "@helpers/read"; export default () => read();', + "src/lib/read.ts": 'export { read } from "./value.js";', + "src/lib/value.ts": + "export function read() { return process.env.TOKEN; }", + }; + try { + for (const [name, source] of Object.entries(files)) { + const path = join(bundleDir, name); + await fs.mkdir(dirname(path), { recursive: true }); + await fs.writeFile(path, source); + } + const result = await createFlowProgram({ + bundleDir, + sourcePaths: Object.keys(files) + .filter((name) => name.endsWith(".ts")) + .map((name) => join(bundleDir, name)), + fs, + }); + expect(result.flowFiles).toHaveLength(1); + const declaration = result.flowFiles[0]?.statements.find( + result.compiler.isImportDeclaration, + ); + const bindings = declaration?.importClause?.namedBindings; + if (bindings === undefined || !result.compiler.isNamedImports(bindings)) + throw new Error("missing named import"); + const name = bindings.elements[0]?.name; + const symbol = + name === undefined + ? undefined + : result.checker.getSymbolAtLocation(name); + if (symbol === undefined) throw new Error("missing import symbol"); + const target = result.checker.getAliasedSymbol(symbol); + expect(target.declarations?.[0]?.getSourceFile().fileName).toBe( + toPosix(join(bundleDir, "src/lib/value.ts")), + ); + } finally { + await rm(bundleDir, { recursive: true, force: true }); + } + }, +); diff --git a/src/shell/flowProgram.ts b/src/shell/flowProgram.ts index 2f2ebbb68..92ce15cc5 100644 --- a/src/shell/flowProgram.ts +++ b/src/shell/flowProgram.ts @@ -4,6 +4,7 @@ import type ts from "typescript"; import { isFlowFile } from "~/core/flowMeta.js"; import { loadTypescript, type TypescriptModule } from "./typescript.js"; +import { makeDefaultFs, type Fs } from "./fs.js"; type FlowProgram = { readonly compiler: TypescriptModule; @@ -18,6 +19,7 @@ type FlowProgram = { function compilerOptions( compiler: TypescriptModule, bundleDir: string, + host: ts.ParseConfigHost, ): ts.CompilerOptions { const fallback: ts.CompilerOptions = { allowJs: true, @@ -30,13 +32,13 @@ function compilerOptions( const configPath = join(bundleDir, "tsconfig.json"); const read = compiler.readConfigFile(configPath, (path) => - compiler.sys.readFile(path), + host.readFile(path), ); if (read.error !== undefined || read.config === undefined) return fallback; const parsed = compiler.parseJsonConfigFileContent( read.config, - compiler.sys, + host, bundleDir, ); return { ...parsed.options, allowJs: true, noEmit: true, skipLibCheck: true }; @@ -45,12 +47,33 @@ function compilerOptions( export async function createFlowProgram(args: { bundleDir: string; sourcePaths: readonly string[]; + fs?: Fs; }): Promise { const compiler = await loadTypescript(); - const program = compiler.createProgram( - [...args.sourcePaths], - compilerOptions(compiler, args.bundleDir), - ); + const fs = args.fs ?? makeDefaultFs(); + const readFile = (path: string): string | undefined => { + try { + return fs.readFileSync(path.replaceAll("\\", "/")); + } catch { + return undefined; + } + }; + const fileExists = (path: string): boolean => + fs.existsSync(path.replaceAll("\\", "/")); + const options = compilerOptions(compiler, args.bundleDir, { + useCaseSensitiveFileNames: compiler.sys.useCaseSensitiveFileNames, + readFile, + fileExists, + readDirectory: () => [], + }); + const host = compiler.createCompilerHost(options); + host.readFile = readFile; + host.fileExists = fileExists; + delete host.directoryExists; + delete host.getDirectories; + delete host.realpath; + delete host.readDirectory; + const program = compiler.createProgram([...args.sourcePaths], options, host); const checker = program.getTypeChecker(); // TypeScript normalizes SourceFile.fileName to forward slashes on Windows.