diff --git a/src/core/envVarAnalysis/envReads.test.ts b/src/core/envVarAnalysis/envReads.test.ts new file mode 100644 index 000000000..b7d822851 --- /dev/null +++ b/src/core/envVarAnalysis/envReads.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from "bun:test"; +import ts from "typescript"; + +import { readEnvVarsFrom } from "./envReads.js"; + +function reads(source: string): { names: string[]; dynamic: boolean } { + const file = ts.createSourceFile( + "a.flow.ts", + source, + ts.ScriptTarget.Latest, + true, + ); + const names = new Set(); + let dynamic = false; + const visit = (node: ts.Node): void => { + const result = readEnvVarsFrom(ts, node); + for (const name of result.names) names.add(name); + dynamic ||= result.dynamic; + ts.forEachChild(node, visit); + }; + visit(file); + return { names: [...names].sort(), dynamic }; +} + +describe("environment read syntax", () => { + it("reads property, bracket, template, and destructured names", () => { + expect( + reads( + 'process.env.DIRECT; process.env["BRACKET"]; process.env[`TEMPLATE`]; const { BOUND: value } = process.env;', + ), + ).toEqual({ + names: ["BOUND", "BRACKET", "DIRECT", "TEMPLATE"], + dynamic: false, + }); + }); + + it("excludes write-only assignments and deletes while retaining compound reads", () => { + expect( + reads(`export default () => { + process.env.WRITTEN = "generated"; + process.env["BRACKET_WRITTEN"] = "generated"; + delete process.env.DELETED; + delete process.env["BRACKET_DELETED"]; + process.env.UPDATED += "suffix"; + process.env.FALLBACK ??= "fallback"; + process.env.COPIED = process.env.READ; + ({ token: process.env.DESTRUCTURED } = { token: "generated" }); + [process.env.ARRAY_WRITTEN] = ["generated"]; + };`), + ).toEqual({ + names: ["FALLBACK", "READ", "UPDATED"], + dynamic: false, + }); + }); + + it("excludes loop write targets while retaining iterable reads", () => { + expect( + reads(`for (process.env.TARGET of [process.env.VALUE]) {} + for (process.env["KEY"] in { [process.env.SOURCE]: true }) {} + for ({ key: process.env.OBJECT } of rows) {} + for ([process.env.ARRAY] of rows) {}`), + ).toEqual({ names: ["SOURCE", "VALUE"], dynamic: false }); + }); + + it("accepts static names containing template markers", () => { + expect( + reads('process.env["TOKEN${SUFFIX}"]; process.env[`LITERAL\\${KEY}`];'), + ).toEqual({ names: ["LITERAL${KEY}", "TOKEN${SUFFIX}"], dynamic: false }); + }); + + it.each([ + "const env = process.env; env.TOKEN;", + "use(process.env);", + "({ ...process.env });", + "Object.keys(process.env);", + ])("flags an unhandled environment object read: %s", (source) => { + expect(reads(source)).toEqual({ names: [], dynamic: true }); + }); + + it("excludes writes to the environment object", () => { + expect( + reads(`process.env = {}; delete process.env; + for (process.env of objects) {}`), + ).toEqual({ names: [], dynamic: false }); + }); + + it("reads quoted and computed literal destructuring keys", () => { + expect( + reads(`const { "TOKEN": token, ["USER"]: user } = process.env;`), + ).toEqual({ + names: ["TOKEN", "USER"], + dynamic: false, + }); + }); + + it.each([ + `const { [key]: value } = process.env;`, + `process.env[key];`, + "process.env[`${key}_EMAIL`];", + `const { ...rest } = process.env;`, + ])("flags a dynamic read: %s", (source) => { + expect(reads(source)).toEqual({ names: [], dynamic: true }); + }); + + it("retains known names alongside uncertainty", () => { + expect(reads(`const { KNOWN, ...rest } = process.env;`)).toEqual({ + names: ["KNOWN"], + dynamic: true, + }); + }); + + it("does not read a name out of a comment", () => { + expect( + reads( + `// process.env.COMMENT\n/** process.env.JSDOC */\nprocess.env.REAL;`, + ), + ).toEqual({ + names: ["REAL"], + dynamic: false, + }); + }); +}); diff --git a/src/core/envVarAnalysis/envReads.ts b/src/core/envVarAnalysis/envReads.ts new file mode 100644 index 000000000..6e13165dc --- /dev/null +++ b/src/core/envVarAnalysis/envReads.ts @@ -0,0 +1,104 @@ +import type ts from "typescript"; + +import type { EnvReads } from "./types.js"; + +function isProcessEnv(compiler: typeof ts, node: ts.Node): boolean { + return ( + compiler.isPropertyAccessExpression(node) && + node.name.text === "env" && + compiler.isIdentifier(node.expression) && + node.expression.text === "process" + ); +} + +function isReadAccess(compiler: typeof ts, node: ts.Node): boolean { + let target = node; + while ( + compiler.isParenthesizedExpression(target.parent) || + compiler.isAsExpression(target.parent) || + compiler.isNonNullExpression(target.parent) || + (compiler.isPropertyAssignment(target.parent) && + target.parent.initializer === target) || + compiler.isObjectLiteralExpression(target.parent) || + compiler.isArrayLiteralExpression(target.parent) || + compiler.isSpreadAssignment(target.parent) || + compiler.isSpreadElement(target.parent) + ) { + target = target.parent; + } + const parent = target.parent; + return ( + !compiler.isDeleteExpression(parent) && + !( + compiler.isBinaryExpression(parent) && + parent.left === target && + parent.operatorToken.kind === compiler.SyntaxKind.EqualsToken + ) && + !( + (compiler.isForInStatement(parent) || + compiler.isForOfStatement(parent)) && + parent.initializer === target + ) + ); +} + +/** Reads on one visited syntax node; execution scope is controlled by the caller. */ +export function readEnvVarsFrom(compiler: typeof ts, node: ts.Node): EnvReads { + const names = new Set(); + let dynamic = false; + if ( + compiler.isPropertyAccessExpression(node) && + isProcessEnv(compiler, node.expression) && + isReadAccess(compiler, node) + ) { + names.add(node.name.text); + } + if ( + compiler.isElementAccessExpression(node) && + isProcessEnv(compiler, node.expression) && + isReadAccess(compiler, node) + ) { + const argument = node.argumentExpression; + if (compiler.isStringLiteralLike(argument)) { + names.add(argument.text); + } else { + dynamic = true; + } + } + if ( + compiler.isVariableDeclaration(node) && + node.initializer !== undefined && + isProcessEnv(compiler, node.initializer) && + compiler.isObjectBindingPattern(node.name) + ) { + for (const element of node.name.elements) { + if (element.dotDotDotToken !== undefined) { + dynamic = true; + continue; + } + const key = element.propertyName ?? element.name; + if (compiler.isIdentifier(key) || compiler.isStringLiteralLike(key)) { + names.add(key.text); + } else if ( + compiler.isComputedPropertyName(key) && + compiler.isStringLiteralLike(key.expression) + ) { + names.add(key.expression.text); + } else { + dynamic = true; + } + } + } + if (isProcessEnv(compiler, node) && isReadAccess(compiler, node)) { + const parent = node.parent; + const handled = + ((compiler.isPropertyAccessExpression(parent) || + compiler.isElementAccessExpression(parent)) && + parent.expression === node) || + (compiler.isVariableDeclaration(parent) && + parent.initializer === node && + compiler.isObjectBindingPattern(parent.name)); + if (!handled) dynamic = true; + } + return { names, dynamic }; +} diff --git a/src/core/envVarAnalysis/types.ts b/src/core/envVarAnalysis/types.ts new file mode 100644 index 000000000..e3b4c32bc --- /dev/null +++ b/src/core/envVarAnalysis/types.ts @@ -0,0 +1,5 @@ +/** Known reads and uncertainty for an executable unit. */ +export type EnvReads = { + names: Set; + dynamic: boolean; +}; diff --git a/src/core/flowMeta.ts b/src/core/flowMeta.ts index 218adc0a8..d47f2bea7 100644 --- a/src/core/flowMeta.ts +++ b/src/core/flowMeta.ts @@ -75,3 +75,12 @@ export type PeekFlowMetaFn = (filePath: string) => Promise; export function extractFlowMeta(source: string): FlowCallMeta { return parseFlowCall(source); } + +const flowExtensions = [".flow.ts", ".flow.js"]; +const sourceExtensions = [".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"]; + +export const isFlowFile = (name: string): boolean => + flowExtensions.some((extension) => name.endsWith(extension)); + +export const isSourceFile = (name: string): boolean => + sourceExtensions.some((extension) => name.endsWith(extension)); diff --git a/src/domains/flows/pull/applyTeamStorageRewrite.ts b/src/domains/flows/pull/applyTeamStorageRewrite.ts index c210ef510..3c68321d8 100644 --- a/src/domains/flows/pull/applyTeamStorageRewrite.ts +++ b/src/domains/flows/pull/applyTeamStorageRewrite.ts @@ -1,39 +1,17 @@ -import { join, relative } from "node:path"; +import { relative } from "node:path"; +import { isFlowFile, isSourceFile } from "~/core/flowMeta.js"; import { makeDefaultFs } from "~/shell/fs.js"; import type { Fs } from "~/shell/fs.js"; +import { walkFiles } from "~/shell/walkFiles.js"; import { rewriteTeamStorage } from "./rewriteTeamStorage.js"; -const sourceExtensions = [".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"]; -const flowExtensions = [".flow.ts", ".flow.js"]; - -function isSourceFile(name: string): boolean { - return sourceExtensions.some((ext) => name.endsWith(ext)); -} - -function isFlowFile(name: string): boolean { - return flowExtensions.some((ext) => name.endsWith(ext)); -} - -async function walk(dir: string, out: string[], fs: Fs): Promise { - const entries = await fs.readdirWithTypes(dir); - for (const e of entries) { - const abs = join(dir, e.name); - if (e.isDirectory()) { - await walk(abs, out, fs); - } else if (e.isFile() && isSourceFile(e.name)) { - out.push(abs); - } - } -} - export async function applyTeamStorageRewrite( rootDir: string, fs: Fs = makeDefaultFs(), ): Promise<{ flowsWithTeamStorageRefs: string[] }> { - const files: string[] = []; - await walk(rootDir, files, fs); + const files = await walkFiles(rootDir, isSourceFile, fs); const results = await Promise.all( files.map(async (file): Promise => { const source = await fs.readFile(file); diff --git a/src/domains/flows/pull/bundle.ts b/src/domains/flows/pull/bundle.ts index d894c775f..67e91b7c1 100644 --- a/src/domains/flows/pull/bundle.ts +++ b/src/domains/flows/pull/bundle.ts @@ -1,5 +1,8 @@ import { join, relative } from "node:path"; +import { isFlowFile } from "~/core/flowMeta.js"; +import { walkFiles } from "~/shell/walkFiles.js"; + import { toPosix } from "~/core/repoRelativePath.js"; import { hashFile } from "~/shell/manifest/io.js"; @@ -48,8 +51,6 @@ function extractQawolfCommitSha( return /-([0-9a-f]{40})$/i.exec(wrapperName)?.[1]; } -const flowExtensions = [".flow.ts", ".flow.js"]; - /** * Tags fetched for an env at pull time, keyed by repo-relative flow path. * Undefined when the fetch did not happen or failed. @@ -74,7 +75,7 @@ export async function buildManifest( }, fs: Fs = makeDefaultFs(), ): Promise { - const flowPaths = await walkForFlows(args.bundleDir, fs); + const flowPaths = await flowPathsIn(args.bundleDir, fs); const flows = await Promise.all( flowPaths.map(async (rel) => ({ // Stored posix so a manifest written on one platform resolves on @@ -102,30 +103,11 @@ export async function buildManifest( }; } -async function walkForFlows(root: string, fs: Fs): Promise { - const out: string[] = []; - await walk(root, root, out, fs); - return out.sort(); -} - -async function walk( - current: string, - root: string, - out: string[], - fs: Fs, -): Promise { - const entries = await fs.readdirWithTypes(current); - for (const e of entries) { - const abs = join(current, e.name); - if (e.isDirectory()) { - await walk(abs, root, out, fs); - } else if ( - e.isFile() && - flowExtensions.some((ext) => e.name.endsWith(ext)) - ) { - out.push(relative(root, abs)); - } - } +// Flow files under `root`, relative to it and sorted, so the manifest lists +// them in the same order on every pull. +async function flowPathsIn(root: string, fs: Fs): Promise { + const found = await walkFiles(root, isFlowFile, fs); + return found.map((path) => toPosix(relative(root, path))).sort(); } // Samples the mtime of any flow file in the bundle. GitHub-archive bundles @@ -136,7 +118,7 @@ export async function sampleQawolfCommittedAt( bundleDir: string, fs: Fs = makeDefaultFs(), ): Promise { - const flowPaths = await walkForFlows(bundleDir, fs); + const flowPaths = await flowPathsIn(bundleDir, fs); const sample = flowPaths[0]; if (!sample) return undefined; return (await fs.stat(join(bundleDir, sample))).mtime.toISOString(); diff --git a/src/domains/flows/pull/bundleOrder.test.ts b/src/domains/flows/pull/bundleOrder.test.ts new file mode 100644 index 000000000..5db90dc0a --- /dev/null +++ b/src/domains/flows/pull/bundleOrder.test.ts @@ -0,0 +1,37 @@ +import { expect, it } from "bun:test"; + +import { makeMemoryFs } from "~/shell/fs.testUtils.js"; + +import { buildManifest } from "./bundle.js"; + +it("sorts manifest paths after normalizing separators", async () => { + const fs = makeMemoryFs(); + await fs.mkdir("/bundle/a", { recursive: true }); + await fs.writeFile("/bundle/a/z.flow.ts", "export default () => 1;"); + await fs.writeFile("/bundle/a[.flow.ts", "export default () => 2;"); + const manifest = await buildManifest( + { + bundleDir: "/bundle", + envId: "env", + cliFlowsVersion: "1.0.0", + now: new Date(0), + envVarsFetchedAt: undefined, + wrapperName: undefined, + qawolfCommittedAt: undefined, + tags: undefined, + }, + { + ...fs, + readdirWithTypes: async () => + ["a\\z.flow.ts", "a[.flow.ts"].map((name) => ({ + name, + isFile: () => true, + isDirectory: () => false, + })), + }, + ); + expect(manifest.flows.map(({ path }) => path)).toEqual([ + "a/z.flow.ts", + "a[.flow.ts", + ]); +}); diff --git a/src/shell/flowProgram.test.ts b/src/shell/flowProgram.test.ts new file mode 100644 index 000000000..c37bd26c6 --- /dev/null +++ b/src/shell/flowProgram.test.ts @@ -0,0 +1,25 @@ +import { expect, it } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createFlowProgram } from "./flowProgram.js"; + +it("recognizes native Windows paths after TypeScript normalizes source file names", async () => { + const bundleDir = await mkdtemp(join(tmpdir(), "qawolf-flow-program-")); + const sourcePath = join(bundleDir, "a.flow.ts"); + try { + await writeFile(sourcePath, "export default () => process.env.TOKEN;"); + const windowsPath = sourcePath.replaceAll("/", "\\"); + const result = await createFlowProgram({ + bundleDir, + sourcePaths: [windowsPath], + }); + expect(result.flowFiles).toHaveLength(1); + expect(result.sourceFiles).toHaveLength(1); + expect(result.isLocalFile(sourcePath.replaceAll("\\", "/"))).toBe(true); + expect(result.isLocalFile(windowsPath)).toBe(true); + } finally { + await rm(bundleDir, { recursive: true, force: true }); + } +}); diff --git a/src/shell/flowProgram.ts b/src/shell/flowProgram.ts new file mode 100644 index 000000000..2f2ebbb68 --- /dev/null +++ b/src/shell/flowProgram.ts @@ -0,0 +1,74 @@ +import { join } from "node:path"; +import type ts from "typescript"; + +import { isFlowFile } from "~/core/flowMeta.js"; + +import { loadTypescript, type TypescriptModule } from "./typescript.js"; + +type FlowProgram = { + readonly compiler: TypescriptModule; + readonly program: ts.Program; + readonly checker: ts.TypeChecker; + readonly sourceFiles: readonly ts.SourceFile[]; + readonly flowFiles: readonly ts.SourceFile[]; + readonly isLocalFile: (fileName: string) => boolean; +}; + +// Pulled bundles need their path aliases, but do not need installed dependencies. +function compilerOptions( + compiler: TypescriptModule, + bundleDir: string, +): ts.CompilerOptions { + const fallback: ts.CompilerOptions = { + allowJs: true, + noEmit: true, + skipLibCheck: true, + target: compiler.ScriptTarget.ESNext, + module: compiler.ModuleKind.ESNext, + moduleResolution: compiler.ModuleResolutionKind.Bundler, + }; + + const configPath = join(bundleDir, "tsconfig.json"); + const read = compiler.readConfigFile(configPath, (path) => + compiler.sys.readFile(path), + ); + if (read.error !== undefined || read.config === undefined) return fallback; + + const parsed = compiler.parseJsonConfigFileContent( + read.config, + compiler.sys, + bundleDir, + ); + return { ...parsed.options, allowJs: true, noEmit: true, skipLibCheck: true }; +} + +export async function createFlowProgram(args: { + bundleDir: string; + sourcePaths: readonly string[]; +}): Promise { + const compiler = await loadTypescript(); + const program = compiler.createProgram( + [...args.sourcePaths], + compilerOptions(compiler, args.bundleDir), + ); + const checker = program.getTypeChecker(); + + // TypeScript normalizes SourceFile.fileName to forward slashes on Windows. + const normalize = (fileName: string): string => + fileName.replaceAll("\\", "/"); + const local = new Set(args.sourcePaths.map(normalize)); + const isLocalFile = (fileName: string): boolean => + local.has(normalize(fileName)); + const sourceFiles = program + .getSourceFiles() + .filter((file) => isLocalFile(file.fileName)); + + return { + compiler, + program, + checker, + sourceFiles, + flowFiles: sourceFiles.filter((file) => isFlowFile(file.fileName)), + isLocalFile, + }; +} diff --git a/src/shell/typescript.ts b/src/shell/typescript.ts new file mode 100644 index 000000000..4c692c1ed --- /dev/null +++ b/src/shell/typescript.ts @@ -0,0 +1,20 @@ +import type ts from "typescript"; + +export type TypescriptModule = typeof ts; + +// External to the bundle and lazy so unrelated commands never parse the compiler. +// CommonJS interop exposes it on `default` in some runtimes, directly in others. +export async function loadTypescript(): Promise { + const loaded: unknown = await import("typescript"); + const withDefault = loaded as { default?: unknown }; + const candidate = + withDefault.default !== undefined ? withDefault.default : loaded; + const compiler = candidate as TypescriptModule; + if (typeof compiler.createProgram !== "function") { + throw new Error( + "The installed typescript package did not expose createProgram. " + + "Reinstall dependencies, or report this at https://github.com/qawolf/cli/issues", + ); + } + return compiler; +} diff --git a/src/shell/walkFiles.ts b/src/shell/walkFiles.ts new file mode 100644 index 000000000..c8c0aa4d9 --- /dev/null +++ b/src/shell/walkFiles.ts @@ -0,0 +1,18 @@ +import { join } from "node:path"; + +import type { Fs } from "./fs.js"; + +export async function walkFiles( + dir: string, + include: (name: string) => boolean, + fs: Fs, +): Promise { + const found: string[] = []; + for (const entry of await fs.readdirWithTypes(dir)) { + const path = join(dir, entry.name); + if (entry.isDirectory()) + found.push(...(await walkFiles(path, include, fs))); + else if (entry.isFile() && include(entry.name)) found.push(path); + } + return found; +}