feat(flows): record environment variable reads during pull

.changeset/flow-env-var-names.md

@@ -0,0 +1,9 @@
+---
+"@qawolf/cli": minor
+---
+
+`qawolf flows pull` now records environment variable names found by static analysis of each flow. It follows module initialization and reachable calls, including literal names passed through helpers such as `requireEnv("NAME")`. Dormant function bodies and unused methods do not add reads merely because their module was imported.
+
+The pull result warns about variables the flows read that the environment does not set, grouping them by name and reporting how many flows read each one. Reads may be optional. Runtime and operating-system settings are excluded from these warnings.
+
+Dynamic keys and unresolved local calls can hide additional reads. Affected flows are counted in the pull result and marked with `envVarsMayBeIncomplete` in the manifest. Recorded names are a static approximation, not a complete list of required variables, and local edits do not refresh them. Older manifests leave the analysis fields absent until the next pull.

src/core/envVarAnalysis/missing.test.ts

@@ -0,0 +1,67 @@
+import { describe, expect, it } from "bun:test";
+
+import { findMissingEnvVars } from "./missing.js";
+
+describe("findMissingEnvVars", () => {
+  const byFlow = (entries: Record<string, string[]>) =>
+    new Map(
+      Object.entries(entries).map(([path, names]) => [
+        path,
+        { names, mayBeIncomplete: false },
+      ]),
+    );
+
+  it("reports a variable no flow's environment defines", () => {
+    expect(
+      findMissingEnvVars({
+        byFlow: byFlow({ "a.flow.ts": ["PRESENT", "ABSENT"] }),
+        definedNames: new Set(["PRESENT"]),
+      }),
+    ).toEqual([{ name: "ABSENT", flowCount: 1 }]);
+  });
+
+  it("counts how many flows read each missing variable", () => {
+    expect(
+      findMissingEnvVars({
+        byFlow: byFlow({
+          "a.flow.ts": ["SHARED"],
+          "b.flow.ts": ["SHARED"],
+          "c.flow.ts": ["RARE"],
+        }),
+        definedNames: new Set(),
+      }),
+    ).toEqual([
+      { name: "SHARED", flowCount: 2 },
+      { name: "RARE", flowCount: 1 },
+    ]);
+  });
+
+  it("breaks ties on count by name", () => {
+    expect(
+      findMissingEnvVars({
+        byFlow: byFlow({ "a.flow.ts": ["ZED", "ALPHA"] }),
+        definedNames: new Set(),
+      }).map((m) => m.name),
+    ).toEqual(["ALPHA", "ZED"]);
+  });
+
+  it("ignores runner- and OS-provided variables", () => {
+    expect(
+      findMissingEnvVars({
+        byFlow: byFlow({
+          "a.flow.ts": ["QAWOLF_EXAMPLE_ID", "TEAM_STORAGE_DIR", "HOME"],
+        }),
+        definedNames: new Set(),
+      }),
+    ).toEqual([]);
+  });
+
+  it("reports nothing when the environment defines everything", () => {
+    expect(
+      findMissingEnvVars({
+        byFlow: byFlow({ "a.flow.ts": ["ALPHA", "BETA"] }),
+        definedNames: new Set(["ALPHA", "BETA"]),
+      }),
+    ).toEqual([]);
+  });
+});

src/core/envVarAnalysis/missing.ts

@@ -0,0 +1,31 @@
+import { isRuntimeProvidedEnvVar } from "~/core/runtimeEnvVars.js";
+
+import type { FlowEnvVars } from "./types.js";
+
+/** One variable flows read that the environment does not define. */
+export type MissingEnvVar = {
+  name: string;
+  flowCount: number;
+};
+
+/** Aggregates missing variables across flows, excluding runner and OS variables. */
+export function findMissingEnvVars(args: {
+  byFlow: ReadonlyMap<string, FlowEnvVars>;
+  definedNames: ReadonlySet<string>;
+}): MissingEnvVar[] {
+  const flowCountByName = new Map<string, number>();
+
+  for (const { names } of args.byFlow.values()) {
+    for (const name of names) {
+      if (args.definedNames.has(name)) continue;
+      if (isRuntimeProvidedEnvVar(name)) continue;
+      flowCountByName.set(name, (flowCountByName.get(name) ?? 0) + 1);
+    }
+  }
+
+  // Most-used first, so the line that matters most is the one that survives
+  // any truncation downstream.
+  return [...flowCountByName]
+    .map(([name, flowCount]) => ({ name, flowCount }))
+    .sort((a, b) => b.flowCount - a.flowCount || a.name.localeCompare(b.name));
+}

src/core/messages/flows.test.ts

@@ -44,3 +44,71 @@ describe("flowsMessages.pull.summary", () => {
     );
   });
 });
+
+describe("flowsMessages.pull.summary incomplete flows", () => {
+  const base = {
+    envDir: "/tmp/env",
+    flowCount: 1,
+    envVarCount: 0,
+    flowsWithTeamStorageRefs: [],
+    assetDownloadedCount: 0,
+    assetReusedCount: 0,
+    assetSkippedCount: 0,
+  };
+
+  it("says nothing when every key is knowable", () => {
+    expect(
+      flowsMessages.pull.summary({ ...base, incompleteFlowCount: 0 }, "/tmp/a"),
+    ).toBe("Pulled 1 flow into /tmp/env");
+  });
+
+  // Said once as a count, not marked on each of the flows it covers.
+  it("counts the flows whose list is a floor", () => {
+    expect(
+      flowsMessages.pull.summary({ ...base, incompleteFlowCount: 3 }, "/tmp/a"),
+    ).toBe(
+      [
+        "Pulled 1 flow into /tmp/env",
+        "3 flows may read more variables than listed; static analysis could not resolve every read.",
+      ].join("\n"),
+    );
+  });
+});
+
+describe("flowsMessages.pull.missingEnvVars", () => {
+  it("names one variable and how many flows read it", () => {
+    expect(
+      flowsMessages.pull.missingEnvVars([{ name: "LOGIN_PW", flowCount: 1 }]),
+    ).toBe(
+      [
+        "1 environment variable is read by flows but not set in this environment (some reads may be optional):",
+        "  - LOGIN_PW (read by 1 flow)",
+      ].join("\n"),
+    );
+  });
+
+  it("pluralizes across several variables and flows", () => {
+    expect(
+      flowsMessages.pull.missingEnvVars([
+        { name: "SHARED", flowCount: 12 },
+        { name: "RARE", flowCount: 1 },
+      ]),
+    ).toBe(
+      [
+        "2 environment variables are read by flows but not set in this environment (some reads may be optional):",
+        "  - SHARED (read by 12 flows)",
+        "  - RARE (read by 1 flow)",
+      ].join("\n"),
+    );
+  });
+
+  it("truncates a long list", () => {
+    const missing = Array.from({ length: 9 }, (_, i) => ({
+      name: `VAR_${String(i)}`,
+      flowCount: 1,
+    }));
+    expect(flowsMessages.pull.missingEnvVars(missing)).toContain(
+      "  ... and 4 more",
+    );
+  });
+});

src/core/messages/flows.ts

@@ -1,14 +1,6 @@
 import { pluralize } from "~/core/pluralize.js";
 
-type PullSummaryInput = {
-  readonly envDir: string;
-  readonly flowCount: number;
-  readonly envVarCount: number;
-  readonly flowsWithTeamStorageRefs: readonly string[];
-  readonly assetDownloadedCount?: number | undefined;
-  readonly assetReusedCount?: number | undefined;
-  readonly assetSkippedCount?: number | undefined;
-};
+import { flowsPullMessages } from "./flowsPull.js";
 
 export const flowsMessages = {
   title: "Flows",
@@ -61,72 +53,7 @@ export const flowsMessages = {
     requiresEnv:
       "An environment is required. Pass --env <env> or set QAWOLF_ENVIRONMENT.",
   },
-  pull: {
-    requiresEnv:
-      "An environment is required. Pass --env <env> or set QAWOLF_ENVIRONMENT.",
-    downloadingBundle: "Downloading flows bundle",
-    fetchingEnvVars: "Fetching environment variables",
-    fetchingTags: "Fetching flow tags",
-    downloadComplete: "Downloaded flows bundle and environment variables",
-    needsYesError: "Re-run with --yes to overwrite locally-modified files",
-    aborted: "Aborted; no changes.",
-    extractingBundle: "Extracting bundle",
-    downloadingTeamStorageAssets: "Downloading team-storage assets",
-    downloadingTeamStorageAssetsProgress: (current: number, total: number) =>
-      `Downloading team-storage assets (${String(current)}/${String(total)})`,
-    teamStorageRequiresTeam:
-      "Team storage needs a team. Pull an environment to name its team, choose a workspace with 'qawolf auth switch', or use a team API key.",
-    summary: (result: PullSummaryInput, assetsAbs: string) => {
-      const flows = pluralize(result.flowCount, "flow");
-      const envVars =
-        result.envVarCount === 0
-          ? ""
-          : ` and ${pluralize(result.envVarCount, "environment variable")}`;
-      const lines = [`Pulled ${flows}${envVars} into ${result.envDir}`];
-      if (result.flowsWithTeamStorageRefs.length > 0) {
-        const refs = pluralize(result.flowsWithTeamStorageRefs.length, "flow");
-        lines.push(`Team-storage assets referenced by ${refs}:`);
-        for (const path of result.flowsWithTeamStorageRefs) {
-          lines.push(`  - ${path}`);
-        }
-      }
-      const downloaded = result.assetDownloadedCount ?? 0;
-      const reused = result.assetReusedCount ?? 0;
-      const skipped = result.assetSkippedCount ?? 0;
-      if (downloaded > 0 || reused > 0 || skipped > 0) {
-        let assetSummary = `Downloaded ${pluralize(
-          downloaded,
-          "team-storage asset",
-        )}`;
-        if (reused > 0) {
-          assetSummary += ` and reused ${pluralize(
-            reused,
-            "team-storage asset",
-          )}`;
-        }
-        assetSummary += ` into ${assetsAbs}`;
-        if (skipped > 0) {
-          assetSummary += ` (${pluralize(
-            skipped,
-            "unsafe or unsupported asset",
-          )} skipped)`;
-        }
-        lines.push(assetSummary);
-      }
-      return lines.join("\n");
-    },
-    symlinkRejected: (path: string) => `symlink entry rejected: ${path}`,
-    unknownEntrySize: (path: string) =>
-      `entry with unknown size rejected: ${path}`,
-    entryTooLarge: (path: string, size: number, maxBytes: number) =>
-      `entry exceeds max size (${path}): ${String(size)} > ${String(maxBytes)}`,
-    localModsWouldOverwrite: (
-      count: number,
-      envDir: string,
-      fileList: string,
-    ) =>
-      `${count} locally-modified file(s) under ${envDir} would be overwritten:\n${fileList}`,
-  },
+  pull: flowsPullMessages,
   ensureDeps: {
     multiPackagePattern: (count: number, listed: string) =>
       `Pattern matches flows from ${count} packages — narrow it to a single package:\n${listed}\n\nHint: pass a pattern scoped to one package, e.g \`qawolf flows run '.qawolf/<env>/**'\`.`,

src/core/messages/flowsPull.ts

@@ -0,0 +1,102 @@
+import type { MissingEnvVar } from "~/core/envVarAnalysis/missing.js";
+import { pluralize } from "~/core/pluralize.js";
+
+type PullSummaryInput = {
+  readonly envDir: string;
+  readonly flowCount: number;
+  readonly envVarCount: number;
+  readonly flowsWithTeamStorageRefs: readonly string[];
+  readonly incompleteFlowCount?: number | undefined;
+  readonly assetDownloadedCount?: number | undefined;
+  readonly assetReusedCount?: number | undefined;
+  readonly assetSkippedCount?: number | undefined;
+};
+
+// Enough to act on without turning a spinner's stop message into a wall of
+// text; the full set is always in the JSON output.
+const maxListedNames = 5;
+
+export const flowsPullMessages = {
+  requiresEnv:
+    "An environment is required. Pass --env <env> or set QAWOLF_ENVIRONMENT.",
+  downloadingBundle: "Downloading flows bundle",
+  fetchingEnvVars: "Fetching environment variables",
+  fetchingTags: "Fetching flow tags",
+  downloadComplete: "Downloaded flows bundle and environment variables",
+  needsYesError: "Re-run with --yes to overwrite locally-modified files",
+  aborted: "Aborted; no changes.",
+  extractingBundle: "Extracting bundle",
+  downloadingTeamStorageAssets: "Downloading team-storage assets",
+  downloadingTeamStorageAssetsProgress: (current: number, total: number) =>
+    `Downloading team-storage assets (${String(current)}/${String(total)})`,
+  teamStorageRequiresTeam:
+    "Team storage needs a team. Pull an environment to name its team, choose a workspace with 'qawolf auth switch', or use a team API key.",
+  summary: (result: PullSummaryInput, assetsAbs: string) => {
+    const flows = pluralize(result.flowCount, "flow");
+    const envVars =
+      result.envVarCount === 0
+        ? ""
+        : ` and ${pluralize(result.envVarCount, "environment variable")}`;
+    const lines = [`Pulled ${flows}${envVars} into ${result.envDir}`];
+    if (result.flowsWithTeamStorageRefs.length > 0) {
+      const refs = pluralize(result.flowsWithTeamStorageRefs.length, "flow");
+      lines.push(`Team-storage assets referenced by ${refs}:`);
+      for (const path of result.flowsWithTeamStorageRefs) {
+        lines.push(`  - ${path}`);
+      }
+    }
+    // Shared dynamic helpers can affect every flow; report their impact once.
+    const incomplete = result.incompleteFlowCount ?? 0;
+    if (incomplete > 0) {
+      lines.push(
+        `${pluralize(incomplete, "flow")} may read more variables than listed; static analysis could not resolve every read.`,
+      );
+    }
+    const downloaded = result.assetDownloadedCount ?? 0;
+    const reused = result.assetReusedCount ?? 0;
+    const skipped = result.assetSkippedCount ?? 0;
+    if (downloaded > 0 || reused > 0 || skipped > 0) {
+      let assetSummary = `Downloaded ${pluralize(
+        downloaded,
+        "team-storage asset",
+      )}`;
+      if (reused > 0) {
+        assetSummary += ` and reused ${pluralize(
+          reused,
+          "team-storage asset",
+        )}`;
+      }
+      assetSummary += ` into ${assetsAbs}`;
+      if (skipped > 0) {
+        assetSummary += ` (${pluralize(
+          skipped,
+          "unsafe or unsupported asset",
+        )} skipped)`;
+      }
+      lines.push(assetSummary);
+    }
+    return lines.join("\n");
+  },
+  missingEnvVars: (missing: readonly MissingEnvVar[]) => {
+    // A read does not prove a variable is required by the flow.
+    const lines = [
+      `${pluralize(missing.length, "environment variable")} ${
+        missing.length === 1 ? "is" : "are"
+      } read by flows but not set in this environment (some reads may be optional):`,
+    ];
+    for (const { name, flowCount } of missing.slice(0, maxListedNames)) {
+      lines.push(`  - ${name} (read by ${pluralize(flowCount, "flow")})`);
+    }
+    if (missing.length > maxListedNames) {
+      lines.push(`  ... and ${String(missing.length - maxListedNames)} more`);
+    }
+    return lines.join("\n");
+  },
+  symlinkRejected: (path: string) => `symlink entry rejected: ${path}`,
+  unknownEntrySize: (path: string) =>
+    `entry with unknown size rejected: ${path}`,
+  entryTooLarge: (path: string, size: number, maxBytes: number) =>
+    `entry exceeds max size (${path}): ${String(size)} > ${String(maxBytes)}`,
+  localModsWouldOverwrite: (count: number, envDir: string, fileList: string) =>
+    `${count} locally-modified file(s) under ${envDir} would be overwritten:\n${fileList}`,
+} as const;