fix(cli): say what a 404 could not find, and which runner

.changeset/not-found-names-what-is-missing.md

@@ -0,0 +1,9 @@
+---
+"@qawolf/cli": minor
+---
+
+A 404 from the QA Wolf API now names what the command could not find, instead of telling everyone to check `--env`. A runner-targeting command says the runner is not running, names it, says whether `--runner`, `QAWOLF_RUNNER_ID` or this directory's stored default chose that id, and gives the launch command. `qawolf run get` says there is no such run on this team, and that ids printed by `qawolf runner run` are the runner's own — read those with `qawolf runner events run-status --run <id>`. Only a request that really is scoped to an environment still points at `--env`.
+
+A failed `@qawolf/cli/runner-sdk` call now carries that second line as `errorDetail`, which the SDK used to build and throw away.
+
+These failures now exit `8` rather than `4`. Exit `4` means retry; a runner that was terminated or idled out never comes back, so a caller that kept retrying burned its budget on an id that could not work. Bound your retries on `4` as before, and stop on `8`.

docs/exit-codes.md

@@ -2,16 +2,17 @@
 
 CI consumers depend on consistent exit codes. The CLI commits to the following codes; do not introduce new ones without updating this document and the central helper in [`src/shell/exit.ts`](../src/shell/exit.ts).
 
-| Code | Name          | Meaning                                                                                                                                                                                                                                                                                            |
-| ---- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `0`  | `success`     | Command completed successfully.                                                                                                                                                                                                                                                                    |
-| `1`  | `testFailure` | A flow failed (non-zero result from running tests); a `qawolf runner run --follow` whose run did not pass; a runner action or snippet that was attempted and did not succeed.                                                                                                                      |
-| `2`  | `invalidArgs` | Commander parse error, unknown subcommand, bad flag value, a `flows run` selection with no runnable flow (pass `--allow-no-match` to exit `0` instead), no runner available, a flow needing a different runner image, or a runner asked for something it can never do (no screen to see or drive). |
-| `3`  | `auth`        | Missing or invalid `QAWOLF_API_KEY`.                                                                                                                                                                                                                                                               |
-| `4`  | `network`     | Apex unreachable, GCS download failure, registry unreachable, or a runner that could not serve the request now (unreachable, or its screen not yet up).                                                                                                                                            |
-| `5`  | `config`      | `qawolf.config.ts` invalid, file collision during `init`, or a run file that could not be read.                                                                                                                                                                                                    |
-| `6`  | `timeout`     | A `--follow` reached its `--timeout`: `runner run` before its run settled (the run may still be going), or `runner events`.                                                                                                                                                                        |
-| `7`  | `payment`     | The QA Wolf API refused the request with HTTP 402: billing prevented it — the organization is over its monthly spend limit or has no valid payment method.                                                                                                                                         |
+| Code | Name          | Meaning                                                                                                                                                                                                                                                                                                                |
+| ---- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `0`  | `success`     | Command completed successfully.                                                                                                                                                                                                                                                                                        |
+| `1`  | `testFailure` | A flow failed (non-zero result from running tests); a `qawolf runner run --follow` whose run did not pass; a runner action or snippet that was attempted and did not succeed.                                                                                                                                          |
+| `2`  | `invalidArgs` | Commander parse error, unknown subcommand, bad flag value, a `flows run` selection with no runnable flow (pass `--allow-no-match` to exit `0` instead), no runner available, a flow needing a different runner image, or a runner asked for something it can never do (no screen to see or drive).                     |
+| `3`  | `auth`        | Missing or invalid `QAWOLF_API_KEY`.                                                                                                                                                                                                                                                                                   |
+| `4`  | `network`     | Apex unreachable, GCS download failure, registry unreachable, or a runner that could not serve the request now (unreachable, or its screen not yet up).                                                                                                                                                                |
+| `5`  | `config`      | `qawolf.config.ts` invalid, file collision during `init`, or a run file that could not be read.                                                                                                                                                                                                                        |
+| `6`  | `timeout`     | A `--follow` reached its `--timeout`: `runner run` before its run settled (the run may still be going), or `runner events`.                                                                                                                                                                                            |
+| `7`  | `payment`     | The QA Wolf API refused the request with HTTP 402: billing prevented it — the organization is over its monthly spend limit or has no valid payment method.                                                                                                                                                             |
+| `8`  | `notFound`    | The QA Wolf API answered HTTP 404: the runner, run, or other thing the command named was not there. A runner will not come back by waiting — launch it, or name one that is running. Other routes can answer `8` for something that does arrive shortly, such as a run still being created, and say so in the message. |
 
 ## Using the helper
 

skills/qawolf-cli/references/run-results.md

@@ -25,6 +25,14 @@ run
 `runId` in the response is canonical and can differ from the id you asked for.
 Use the returned value for follow-up calls.
 
+`run get` resolves platform runs only. A run id printed by `qawolf runner run`
+belongs to that runner, so `run get` answers exit `8` and no such run on this
+team. Read one of those with `qawolf runner events run-status --run <id>`.
+
+A run that has been requested but not yet created answers exit `8` too, and
+says it is still being created. That one clears on its own, so read the message
+rather than the code before deciding whether to poll.
+
 Poll `status` until it reaches `passed`, `failed` or `canceled`. The other
 values mean the run is still going.
 

skills/qawolf-cli/references/runner.md

@@ -30,8 +30,8 @@ then launched without `--id` ends up with a pod it is not addressing. Pass
 And a runner id that is set is treated as found, whether or not anything is
 running under it. So exporting `QAWOLF_RUNNER_ID=agent-1` turns off the
 auto-launch described next: instead of starting `agent-1`, commands try to reach
-it and fail with exit code `4`, which reads as "retry" and never succeeds.
-Launch that id once yourself and the rest follows.
+it and fail with exit code `8`, naming the id and saying the variable is what
+chose it. Launch that id once yourself and the rest follows.
 
 And launching an id that differs from `QAWOLF_RUNNER_ID` prints a warning on
 stderr naming both ids: the variable still outranks the directory default, so
@@ -119,11 +119,14 @@ and says so on stderr. Everything else on this page waits for a run.
 
 Retry on the exit code, not on the message text:
 
-- `4` is usually transient. The screen is up but cannot serve this instant:
-  restarting after a display-size change, or busy with another request. Retry in
-  a second or two — but bound the retries, because `4` also covers a runner that
-  was reaped after inactivity, which no amount of retrying brings back. If `4`
-  persists past a few tries, relaunch the id.
+- `4` is transient. The screen is up but cannot serve this instant: restarting
+  after a display-size change, or busy with another request. Retry in a second
+  or two, and bound the retries.
+- `8` means there is no such runner. It was never launched, or it was
+  terminated, or it idled out. Retrying never brings one back, so stop and
+  launch the id or name one that is running. The message says which runner was
+  meant and whether `--runner`, `QAWOLF_RUNNER_ID` or this directory's stored
+  default chose it — read that line before you pick an id to launch.
 - `2` will not clear on its own. Either nothing has run on this runner yet, so
   run a flow, or the runner has no browser at all, so launch with
   `--name playwright` instead. The message says which.
@@ -224,7 +227,8 @@ what the page shows.
 One failure covers three causes, because a runner cannot tell them apart: no
 live page, no element matching the selector, no variable under that name. All
 three exit `2` and none clears by waiting, so read the message, which carries
-whatever the runner said. An unreachable runner exits `4` and is worth retrying.
+whatever the runner said. An unreachable runner exits `4` and is worth retrying;
+a runner that is not running at all exits `8` and is not.
 
 Use `inspect` before reaching for `exec`. Reading a value through a snippet
 means printing it and then fishing it back out of the `console` stream, which is

src/commands/qawolfCliRunResults.template.md

@@ -25,6 +25,14 @@ run
 `runId` in the response is canonical and can differ from the id you asked for.
 Use the returned value for follow-up calls.
 
+`run get` resolves platform runs only. A run id printed by `qawolf runner run`
+belongs to that runner, so `run get` answers exit `8` and no such run on this
+team. Read one of those with `qawolf runner events run-status --run <id>`.
+
+A run that has been requested but not yet created answers exit `8` too, and
+says it is still being created. That one clears on its own, so read the message
+rather than the code before deciding whether to poll.
+
 Poll `status` until it reaches `passed`, `failed` or `canceled`. The other
 values mean the run is still going.
 

src/core/interactiveRunner/runnerIdSource.ts

@@ -0,0 +1,8 @@
+import type { IdSource } from "~/core/resolveId.js";
+
+/**
+ * Where the runner a command drives was named: the three levels `resolveIdFrom`
+ * picks between, the runner the CLI started because none of them named one, and
+ * the id an SDK caller passed in directly.
+ */
+export type RunnerIdSource = IdSource | "launched" | "given";

src/core/messages/authErrors.ts

@@ -19,8 +19,36 @@ export const authErrorMessages = {
       `QA Wolf API refused the${noun ? ` ${noun}` : ""} request (HTTP 402): billing prevented it.`,
     rejected403: (noun: string | undefined) =>
       `QA Wolf API rejected the${noun ? ` ${noun}` : ""} request (HTTP 403). Check that your API key has access to this environment.`,
-    notFound404: (noun: string | undefined) =>
+    /**
+     * A 404 is answered by what the request named, because almost none of them
+     * are about an environment. Only `notFound404Environment` keeps the wording
+     * that blames one, and only routes that carry an environment reach it.
+     */
+    notFound404Environment: (noun: string | undefined) =>
       `QA Wolf API could not find ${noun ? `${noun} for that environment` : "that environment"} (HTTP 404). Check the --env value.`,
+    notFound404Runner: (runnerId: string | undefined) =>
+      runnerId === undefined
+        ? "That runner is not running (HTTP 404)."
+        : `Runner ${runnerId} is not running (HTTP 404).`,
+    /** Why a runner is gone, when the platform did not say. */
+    runnerIsGone:
+      "It was never launched, or it has since been terminated or idled out.",
+    /** True whatever the platform said, so it is offered either way. */
+    launchTheRunner: (runnerId: string | undefined) =>
+      runnerId === undefined
+        ? "Launch one with qawolf runner launch --id <id>, or name a running one with --runner."
+        : `Launch it with qawolf runner launch --id ${runnerId}, or send this to a different runner with --runner.`,
+    notFound404Run: (runId: string | undefined) =>
+      `QA Wolf has no run ${runId ?? "by that id"} on this team (HTTP 404).`,
+    /**
+     * Why a run id that exists can still be unknown to the platform. A guess,
+     * so it is offered only when the platform did not say — a run being created
+     * also answers 404, and that one clears by waiting.
+     */
+    runIdMayBeRunnerLocal: (runId: string | undefined) =>
+      `A run id printed by qawolf runner run belongs to that runner rather than to the platform, so this command cannot resolve it. Read that run with qawolf runner events run-status --run ${runId ?? "<id>"}.`,
+    notFound404: (noun: string | undefined) =>
+      `QA Wolf API could not find ${noun ?? "what the request named"} (HTTP 404).`,
     failedWithStatus: (status: number, noun: string | undefined) =>
       `QA Wolf API${noun ? ` ${noun}` : ""} request failed (HTTP ${status}).`,
     networkUnreachable: (baseUrl: string, noun: string | undefined) =>

src/core/messages/interactiveRunner/lifecycle.ts

@@ -1,3 +1,5 @@
+import type { RunnerIdSource } from "~/core/interactiveRunner/runnerIdSource.js";
+
 // Ends the line with no period: a terminal that linkifies takes the period as
 // part of the address.
 //
@@ -42,4 +44,16 @@ export const lifecycleMessages = {
     `Runner ${id} had nothing to stop. The run had already finished, or none had been submitted.`,
   runnerUnreachable:
     "The runner could not be reached. It may still be starting, or it may have terminated after inactivity. Retry, or launch it again.",
+  // Named on every failure to reach a runner, because a transcript that only
+  // says which runner was missed leaves a reader unable to tell whether the
+  // wrong id was typed, exported, or left behind in .qawolf by an earlier
+  // launch — three different things to go and change.
+  runnerIdCameFrom: (id: string, source: RunnerIdSource) =>
+    ({
+      environment: `The id ${id} came from QAWOLF_RUNNER_ID.`,
+      flag: `The id ${id} came from --runner.`,
+      given: `The id ${id} is the one this call named.`,
+      launched: `Runner ${id} was launched for this command.`,
+      stored: `The id ${id} came from the runner this directory last launched, recorded in .qawolf.`,
+    })[source],
 } as const;

src/core/publicApi/notFoundSubject.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from "bun:test";
+
+import { notFoundSubject } from "./notFoundSubject.js";
+
+describe("notFoundSubject", () => {
+  it("reads the runner id off a route relayed to one runner", () => {
+    expect(notFoundSubject("runner.takeScreenshot", { id: "agent-1" })).toEqual(
+      { kind: "runner", runnerId: "agent-1" },
+    );
+  });
+
+  // Launching creates a runner and listing names none, so neither can 404 over
+  // a runner that has gone.
+  it.each(["runner.launch", "runner.list"])(
+    "does not read %s as a missing runner",
+    (contractName) => {
+      expect(notFoundSubject(contractName, { id: "agent-1" }).kind).not.toBe(
+        "runner",
+      );
+    },
+  );
+
+  it("reads the run id off a route that resolves one run", () => {
+    expect(notFoundSubject("run.get", { runId: "abc" })).toEqual({
+      kind: "run",
+      runId: "abc",
+    });
+  });
+
+  it("reads a request that names an environment as environment-scoped", () => {
+    expect(notFoundSubject("run.create", { environmentId: "env-1" })).toEqual({
+      kind: "environment",
+    });
+    expect(
+      notFoundSubject("environment.get", { environmentId: "env-1" }),
+    ).toEqual({ kind: "environment" });
+  });
+
+  // The bug this replaces: a trigger or an issue that does not exist was
+  // reported as an environment problem.
+  it("leaves a request that names no environment unattributed", () => {
+    expect(notFoundSubject("trigger.get", { triggerId: "t-1" })).toEqual({
+      kind: "other",
+    });
+  });
+
+  it("survives an input that is not an object", () => {
+    expect(notFoundSubject("runner.inspect", undefined)).toEqual({
+      kind: "runner",
+      runnerId: undefined,
+    });
+  });
+});

src/core/publicApi/notFoundSubject.ts

@@ -0,0 +1,54 @@
+/**
+ * What a request asked the platform to find, so a 404 can name that thing
+ * rather than blame the environment for every miss.
+ */
+export type NotFoundSubject =
+  | { kind: "runner"; runnerId: string | undefined }
+  | { kind: "run"; runId: string | undefined }
+  | { kind: "environment" }
+  | { kind: "other" };
+
+// `launch` starts a runner and `list` names none, so neither can 404 over a
+// runner that has gone. Every other runner route is relayed to one live pod.
+const runnerRoutesThatNameNoRunner: ReadonlySet<string> = new Set([
+  "runner.launch",
+  "runner.list",
+]);
+
+// The run routes that resolve one run by id. `run.create` and `run.find` take
+// an environment instead, and fall through to the environment rule below.
+const runRoutesThatResolveOneRun: ReadonlySet<string> = new Set([
+  "run.diagnose",
+  "run.get",
+  "run.reattempt",
+  "run.stop",
+]);
+
+function field(input: unknown, name: string): string | undefined {
+  if (typeof input !== "object" || input === null) return undefined;
+  const value = (input as Record<string, unknown>)[name];
+  return typeof value === "string" ? value : undefined;
+}
+
+/**
+ * Reads the subject off the contract name and the input that was sent, rather
+ * than off a hand-kept list of every contract: a route that resolves one runner
+ * carries its id, and a route scoped to an environment carries that.
+ */
+export function notFoundSubject(
+  contractName: string,
+  input: unknown,
+): NotFoundSubject {
+  if (
+    contractName.startsWith("runner.") &&
+    !runnerRoutesThatNameNoRunner.has(contractName)
+  ) {
+    return { kind: "runner", runnerId: field(input, "id") };
+  }
+  if (runRoutesThatResolveOneRun.has(contractName)) {
+    return { kind: "run", runId: field(input, "runId") };
+  }
+  if (field(input, "environmentId") !== undefined)
+    return { kind: "environment" };
+  return { kind: "other" };
+}