From 928954e341c4bd170290f0d0162522f7197e5477 Mon Sep 17 00:00:00 2001 From: Ari Aye Date: Thu, 10 Sep 2026 12:31:24 -0700 Subject: [PATCH] feat: canonical identity function for agent invocations (closes #1485) New factory/identity.py with normalize_prompt() and invocation_identity(): shared infrastructure for node-level caching, record-and-replay testing, and artifact provenance. Consumed by nothing yet, by design. normalize_prompt() canonicalizes prompt/task text per a documented table (module docstring): project/home/temp roots become placeholders, UUIDs, ISO-8601 datetimes, and 13-digit epoch-millis are templated, whitespace runs collapse. Deliberately conservative: dates, small numbers, unknown paths, and all semantic content are kept. invocation_identity() digests (role, model, normalized prompt, normalized task, attempt, sorted input artifact hashes) as sha256 hex. Attempt number is load-bearing: a RELOOP retry runs with byte-identical inputs, so an identity that excludes the attempt would make an identity-keyed cache replay the just-rejected output forever. Co-Authored-By: Claude Code --- factory/identity.py | 173 ++++++++++++++++++++++++++++ tests/test_identity.py | 249 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 factory/identity.py create mode 100644 tests/test_identity.py diff --git a/factory/identity.py b/factory/identity.py new file mode 100644 index 000000000..16c0618d3 --- /dev/null +++ b/factory/identity.py @@ -0,0 +1,173 @@ +"""Canonical identity for agent invocations. + +Multiple subsystems need to answer the same question — *is this agent +invocation the same as that one?* Node-level caching (the Ledger proposal), +record-and-replay integration testing, and artifact provenance all need a +stable key over an ``AgentRunRequest``. This module is that key, and nothing +else: no storage, no cassettes, no cache machinery (see issue #1485). + +The naive approach — hashing the raw request — fails because prompts embed +volatility: absolute paths that differ per machine and per checkout, run IDs, +timestamps. Correctness lives in the normalization applied before hashing, +and normalization is a blunt instrument in both directions: + +- Too aggressive (strips semantic content): a cache silently replays stale + outputs and hides prompt regressions. +- Too timid (keeps volatility): a cache never hits; replay cassettes must be + re-recorded on every innocuous change. + +The rules below are deliberately conservative — only true volatility is +removed. **Any change to them is a cache-invalidation event for every +consumer** and should be treated with the same care as a prompt-template +change. + +## Normalization table + +What is applied to prompt and task text, in order: + +| # | Rule | Input example | Output | Why | +|---|------|---------------|--------|-----| +| 1 | Project path → ``{project}`` | ``/Users/x/code/app/src/main.py`` | ``{project}/src/main.py`` | Checkout location is machine state, not semantics. Both the literal and resolved forms are replaced. | +| 2 | Home directory → ``{home}`` | ``/Users/x/.factory/agents/prompts/r.md`` | ``{home}/.factory/...`` | Usernames differ across machines. | +| 3 | Temp directory → ``{tmp}`` | ``/var/folders/zy/T/pytest-123/x.md`` | ``{tmp}/pytest-123/x.md`` | OS temp roots differ per machine, per run. | +| 4 | UUIDs → ``{uuid}`` | ``a3f1...`` (dashed or 32-hex) | ``{uuid}`` | Session IDs, trace IDs, and other opaque run-scoped identifiers. | +| 5 | ISO-8601 timestamps → ``{timestamp}`` | ``2026-09-09T14:33:21+00:00`` | ``{timestamp}`` | Wall-clock time is not semantics. Date-only strings are kept (a date in a prompt is usually content). | +| 6 | Epoch-milliseconds → ``{timestamp}`` | ``1757428401123`` | ``{timestamp}`` | 13-digit epoch-millis appear in run-scoped IDs; 13-digit integers are vanishingly rare as prompt content. Plain 10-digit values are kept (they can be legitimate content). | +| 7 | Whitespace runs → single space | ``"a\\n\\n b"`` | ``"a b"`` | Re-templating that only changes indentation must not invalidate. | + +What is deliberately **kept**: everything else, including numbers (except +13-digit epoch-millis), file contents, instructions, constraints, paths that +are not under the project/home/temp roots, and the request's ``role``, +``model``, and ``task``. + +## Identity composition + +``invocation_identity`` digests, in a fixed field order: + +1. ``role`` +2. ``model`` (empty string when unset) +3. normalized ``prompt`` +4. normalized ``task`` +5. ``attempt`` — see below +6. sorted ``input_artifact_hashes`` + +**Attempt number is load-bearing.** A node rejected and retried (e.g. via a +RELOOP verdict) runs with byte-identical inputs. If the identity excluded the +attempt, a cache would replay the just-rejected output, the gate would reject +it again, and the workflow would loop forever without consuming budget. Same +inputs plus a different attempt must be a cache miss. + +The identity is project-agnostic by design: two identical requests against +different projects produce the same digest. Consumers that need per-project +separation should namespace keys themselves (e.g. one cache directory per +project). + +**Consumer contract for volatile roots.** Normalization only templates paths +under roots it knows about: the given ``project_path``, the user's home, and +the system temp root. A consumer whose prompts embed a per-run workspace +(a tmp scratch dir, a worktree) should pass that root as ``project_path`` — +then the run-scoped root is templated while its semantic subpaths survive +(``{project}/inputs/x.json``). +""" + +from __future__ import annotations + +import hashlib +import re +import tempfile +from pathlib import Path + +from factory.models import AgentRunRequest + +# RFC 4122 UUIDs: dashed 8-4-4-4-12, or bare 32 hex chars. +_UUID_RE = re.compile( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" + r"|\b[0-9a-fA-F]{32}\b" +) + +# ISO-8601 datetimes with a time component (date-only is content, not noise). +_ISO_DATETIME_RE = re.compile( + r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?\b" +) + +# Epoch milliseconds: 13 digits, standalone. +_EPOCH_MS_RE = re.compile(r"(? list[tuple[str, str]]: + """Literal → placeholder pairs for known-volatile roots, longest first.""" + pairs: list[tuple[str, str]] = [] + if project_path is not None: + candidates = [project_path] + try: + resolved = project_path.resolve() + except OSError: + resolved = None + if resolved is not None and str(resolved) != str(project_path): + candidates.append(resolved) + for candidate in candidates: + pairs.append((str(candidate), "{project}")) + + home = Path.home() + pairs.append((str(home), "{home}")) + try: + home_resolved = home.resolve() + except OSError: + home_resolved = None + if home_resolved is not None and str(home_resolved) != str(home): + pairs.append((str(home_resolved), "{home}")) + + temp = Path(tempfile.gettempdir()) + pairs.append((str(temp), "{tmp}")) + try: + temp_resolved = temp.resolve() + except OSError: + temp_resolved = None + if temp_resolved is not None and str(temp_resolved) != str(temp): + pairs.append((str(temp_resolved), "{tmp}")) + + # Longest literal first so /home/u/proj is replaced before /home/u. + return sorted(pairs, key=lambda pair: len(pair[0]), reverse=True) + + +def normalize_prompt(prompt: str, *, project_path: Path | None = None) -> str: + """Canonicalize prompt/task text per the normalization table. + + Deterministic, environment-independent in output, and conservative: only + paths under known-volatile roots, UUIDs, timestamps, epoch-millis, and + whitespace runs are touched. + """ + text = prompt + for literal, placeholder in _path_replacements(project_path): + text = text.replace(literal, placeholder) + text = _UUID_RE.sub("{uuid}", text) + text = _ISO_DATETIME_RE.sub("{timestamp}", text) + text = _EPOCH_MS_RE.sub("{timestamp}", text) + return _WHITESPACE_RE.sub(" ", text).strip() + + +def invocation_identity( + request: AgentRunRequest, + *, + attempt: int = 0, + input_artifact_hashes: list[str] | None = None, +) -> str: + """Stable sha256 hex digest over the identity of an agent invocation. + + Covers (role, model, normalized prompt, normalized task, attempt, sorted + input artifact hashes). The digest is project- and path-independent; + consumers needing project separation should namespace it themselves. + """ + project_path = request.project_path + payload_parts = [ + f"role={request.role}", + f"model={request.model or ''}", + f"prompt={normalize_prompt(request.prompt, project_path=project_path)}", + f"task={normalize_prompt(request.task, project_path=project_path)}", + f"attempt={attempt}", + "artifacts=" + + ",".join(sorted(input_artifact_hashes or [])), + ] + return hashlib.sha256("\n".join(payload_parts).encode()).hexdigest() diff --git a/tests/test_identity.py b/tests/test_identity.py new file mode 100644 index 000000000..8ba786a76 --- /dev/null +++ b/tests/test_identity.py @@ -0,0 +1,249 @@ +"""Tests for the canonical agent-invocation identity (factory/identity.py). + +Covers the six properties from issue #1485 plus the RELOOP/attempt +regression: a retry with byte-identical inputs must NOT hit the identity of +the rejected first attempt. +""" + +from __future__ import annotations + +from pathlib import Path + +from factory.identity import invocation_identity, normalize_prompt +from factory.models import AgentRunRequest + +HOME = str(Path.home()) + + +def _request( + prompt: str = "Build the auth feature described in {project}/docs/spec.md", + task: str = "ship the auth feature", + role: str = "builder", + model: str | None = "opus", + project: str = "/proj/checkout", +) -> AgentRunRequest: + prompt = prompt.replace("{project}", project) + return AgentRunRequest( + prompt=prompt, + task=task, + cwd=Path("/tmp/work"), + role=role, + model=model, + project_path=Path(project), + ) + + +class TestNormalizePrompt: + def test_project_path_becomes_placeholder(self): + out = normalize_prompt( + f"Read {HOME}/code/app/src/main.py", project_path=Path(f"{HOME}/code/app") + ) + assert out == "Read {project}/src/main.py" + + def test_project_path_resolved_form_replaced(self): + # macOS: /var/folders is a symlink to /private/var/folders + raw = normalize_prompt("/tmp/foo/../bar/x.md", project_path=Path("/tmp/foo/../bar")) + assert raw == "{project}/x.md" + + def test_home_paths_become_placeholder_without_project(self): + out = normalize_prompt(f"See {HOME}/.factory/agents/prompts/r.md") + assert out == "See {home}/.factory/agents/prompts/r.md" + + def test_unknown_absolute_paths_kept(self): + out = normalize_prompt("Edit /etc/hosts carefully") + assert out == "Edit /etc/hosts carefully" + + def test_dashed_uuid_replaced(self): + out = normalize_prompt("session 550e8400-e29b-41d4-a716-446655440000 done") + assert out == "session {uuid} done" + + def test_bare_hex32_replaced(self): + out = normalize_prompt("trace 550e8400e29b41d4a716446655440000 done") + assert out == "trace {uuid} done" + + def test_iso_datetime_replaced(self): + for stamp in ( + "2026-09-09T14:33:21+00:00", + "2026-09-09T14:33:21Z", + "2026-09-09 14:33:21", + "2026-09-09T14:33", + "2026-09-09T14:33:21.123456", + ): + assert normalize_prompt(f"at {stamp} then") == "at {timestamp} then", stamp + + def test_date_only_kept(self): + out = normalize_prompt("release scheduled for 2026-09-09") + assert out == "release scheduled for 2026-09-09" + + def test_epoch_millis_replaced(self): + out = normalize_prompt("run ts 1757428401123 ok") + assert out == "run ts {timestamp} ok" + + def test_small_numbers_kept(self): + out = normalize_prompt("retry 3 times over 10 iterations at depth 42") + assert out == "retry 3 times over 10 iterations at depth 42" + + def test_whitespace_collapsed(self): + out = normalize_prompt("line one\n\n line two\t\ttabbed") + assert out == "line one line two tabbed" + + def test_semantic_content_preserved(self): + out = normalize_prompt("Use bcrypt, not sha1, for password hashing") + assert out == "Use bcrypt, not sha1, for password hashing" + + +class TestInvocationIdentity: + def test_path_independence(self): + a = _request(project="/Users/alice/code/app") + b = _request(project="/tmp/pytest-42/bob/app") + assert invocation_identity(a) == invocation_identity(b) + + def test_path_independence_across_home_roots(self, monkeypatch): + # The same logical prompt written under bob's home on one machine and + # ours on another must normalize identically. Each digest is computed + # while its machine's home is active. + monkeypatch.setattr(Path, "home", classmethod(lambda cls: Path("/home/bob"))) + b = AgentRunRequest( + prompt="Read /home/bob/docs/a.md and act", + task="t", + cwd=Path("/tmp/y"), + role="builder", + ) + digest_b = invocation_identity(b) + + monkeypatch.setattr(Path, "home", classmethod(lambda cls: Path(HOME))) + a = AgentRunRequest( + prompt=f"Read {HOME}/docs/a.md and act", + task="t", + cwd=Path("/tmp/x"), + role="builder", + ) + digest_a = invocation_identity(a) + + assert digest_a == digest_b + + def test_cwd_not_part_of_identity(self): + a = _request() + b = a.model_copy(update={"cwd": Path("/somewhere/else")}) + assert invocation_identity(a) == invocation_identity(b) + + def test_session_fields_not_part_of_identity(self): + a = _request() + b = a.model_copy( + update={"session_id": "s2", "session_name": "tmux-b", "resume_session_id": "s0"} + ) + assert invocation_identity(a) == invocation_identity(b) + + def test_semantic_prompt_change_changes_digest(self): + a = _request() + b = a.model_copy(update={"prompt": a.prompt + " Also add integration tests"}) + assert invocation_identity(a) != invocation_identity(b) + + def test_semantic_task_change_changes_digest(self): + a = _request() + b = a.model_copy(update={"task": "ship it, but keep the old auth module"}) + assert invocation_identity(a) != invocation_identity(b) + + def test_attempt_sensitivity(self): + a = _request() + assert invocation_identity(a) != invocation_identity(a, attempt=1) + assert invocation_identity(a, attempt=1) != invocation_identity(a, attempt=2) + + def test_model_sensitivity(self): + a = _request() + b = a.model_copy(update={"model": "sonnet"}) + assert invocation_identity(a) != invocation_identity(b) + + def test_role_sensitivity(self): + a = _request() + b = a.model_copy(update={"role": "code_reviewer"}) + assert invocation_identity(a) != invocation_identity(b) + + def test_unset_model_vs_empty(self): + a = _request(model=None) + b = _request(model="") + assert invocation_identity(a) == invocation_identity(b) + + def test_artifact_hashes_order_insensitive(self): + a = _request() + left = invocation_identity(a, input_artifact_hashes=["h1", "h2", "h3"]) + right = invocation_identity(a, input_artifact_hashes=["h3", "h1", "h2"]) + assert left == right + + def test_artifact_hashes_content_sensitive(self): + a = _request() + left = invocation_identity(a, input_artifact_hashes=["h1"]) + right = invocation_identity(a, input_artifact_hashes=["h2"]) + assert left != right + + def test_artifact_hashes_vs_none(self): + a = _request() + assert invocation_identity(a) != invocation_identity(a, input_artifact_hashes=["h1"]) + + def test_determinism(self): + a = _request() + assert invocation_identity(a) == invocation_identity(a) + assert len(invocation_identity(a)) == 64 + + def test_digest_is_hex(self): + import re + + assert re.fullmatch(r"[0-9a-f]{64}", invocation_identity(_request())) + + +class TestReloopRegression: + """The correctness trap from issue #1485: a RELOOP retry runs with + byte-identical inputs; if the identity excluded the attempt number, an + identity-keyed cache would replay the just-rejected output forever.""" + + def test_rejected_retry_must_be_cache_miss(self): + first = _request() + rejected_output = "the output the gate rejected" + + cache: dict[str, str] = {} + cache[invocation_identity(first)] = rejected_output + + # Retry after RELOOP: same request object, attempt incremented. + retry_key = invocation_identity(first, attempt=1) + assert retry_key not in cache, "retry must not hit the rejected attempt's entry" + + cache[retry_key] = "fixed output" + again = invocation_identity(first, attempt=2) + assert again not in cache + + def test_prompt_volatility_does_not_break_hits(self): + """The flip side: innocuous volatility (workspace paths, timestamps, + session IDs) must not turn a genuine repeat into a miss. The consumer + contract is to pass the per-run workspace as project_path so the + run-scoped root is templated while its semantic subpaths survive.""" + def make(workspace: str, ts: str, session: str) -> AgentRunRequest: + return AgentRunRequest( + prompt=( + "Build the auth feature described in /proj/checkout/docs/spec.md. " + f"Workspace: {workspace}/inputs. Started {ts}. Session {session}." + ), + task="ship the auth feature", + cwd=Path(workspace), + role="builder", + model="opus", + project_path=Path(workspace), + ) + + run_a = make( + "/tmp/pytest-111/work", + "2026-09-09T09:00:00Z", + "550e8400-e29b-41d4-a716-446655440000", + ) + run_b = make( + "/tmp/pytest-222/work", + "2026-09-09T11:30:45Z", + "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + ) + assert invocation_identity(run_a) == invocation_identity(run_b) + + +class TestProjectAgnosticism: + def test_identical_requests_same_digest_across_projects(self): + a = _request(project="/proj/alpha") + b = _request(project="/proj/beta") + assert invocation_identity(a) == invocation_identity(b)