From 039a4b5815da84f999d773dc3e8cd5383ad6f64d Mon Sep 17 00:00:00 2001 From: colehurwitz Date: Wed, 9 Sep 2026 12:00:06 -0400 Subject: [PATCH 1/8] feat: add resolve_task() shared resolver and --task CLI flag for outer loop (#1478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of the task→workflow→outer-loop creation pipeline. - Add resolve_task() to factory/task.py: 3-step resolution (TOML file → Python file → module:Class string) with project-relative path support - Add --task flag to both calibrate and evaluate subcommands in factory/cli/outer_loop.py, using the shared resolver (one helper, not two — lesson from #1449) - Keep --task-module as a functional escape hatch - Resolved tasks attach via set_task() for highest precedence in SwarmConfig.get_task() - 12 new tests covering all resolution paths and equivalence - Updated docs/outer-loop.md with --task documentation Co-Authored-By: Claude Opus 4.6 --- docs/outer-loop.md | 50 +++++++++--- factory/cli/outer_loop.py | 24 ++++++ factory/task.py | 71 +++++++++++++++++ tests/test_resolve_task.py | 159 +++++++++++++++++++++++++++++++++++++ 4 files changed, 291 insertions(+), 13 deletions(-) create mode 100644 tests/test_resolve_task.py diff --git a/docs/outer-loop.md b/docs/outer-loop.md index 7c0ea39f1..8fe722d37 100644 --- a/docs/outer-loop.md +++ b/docs/outer-loop.md @@ -337,17 +337,43 @@ factory outer-loop calibrate \ ## Task Discovery -Projects with custom `Task` subclasses (e.g. `chess-evolve`, `harbor`) can pass their task class to the outer loop via the `--task-module` flag. This uses the same `module:ClassName` import-string pattern as `EvaluatorRef`. +Tasks can be passed to the outer loop via `--task` (recommended) or `--task-module` (escape hatch). -### Format +### `--task` (recommended) +The `--task` flag accepts three formats, resolved in order by `resolve_task()`: + +1. **TOML file** — path ending in `.toml` or resolving to an existing `.toml` file. Parsed via `TaskDefinition.from_toml()`. +2. **Python file** — path ending in `.py` or resolving to an existing `.py` file. The module is loaded and introspected for a single `Task` subclass. +3. **Module:Class string** — `module.path:ClassName` format, resolved via `TaskRef.resolve()`. Same as `--task-module`. + +```bash +# TOML task +factory outer-loop calibrate /path/to/factory \ + --task .factory/tasks/chess-evolve.toml \ + --seed-workflow chess_pkg:build_pipeline + +# Python task file +factory outer-loop calibrate /path/to/factory \ + --task examples/chess_evolve_task.py \ + --seed-workflow chess_pkg:build_pipeline + +# Module:Class (same as --task-module) +factory outer-loop calibrate /path/to/factory \ + --task chess_evolve.task:ChessEvolveTask \ + --seed-workflow chess_pkg:build_pipeline + +# Override on evaluate +factory outer-loop evaluate /path/to/factory \ + --generation 0 \ + --task .factory/tasks/chess-evolve.toml ``` -module.path:ClassName -``` -The module is imported via `importlib.import_module()` and the class is resolved via `getattr()`. The class must be a subclass of `factory.task.Task`. +Relative paths are resolved against the project directory. + +### `--task-module` (escape hatch) -### Precondition +The `--task-module` flag accepts only `module.path:ClassName` import strings. It is retained for backward compatibility. The module is imported via `importlib.import_module()` and the class is resolved via `getattr()`. The class must be a subclass of `factory.task.Task`. The task's package must be importable — install it first: @@ -355,16 +381,14 @@ The task's package must be importable — install it first: pip install -e /path/to/my-project ``` -### Usage - ```bash -# Calibrate with a custom task +# Calibrate with --task-module factory outer-loop calibrate /path/to/factory \ --benchmark chess-evolve \ --task-module chess_evolve.task:ChessEvolveTask \ --project-dir /path/to/chess-evolve -# Evaluate with a custom task (overrides persisted config) +# Evaluate with --task-module (overrides persisted config) factory outer-loop evaluate /path/to/factory \ --generation 0 \ --task-module chess_evolve.task:ChessEvolveTask @@ -374,8 +398,8 @@ factory outer-loop evaluate /path/to/factory \ `SwarmConfig.get_task()` resolves tasks with 3-tier precedence: -1. **`set_task()`** — explicit runtime attachment (highest, used by tests and in-process callers) -2. **`task_module`** — `module:ClassName` string from CLI flag or persisted config +1. **`set_task()`** — explicit runtime attachment (highest, used by `--task` CLI flag, tests, and in-process callers) +2. **`task_module`** — `module:ClassName` string from `--task-module` CLI flag or persisted config 3. **`Task.from_legacy()`** — constructed from flat fields (`test_command`, `test_format`, etc.) -The `task_module` field is serialized with `SwarmConfig`, so it persists across `save_config`/`load_config` — no need to re-specify it on every `evaluate` invocation after `calibrate`. +The `task_module` field is serialized with `SwarmConfig`, so it persists across `save_config`/`load_config` — no need to re-specify it on every `evaluate` invocation after `calibrate`. Tasks resolved via `--task` are attached at runtime via `set_task()` and take highest precedence. diff --git a/factory/cli/outer_loop.py b/factory/cli/outer_loop.py index 5882ce63d..51d24ebdb 100644 --- a/factory/cli/outer_loop.py +++ b/factory/cli/outer_loop.py @@ -196,6 +196,7 @@ def _cmd_calibrate(args: argparse.Namespace) -> int: resolved_prep_command = bench_config.prep_command if bench_config else "" task_module = getattr(args, "task_module", "") + cli_task_ref = getattr(args, "task", "") seed_workflow_module = getattr(args, "seed_workflow", "") config = SwarmConfig( benchmark=benchmark, @@ -216,6 +217,12 @@ def _cmd_calibrate(args: argparse.Namespace) -> int: if task_module: _log.info("task_module_resolved", ref=task_module) + if cli_task_ref: + from factory.task import resolve_task + resolved = resolve_task(cli_task_ref, project_path) + config.set_task(resolved) + _log.info("task_resolved_from_cli", ref=cli_task_ref, name=resolved.name) + root = init_filesystem(project_path, config) benchmark = config.benchmark @@ -461,6 +468,13 @@ def _cmd_evaluate(args: argparse.Namespace) -> int: config = config.model_copy(update={"task_module": cli_task_module}) _log.info("task_module_override", ref=cli_task_module) + cli_task_ref = getattr(args, "task", "") + if cli_task_ref: + from factory.task import resolve_task + resolved = resolve_task(cli_task_ref, project_path) + config.set_task(resolved) + _log.info("task_resolved_from_cli", ref=cli_task_ref, name=resolved.name) + eval_project_dir = getattr(args, "project_dir", None) if eval_project_dir is not None: eval_project_dir = str(Path(eval_project_dir).resolve()) @@ -928,6 +942,11 @@ def add_outer_loop_parser(subparsers: argparse._SubParsersAction) -> None: # ty default="", help="Task class ref as 'module.path:ClassName' (e.g. chess_evolve.task:ChessEvolveTask)", ) + cal.add_argument( + "--task", + default="", + help="Task reference: .toml file, .py file, or module:ClassName (e.g. .factory/tasks/chess.toml)", + ) cal.add_argument( "--seed-workflow", required=True, @@ -947,6 +966,11 @@ def add_outer_loop_parser(subparsers: argparse._SubParsersAction) -> None: # ty default="", help="Task class ref as 'module.path:ClassName' (overrides config value)", ) + ev.add_argument( + "--task", + default="", + help="Task reference: .toml file, .py file, or module:ClassName (overrides config value)", + ) ref = outer_sub.add_parser("reflect", help="Run reflection on generation") ref.add_argument("project_path", nargs="?", default=".") diff --git a/factory/task.py b/factory/task.py index f896970d3..48dea1faf 100644 --- a/factory/task.py +++ b/factory/task.py @@ -622,3 +622,74 @@ def _try_parse_json_output(result: _RunResult) -> VerifyResult | None: except Exception: pass return None + + +# ── Shared task resolver ──────────────────────────────────────── + + +def resolve_task(task_ref: str, project_path: Path | None = None) -> Task: + """Resolve a task reference to a live Task object. + + Three-step resolution: + 1. If string ends in .toml or resolves to an existing file with a [task] + section → TaskDefinition.from_toml(path) → Task + 2. If string ends in .py or resolves to an existing .py file → import the + module and introspect for a single Task subclass + 3. Else treat as module.path:ClassName via TaskRef.resolve() + """ + ref_path = Path(task_ref) + if not ref_path.is_absolute() and project_path is not None: + candidate = project_path / ref_path + if candidate.exists(): + ref_path = candidate + + # Step 1: TOML file + if task_ref.endswith(".toml") or (ref_path.exists() and ref_path.suffix == ".toml"): + resolved = ref_path if ref_path.exists() else Path(task_ref) + if not resolved.exists(): + raise FileNotFoundError(f"TOML task file not found: {task_ref}") + log.info("resolve_task_toml", path=str(resolved)) + return Task.from_toml(resolved) + + # Step 2: Python file + if task_ref.endswith(".py") or (ref_path.exists() and ref_path.suffix == ".py"): + resolved = ref_path if ref_path.exists() else Path(task_ref) + if not resolved.exists(): + raise FileNotFoundError(f"Python task file not found: {task_ref}") + log.info("resolve_task_python", path=str(resolved)) + return _load_task_from_python_file(resolved) + + # Step 3: module:ClassName import string + log.info("resolve_task_module", ref=task_ref) + return TaskRef(ref=task_ref).resolve() + + +def _load_task_from_python_file(path: Path) -> Task: + """Import a Python file and find the single Task subclass in it.""" + import importlib.util + + spec = importlib.util.spec_from_file_location("_task_module", str(path.resolve())) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load Python module from {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) # type: ignore[union-attr] + + task_classes: list[type[Task]] = [] + for attr_name in dir(mod): + obj = getattr(mod, attr_name) + if ( + isinstance(obj, type) + and issubclass(obj, Task) + and obj is not Task + ): + task_classes.append(obj) + + if len(task_classes) == 0: + raise ImportError(f"No Task subclass found in {path}") + if len(task_classes) > 1: + names = [c.__name__ for c in task_classes] + raise ImportError( + f"Multiple Task subclasses in {path}: {names}. " + f"Expected exactly one." + ) + return task_classes[0]() diff --git a/tests/test_resolve_task.py b/tests/test_resolve_task.py new file mode 100644 index 000000000..33942b19d --- /dev/null +++ b/tests/test_resolve_task.py @@ -0,0 +1,159 @@ +"""Tests for factory.task.resolve_task — TOML, Python file, and module:Class resolution.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from factory.task import Task, resolve_task + + +# ── TOML resolution ───────────────────────────────────────────── + + +class TestResolveTOML: + def test_toml_absolute_path(self, tmp_path: Path): + toml = tmp_path / "my_task.toml" + toml.write_text( + '[task]\nname = "my-task"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "echo ok"\n' + ) + task = resolve_task(str(toml)) + assert isinstance(task, Task) + assert task.name == "my-task" + + def test_toml_relative_to_project(self, tmp_path: Path): + tasks_dir = tmp_path / ".factory" / "tasks" + tasks_dir.mkdir(parents=True) + toml = tasks_dir / "foo.toml" + toml.write_text( + '[task]\nname = "foo"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "true"\n' + ) + task = resolve_task(".factory/tasks/foo.toml", project_path=tmp_path) + assert task.name == "foo" + + def test_toml_missing_raises(self): + with pytest.raises(FileNotFoundError, match="TOML task file not found"): + resolve_task("nonexistent.toml") + + def test_toml_with_scoring_contract(self, tmp_path: Path): + toml = tmp_path / "scored.toml" + toml.write_text( + '[task]\nname = "scored"\n' + '[scoring]\nmethod = "json"\nmetric_path = "stats.accuracy"\n' + '[verify]\ncommand = "python eval.py"\n' + ) + task = resolve_task(str(toml)) + assert task.scoring.method == "json" + assert task.scoring.metric_path == "stats.accuracy" + + +# ── Python file resolution ────────────────────────────────────── + + +class TestResolvePythonFile: + def test_python_file_with_task_subclass(self, tmp_path: Path): + py_file = tmp_path / "my_task.py" + py_file.write_text( + "from factory.task import Task, TaskDefinition, TaskInstance\n" + "from typing import Iterator\n" + "\n" + "class MyCustomTask(Task):\n" + " def __init__(self):\n" + " super().__init__(TaskDefinition(name='custom'))\n" + " def instances(self) -> Iterator[TaskInstance]:\n" + " yield TaskInstance(id='default')\n" + ) + task = resolve_task(str(py_file)) + assert isinstance(task, Task) + assert task.name == "custom" + + def test_python_file_relative_to_project(self, tmp_path: Path): + py_file = tmp_path / "tasks" / "simple.py" + py_file.parent.mkdir(parents=True) + py_file.write_text( + "from factory.task import Task, TaskDefinition\n" + "\n" + "class SimpleTask(Task):\n" + " def __init__(self):\n" + " super().__init__(TaskDefinition(name='simple'))\n" + ) + task = resolve_task("tasks/simple.py", project_path=tmp_path) + assert task.name == "simple" + + def test_python_file_missing_raises(self): + with pytest.raises(FileNotFoundError, match="Python task file not found"): + resolve_task("nonexistent.py") + + def test_python_file_no_task_subclass_raises(self, tmp_path: Path): + py_file = tmp_path / "empty.py" + py_file.write_text("x = 42\n") + with pytest.raises(ImportError, match="No Task subclass found"): + resolve_task(str(py_file)) + + def test_python_file_multiple_subclasses_raises(self, tmp_path: Path): + py_file = tmp_path / "multi.py" + py_file.write_text( + "from factory.task import Task, TaskDefinition\n" + "\n" + "class TaskA(Task):\n" + " def __init__(self):\n" + " super().__init__(TaskDefinition(name='a'))\n" + "\n" + "class TaskB(Task):\n" + " def __init__(self):\n" + " super().__init__(TaskDefinition(name='b'))\n" + ) + with pytest.raises(ImportError, match="Multiple Task subclasses"): + resolve_task(str(py_file)) + + +# ── Module:Class resolution (backward compat) ────────────────── + + +class TestResolveModuleClass: + def test_module_class_format(self): + with pytest.raises((ImportError, ValueError)): + resolve_task("nonexistent.module:FakeTask") + + def test_invalid_format_no_colon(self): + with pytest.raises(ValueError, match="Expected 'module.path:ClassName'"): + resolve_task("not_a_file_and_no_colon") + + +# ── Integration: TOML and Python produce equivalent Tasks ─────── + + +class TestEquivalence: + def test_toml_and_python_same_task(self, tmp_path: Path): + toml = tmp_path / "equiv.toml" + toml.write_text( + '[task]\nname = "equiv"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "pytest -xvs"\n' + '[constraints]\ntimeout = 300\n' + ) + py_file = tmp_path / "equiv_task.py" + py_file.write_text( + "from factory.task import Task, TaskDefinition, ScoringContract, " + "TaskConstraints, VerifyConfig\n" + "\n" + "class EquivTask(Task):\n" + " def __init__(self):\n" + " super().__init__(TaskDefinition(\n" + " name='equiv',\n" + " scoring=ScoringContract(method='exit_code'),\n" + " verify_config=VerifyConfig(command='pytest -xvs'),\n" + " constraints=TaskConstraints(timeout=300),\n" + " ))\n" + ) + toml_task = resolve_task(str(toml)) + py_task = resolve_task(str(py_file)) + assert toml_task.name == py_task.name + assert toml_task.scoring.method == py_task.scoring.method + assert toml_task.definition.verify_config.command == py_task.definition.verify_config.command + assert toml_task.constraints.timeout == py_task.constraints.timeout From 2881c3de9198c594412a91cf72648c6087456c72 Mon Sep 17 00:00:00 2001 From: colehurwitz Date: Wed, 9 Sep 2026 12:04:27 -0400 Subject: [PATCH 2/8] feat: add task-setup workflow mode for scaffolding Task files (#1478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of the task→workflow→outer-loop creation pipeline. - Add task_setup_workflow() to factory/workflow/definitions.py: fork(researcher_domain, researcher_verification) → join → CEO gate → strategist → user gate → builder → FnNode(validate) → archivist - Register in _get_builtin_registry() for automatic discovery - Add WORKFLOW_META entry in skill_export.py for SKILL.md generation - Add 'task-setup' to CEO_MODES in _helpers.py - Add mode suffix in _task_builder.py for CEO task string - Add mode detection line in ceo.md prompt - SKILL.md auto-generates via skill_cache (139 lines) Co-Authored-By: Claude Opus 4.6 --- factory/agents/prompts/ceo.md | 1 + factory/cli/_helpers.py | 1 + factory/cli/_task_builder.py | 8 ++ factory/workflow/definitions.py | 151 +++++++++++++++++++++++++++++++ factory/workflow/skill_export.py | 11 +++ 5 files changed, 172 insertions(+) diff --git a/factory/agents/prompts/ceo.md b/factory/agents/prompts/ceo.md index ba7957b73..acb40add6 100644 --- a/factory/agents/prompts/ceo.md +++ b/factory/agents/prompts/ceo.md @@ -323,6 +323,7 @@ Each mode's full instructions live in a workflow skill under `skills/workflow- float: "review", "deep-qa", "create", + "task-setup", "study", "swebench", "frontend-design", diff --git a/factory/cli/_task_builder.py b/factory/cli/_task_builder.py index 68e29ba19..f8c077ebd 100644 --- a/factory/cli/_task_builder.py +++ b/factory/cli/_task_builder.py @@ -69,6 +69,14 @@ def _mode_suffix(mode: str, discover_only: bool) -> str: "with structural graph context included. " "Terminal mode — does not chain to other modes." ), + "task-setup": ( + "\n\nRun Task Setup mode: scaffold a Task file for the target project. " + "Study the repository to understand its domain and verification methods. " + "Classify whether the task needs TOML (executable verification) or Python " + "(judgmental verification). Produce a validated .factory/tasks/.toml " + "or .py file. Terminal mode — does not chain to other modes. " + "The full step-by-step playbook is in your system prompt above." + ), } if mode == "discover": if discover_only: diff --git a/factory/workflow/definitions.py b/factory/workflow/definitions.py index 15b39ace8..76e06bc57 100644 --- a/factory/workflow/definitions.py +++ b/factory/workflow/definitions.py @@ -41,6 +41,7 @@ "design_workflow", "register_all", "spec_generate_workflow", + "task_setup_workflow", ] DOC_FRESHNESS_GATE_PROMPT = ( @@ -1210,6 +1211,155 @@ def trigger(state: ProjectState, ctx: dict[str, Any]) -> bool: +# ── W₁₄: Task Setup Mode ────────────────────────────────────── + + +def task_setup_workflow() -> Workflow: + """W₁₄: Task Setup — scaffold Task files from a target repository. + + A conversational wizard that studies a target repo, classifies whether + the task needs TOML or Python, and produces a validated TaskDefinition. + + Fork(researcher_domain, researcher_verification) → Join → CEO gate → + Strategist → User gate → Builder → FnNode(validate) → Archivist + """ + nodes: dict[str, Any] = {} + edges: list[Edge] = [] + + _TASK_SETUP_RESEARCHERS = [ + ResearcherConfig( + id="domain", + prompt_template=( + "Domain analysis for task setup. " + "Study the target repository: language, framework, test infrastructure, " + "CI/CD setup, and existing evaluation patterns. " + "Identify what the project does, what its key outputs are, and how " + "quality is currently measured (test suites, linting, benchmarks). " + "Document: project purpose, tech stack, existing test commands, " + "directory structure, and key source files. " + "Write findings to .factory/strategy/research-domain.md." + ), + ), + ResearcherConfig( + id="verification", + prompt_template=( + "Verification method analysis for task setup. " + "Study how the target project verifies correctness: " + "- Does it use pytest, unittest, or another test framework? " + "- Are there integration tests, benchmarks, or eval scripts? " + "- Does any test output structured JSON with scores? " + "- Is verification binary (pass/fail) or graded (partial credit)? " + "Classify the verification type: " + "- EXECUTABLE: shell command + exit code or JSON parse → TOML task " + "- JUDGMENTAL: custom control flow, multi-stage, or LLM-based → Python task " + "This classification follows the eval_spec.py classify_eval_spec_item pattern. " + "Write findings to .factory/strategy/research-verification.md." + ), + ), + ] + r_nodes, r_edges = _research_subgraph( + researchers=_TASK_SETUP_RESEARCHERS, + gate_prompt=( + "Is the domain well-documented? Is the verification classification " + "(EXECUTABLE vs JUDGMENTAL) supported by evidence from the codebase? " + "PROCEED if both researchers produced substantive findings. " + "RELOOP if either is shallow or missing." + ), + ) + nodes.update(r_nodes) + + nodes["strategist"] = AgentNode( + id="strategist", + role=AgentRole.STRATEGIST, + prompt_template=( + "Draft a TaskDefinition for this project. " + "Read ALL research files at .factory/strategy/research-*.md. " + "Based on the verification classification: " + "- If EXECUTABLE: draft a TOML task definition with [task], [instances], " + " [setup], [prompt], [verify], [scoring], and [constraints] sections. " + " The verify command should be a shell command that exits 0 on success. " + " Choose scoring method: 'exit_code' for binary, 'json' for graded. " + "- If JUDGMENTAL: draft a Python Task subclass skeleton with custom " + " instances(), setup(), prompt(), and verify() hooks. Include docstrings " + " explaining what each hook should do for this specific domain. " + "Include a proposed task name (kebab-case), description, timeout, " + "and required capabilities. " + "Write the complete draft to .factory/strategy/current.md." + ), + reads={ + ".factory/strategy/research-domain.md", + ".factory/strategy/research-verification.md", + }, + writes={".factory/strategy/current.md"}, + ) + + nodes["gate_strategy"] = GateNode( + id="gate_strategy", + evaluator_type="user", + reads={".factory/strategy/current.md"}, + ) + + nodes["builder"] = AgentNode( + id="builder", + role=AgentRole.BUILDER, + timeout=600, + prompt_template=( + "Write the task file from the approved specification. " + "Read the approved spec at .factory/strategy/current.md. " + "If the spec describes a TOML task: write .factory/tasks/.toml " + "with all required sections. " + "If the spec describes a Python task: write .factory/tasks/.py " + "with a Task subclass implementing the four hooks. " + "Ensure the task directory exists (mkdir -p .factory/tasks/). " + "After writing, run: factory task validate " + "to verify the task definition is valid." + ), + reads={".factory/strategy/current.md"}, + writes={".factory/reviews/builder-latest.md"}, + ) + + nodes["validate_task"] = FnNode( + id="validate_task", + command="factory task validate {task_name}", + notes=( + "Hard validation gate — the task must pass all checks. " + "The {task_name} placeholder is replaced by the CEO with the " + "actual task name from the builder output." + ), + ) + + nodes["archivist"] = AgentNode( + id="archivist", + role=AgentRole.ARCHIVIST, + prompt_template="Archive the task setup results and task definition.", + reads={".factory/reviews/builder-latest.md"}, + writes={".factory/archive/task-setup.md"}, + blocking=False, + ) + + edges = [ + *r_edges, + Edge(source="gate_research", target="strategist", condition=VerdictType.PROCEED), + Edge(source="gate_research", target="fork_research", condition=VerdictType.RELOOP), + Edge(source="strategist", target="gate_strategy"), + Edge(source="gate_strategy", target="builder", condition=VerdictType.PROCEED), + Edge(source="gate_strategy", target="strategist", condition=VerdictType.RELOOP), + Edge(source="builder", target="validate_task"), + Edge(source="validate_task", target="archivist"), + ] + + def trigger(state: ProjectState, ctx: dict[str, Any]) -> bool: + return ctx.get("mode") == "task-setup" + + return Workflow( + name="task-setup", + nodes=nodes, + edges=edges, + start_node="fork_research", + trigger=trigger, + ) + + # ── W₁₃: Spec Generate Mode ──────────────────────────────────── @@ -1350,6 +1500,7 @@ def _get_builtin_registry() -> dict[str, Any]: _BUILTIN_REGISTRY = { "design": design_workflow, "create": create_workflow, + "task-setup": task_setup_workflow, "spec-generate": spec_generate_workflow, "swebench": lambda: __import__( "factory.workflow.contributed.swebench", fromlist=["workflow"] diff --git a/factory/workflow/skill_export.py b/factory/workflow/skill_export.py index 44d210239..914b2f08c 100644 --- a/factory/workflow/skill_export.py +++ b/factory/workflow/skill_export.py @@ -64,6 +64,17 @@ ), "argument_hint": '"mode description" or "existing_mode: change description"', }, + "task-setup": { + "description": ( + "Task setup mode — scaffolds Task files (.factory/tasks/.toml or .py) " + "from a target repository. A conversational wizard that studies the repo, " + "classifies whether the task needs TOML (shell command + exit code/JSON) " + "or Python (custom control flow), and produces a validated TaskDefinition. " + "Use when the user says 'set up a task', 'create an evaluation harness', " + "or wants to define what to evaluate for the outer loop." + ), + "argument_hint": " --focus 'task description'", + }, "swebench": { "description": ( "SWE-bench benchmark mode — minimal 4-node pipeline for solving " From 18c36273f3869c54cbfdc42767193aed66d6b4b7 Mon Sep 17 00:00:00 2001 From: colehurwitz Date: Wed, 9 Sep 2026 12:07:07 -0400 Subject: [PATCH 3/8] feat: add --task flag to create mode with OptKnob auto-generation (#1478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 of the task→workflow→outer-loop creation pipeline. - Add --task CLI flag to factory ceo parser for task-aware create mode - Add _build_task_aware_directive() in _task_builder.py: resolves the task, extracts scoring contract/constraints, injects directive section with OptKnob mechanical derivation table (same directive-injection pattern as Plugin Package and Update Existing Mode extensions) - Wire --task through cmd_ceo → _execute_ceo → _build_ceo_task - Update create_workflow() strategist prompt to handle Task-Aware directive and include OptKnob generation instructions - Update create_workflow() builder prompt with knob_values/knob_bounds and compose.py validate_composition() post-build gate - 12 new tests for directive injection and task-setup workflow Co-Authored-By: Claude Opus 4.6 --- factory/cli/_ceo_helpers.py | 2 + factory/cli/_parser_groups.py | 4 + factory/cli/_task_builder.py | 56 +++++++++++++ factory/cli/ceo.py | 2 + factory/workflow/definitions.py | 10 +++ tests/test_task_aware_create.py | 138 ++++++++++++++++++++++++++++++++ 6 files changed, 212 insertions(+) create mode 100644 tests/test_task_aware_create.py diff --git a/factory/cli/_ceo_helpers.py b/factory/cli/_ceo_helpers.py index 4f7fd4588..c5f3f68ec 100644 --- a/factory/cli/_ceo_helpers.py +++ b/factory/cli/_ceo_helpers.py @@ -535,6 +535,7 @@ def _execute_ceo( update_existing_mode: str | None, plugin_mode: bool = False, plugin_folder: str | None = None, + task_ref: str | None = None, deferred_spec: str | None, needs_materialize: bool, refine_request: str | None, @@ -715,6 +716,7 @@ def _execute_ceo( update_existing_mode=update_existing_mode, plugin_mode=plugin_mode, plugin_folder=plugin_folder, + task_ref=task_ref, from_plan=resolved_plan.plan if resolved_plan else None, from_plan_feedback=resolved_plan.feedback if resolved_plan else None, just_plan=just_plan, diff --git a/factory/cli/_parser_groups.py b/factory/cli/_parser_groups.py index 1f938c5d5..ea1ddec62 100644 --- a/factory/cli/_parser_groups.py +++ b/factory/cli/_parser_groups.py @@ -454,6 +454,10 @@ def add_entry_point_parsers(sub: argparse._SubParsersAction) -> None: # type: i p.add_argument("--folder", default=None, metavar="PATH", help="Output directory for plugin package (default: ./-plugin). " "Only used with --plugin.") + p.add_argument("--task", default=None, metavar="PATH", + help="Task reference for task-aware create mode: .toml file, .py file, " + "or module:ClassName. Injects the resolved TaskDefinition's scoring " + "contract into create mode's workflow generation.") p.add_argument("--engine", choices=["skill", "tool", "deterministic"], default="skill", help="Execution engine: skill (CEO follows SKILL.md, default), " "tool (CEO drives via factory workflow tool commands), " diff --git a/factory/cli/_task_builder.py b/factory/cli/_task_builder.py index f8c077ebd..82c086263 100644 --- a/factory/cli/_task_builder.py +++ b/factory/cli/_task_builder.py @@ -170,6 +170,58 @@ def _append_deep_research_topic(task: str, focus: str) -> str: ) +def _build_task_aware_directive(task_ref: str, project_path: Path) -> str: + """Build the task-aware create mode directive section. + + Same directive-injection pattern as Plugin Package and Update Existing Mode + extensions — injects into the CEO task text, same file, same audit surface. + """ + from factory.task import resolve_task + + try: + resolved = resolve_task(task_ref, project_path) + defn = resolved.definition + scoring_info = f"method={defn.scoring.method}, metric_path={defn.scoring.metric_path}" + constraints_info = ( + f"timeout={defn.constraints.timeout}s, " + f"max_retries={defn.constraints.max_retries}" + ) + if defn.constraints.required_capabilities: + caps = ", ".join(str(c) for c in defn.constraints.required_capabilities) + constraints_info += f", required_capabilities=[{caps}]" + verify_info = defn.verify_config.command or "(no verify command)" + except Exception as exc: + return ( + f"\n\n## Create Mode (Task-Aware) — RESOLUTION FAILED\n\n" + f"Could not resolve --task {task_ref!r}: {exc}\n" + f"Proceed without task awareness.\n" + ) + + return ( + f"\n\n## Create Mode (Task-Aware)\n\n" + f"A TaskDefinition has been provided via `--task {task_ref}`.\n\n" + f"**Resolved Task:** {defn.name}\n" + f"**Description:** {defn.description}\n" + f"**Scoring Contract:** {scoring_info}\n" + f"**Verify Command:** {verify_info}\n" + f"**Constraints:** {constraints_info}\n\n" + f"The generated workflow MUST be compatible with this task's scoring contract.\n" + f"The Strategist and Builder should tailor the workflow to this task's needs:\n" + f"- Agent prompts should reference the task's verification method\n" + f"- Gate conditions should align with the scoring method ({defn.scoring.method})\n" + f"- Timeout values should respect the task's constraint ({defn.constraints.timeout}s)\n\n" + f"### Workflow-Level OptKnob Auto-Generation\n\n" + f"The generated workflow MUST include OptKnobs using this mechanical derivation:\n\n" + f"| Node Field | OptKnob Kind | Bounds | Expandable |\n" + f"|---|---|---|---|\n" + f"| AgentNode.role (each agent) | model | ['haiku', 'sonnet', 'opus'] | False |\n" + f"| AgentNode.timeout | threshold | [default/2, default, default*2] | False |\n" + f"| AgentNode.prompt_template (when non-empty) | prompt | [current_value] | True |\n\n" + f"Never auto-generate kind='topology' knobs.\n" + f"Use `compose.py validate_composition()` as a post-build gate.\n" + ) + + def _build_ceo_task( project_path: Path, mode: str, @@ -196,6 +248,7 @@ def _build_ceo_task( update_existing_mode: str | None = None, plugin_mode: bool = False, plugin_folder: str | None = None, + task_ref: str | None = None, from_plan: str | None = None, from_plan_feedback: list[str] | None = None, just_plan: bool = False, @@ -456,6 +509,9 @@ def _build_ceo_task( f"factory/workflow/skill_export.py, factory/cli.py, tests/.\n" ) + if task_ref and create_description: + task += _build_task_aware_directive(task_ref, project_path) + if prompt_file: task += ( f"\n\n## Directive\n\n" diff --git a/factory/cli/ceo.py b/factory/cli/ceo.py index 843faf6f3..604014d8a 100644 --- a/factory/cli/ceo.py +++ b/factory/cli/ceo.py @@ -75,6 +75,7 @@ def cmd_ceo(args: argparse.Namespace) -> int: plugin_mode = getattr(args, "plugin", False) plugin_folder = getattr(args, "folder", None) + task_ref = getattr(args, "task", None) if plugin_mode and mode != "create": print( @@ -161,6 +162,7 @@ def cmd_ceo(args: argparse.Namespace) -> int: update_existing_mode=update_existing_mode, plugin_mode=plugin_mode, plugin_folder=plugin_folder, + task_ref=task_ref, deferred_spec=deferred_spec, needs_materialize=needs_materialize, refine_request=refine_request, diff --git a/factory/workflow/definitions.py b/factory/workflow/definitions.py index 76e06bc57..e260fc86d 100644 --- a/factory/workflow/definitions.py +++ b/factory/workflow/definitions.py @@ -997,6 +997,11 @@ def create_workflow() -> Workflow: "7) Interactive vs headless behavior " "Follow conventions from existing workflows — use the same patterns for " "builder→gate→QA→gate loops, archivist placement, and research forks. " + "If the CEO task includes '## Create Mode (Task-Aware)', include workflow-level " + "OptKnobs using the mechanical derivation table from the directive: " + "AgentNode.role → OptKnob(kind='model'), timeout → OptKnob(kind='threshold'), " + "prompt_template → OptKnob(kind='prompt', expandable=True). " + "Never auto-generate kind='topology' knobs. " "Write the specification to .factory/strategy/current.md." ), reads={ @@ -1073,6 +1078,11 @@ def create_workflow() -> Workflow: "7) Run factory workflow export-skills --project-path $PROJECT_PATH to generate the SKILL.md " "8) Write tests in tests/ " "9) Run pytest and ruff check to verify " + "If the CEO task includes '## Create Mode (Task-Aware)', also: " + "10) Add knob_values and knob_bounds dicts to the Workflow object using the " + "mechanical derivation table from the directive (model/threshold/prompt knobs). " + "11) Run compose.py validate_composition() as a post-build gate to verify the " + "generated workflow is compatible with the resolved task. " "Commit changes and open a draft PR." ), reads={".factory/strategy/current.md"}, diff --git a/tests/test_task_aware_create.py b/tests/test_task_aware_create.py new file mode 100644 index 000000000..915d99702 --- /dev/null +++ b/tests/test_task_aware_create.py @@ -0,0 +1,138 @@ +"""Tests for task-aware create mode directive injection.""" + +from __future__ import annotations + +from pathlib import Path + +from factory.cli._task_builder import _build_ceo_task, _build_task_aware_directive + + +class TestTaskAwareDirective: + def test_directive_injected_with_toml(self, tmp_path: Path): + toml = tmp_path / "test.toml" + toml.write_text( + '[task]\nname = "test-task"\n' + 'description = "A test task"\n' + '[scoring]\nmethod = "json"\nmetric_path = "accuracy"\n' + '[verify]\ncommand = "python eval.py"\n' + '[constraints]\ntimeout = 300\n' + ) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "## Create Mode (Task-Aware)" in directive + assert "test-task" in directive + assert "json" in directive + assert "accuracy" in directive + assert "300s" in directive + assert "OptKnob" in directive + + def test_directive_includes_optknob_table(self, tmp_path: Path): + toml = tmp_path / "scored.toml" + toml.write_text( + '[task]\nname = "scored"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "pytest -xvs"\n' + ) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "model" in directive + assert "threshold" in directive + assert "prompt" in directive + assert "topology" in directive.lower() # "Never auto-generate kind='topology'" + + def test_directive_handles_resolution_failure(self, tmp_path: Path): + directive = _build_task_aware_directive("nonexistent.toml", tmp_path) + assert "RESOLUTION FAILED" in directive + assert "Proceed without task awareness" in directive + + def test_build_ceo_task_includes_directive(self, tmp_path: Path): + toml = tmp_path / "task.toml" + toml.write_text( + '[task]\nname = "my-task"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "true"\n' + ) + task = _build_ceo_task( + tmp_path, + "create", + create_description="Build a workflow for my-task", + task_ref=str(toml), + ) + assert "## Create Mode (Task-Aware)" in task + assert "my-task" in task + assert "## Create Mode (New Factory Mode)" in task + + def test_no_directive_without_task_ref(self, tmp_path: Path): + task = _build_ceo_task( + tmp_path, + "create", + create_description="Build a workflow", + ) + assert "## Create Mode (Task-Aware)" not in task + assert "## Create Mode (New Factory Mode)" in task + + def test_no_directive_without_create_description(self, tmp_path: Path): + toml = tmp_path / "task.toml" + toml.write_text( + '[task]\nname = "my-task"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "true"\n' + ) + task = _build_ceo_task( + tmp_path, + "design", + task_ref=str(toml), + ) + assert "## Create Mode (Task-Aware)" not in task + + +class TestTaskSetupWorkflow: + def test_task_setup_workflow_validates(self): + from factory.workflow.definitions import task_setup_workflow + + wf = task_setup_workflow() + assert wf.name == "task-setup" + issues = wf.validate_graph() + assert issues == [], f"Validation issues: {issues}" + + def test_task_setup_registered(self): + from factory.workflow.definitions import _get_builtin_registry + + reg = _get_builtin_registry() + assert "task-setup" in reg + + def test_task_setup_trigger(self): + from factory.models import ProjectState + from factory.workflow.definitions import task_setup_workflow + + wf = task_setup_workflow() + assert wf.trigger is not None + assert wf.trigger(ProjectState.NO_REPO, {"mode": "task-setup"}) + assert not wf.trigger(ProjectState.NO_REPO, {"mode": "create"}) + + def test_task_setup_has_expected_nodes(self): + from factory.workflow.definitions import task_setup_workflow + + wf = task_setup_workflow() + node_ids = set(wf.nodes.keys()) + assert "fork_research" in node_ids + assert "researcher_domain" in node_ids + assert "researcher_verification" in node_ids + assert "join_research" in node_ids + assert "gate_research" in node_ids + assert "strategist" in node_ids + assert "gate_strategy" in node_ids + assert "builder" in node_ids + assert "validate_task" in node_ids + assert "archivist" in node_ids + + def test_task_setup_in_ceo_modes(self): + from factory.cli._helpers import CEO_MODES + + assert "task-setup" in CEO_MODES + + def test_task_setup_skill_meta(self): + from factory.workflow.skill_export import WORKFLOW_META + + assert "task-setup" in WORKFLOW_META + meta = WORKFLOW_META["task-setup"] + assert "description" in meta + assert "scaffold" in meta["description"].lower() or "task" in meta["description"].lower() From 27a13fa9ee7e943295887efdbeeed895772e56ba Mon Sep 17 00:00:00 2001 From: colehurwitz Date: Wed, 9 Sep 2026 12:07:53 -0400 Subject: [PATCH 4/8] feat: add ScoringContract.threshold and domain-level OptKnob generation (#1478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 4 of the task→workflow→outer-loop creation pipeline. - Add optional threshold: float | None = None field to ScoringContract - Parse threshold from TOML [scoring] section via from_toml() - Add domain-level OptKnob suggestion instructions to the task-aware create mode directive: CEO-gated, Strategist-proposed, restricted to prompt/model/threshold kinds (never topology), expandable=False by default - Use ScoringContract.threshold for threshold-kind domain knobs when scoring.method is json with a numeric metric_path - 6 new tests for threshold field and domain-level knob directives Co-Authored-By: Claude Opus 4.6 --- factory/cli/_task_builder.py | 18 +++++++++- factory/task.py | 3 ++ tests/test_task_aware_create.py | 62 +++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/factory/cli/_task_builder.py b/factory/cli/_task_builder.py index 82c086263..f46cf42b4 100644 --- a/factory/cli/_task_builder.py +++ b/factory/cli/_task_builder.py @@ -218,7 +218,23 @@ def _build_task_aware_directive(task_ref: str, project_path: Path) -> str: f"| AgentNode.timeout | threshold | [default/2, default, default*2] | False |\n" f"| AgentNode.prompt_template (when non-empty) | prompt | [current_value] | True |\n\n" f"Never auto-generate kind='topology' knobs.\n" - f"Use `compose.py validate_composition()` as a post-build gate.\n" + f"Use `compose.py validate_composition()` as a post-build gate.\n\n" + f"### Domain-Level OptKnob Suggestions\n\n" + f"When a TaskDefinition is provided, the Strategist should also propose " + f"domain-level knobs as LLM-authored suggestions (not deterministic derivations). " + f"These are CEO-gated — the Strategist proposes them, the CEO reviews.\n\n" + f"**Rules:**\n" + f"- Restrict to prompt/model/threshold kinds only (never topology)\n" + f"- Default expandable=False for auto-generated domain knobs\n" + f"- When scoring.method is 'json' with a numeric metric_path, propose a " + f"threshold-kind knob using ScoringContract.threshold\n" + + ( + f"- Current threshold: {defn.scoring.threshold}\n" + if defn.scoring.threshold is not None + else f"- No threshold configured — Strategist may propose one\n" + ) + + f"- Domain knobs should be grounded in the task's actual constraints, " + f"not hallucinated from descriptions\n" ) diff --git a/factory/task.py b/factory/task.py index 48dea1faf..e9191b799 100644 --- a/factory/task.py +++ b/factory/task.py @@ -71,6 +71,7 @@ class ScoringContract(BaseModel): method: Literal["json", "exit_code"] = "exit_code" metric_path: str = "score" + threshold: float | None = None # ── Capability StrEnum ─────────────────────────────────────────── @@ -287,9 +288,11 @@ def from_toml(cls, path: str | Path) -> TaskDefinition: method = "exit_code" if method not in ("json", "exit_code"): raise ValueError(f"Unknown scoring method: {method}") + raw_threshold = scoring_section.get("threshold") scoring = ScoringContract( method=method, metric_path=scoring_section.get("metric_path", "score"), + threshold=float(raw_threshold) if raw_threshold is not None else None, ) return cls( diff --git a/tests/test_task_aware_create.py b/tests/test_task_aware_create.py index 915d99702..1608228a0 100644 --- a/tests/test_task_aware_create.py +++ b/tests/test_task_aware_create.py @@ -84,6 +84,68 @@ def test_no_directive_without_create_description(self, tmp_path: Path): assert "## Create Mode (Task-Aware)" not in task +class TestScoringContractThreshold: + def test_threshold_default_none(self): + from factory.task import ScoringContract + + s = ScoringContract() + assert s.threshold is None + + def test_threshold_explicit(self): + from factory.task import ScoringContract + + s = ScoringContract(method="json", threshold=0.85) + assert s.threshold == 0.85 + + def test_threshold_from_toml(self, tmp_path: Path): + toml = tmp_path / "thresh.toml" + toml.write_text( + '[task]\nname = "thresh"\n' + '[scoring]\nmethod = "json"\nthreshold = 0.7\n' + '[verify]\ncommand = "python eval.py"\n' + ) + from factory.task import TaskDefinition + + defn = TaskDefinition.from_toml(toml) + assert defn.scoring.threshold == 0.7 + + def test_threshold_absent_in_toml(self, tmp_path: Path): + toml = tmp_path / "no_thresh.toml" + toml.write_text( + '[task]\nname = "no-thresh"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "true"\n' + ) + from factory.task import TaskDefinition + + defn = TaskDefinition.from_toml(toml) + assert defn.scoring.threshold is None + + +class TestDomainLevelKnobs: + def test_directive_with_threshold(self, tmp_path: Path): + toml = tmp_path / "with_thresh.toml" + toml.write_text( + '[task]\nname = "thresh-task"\n' + '[scoring]\nmethod = "json"\nthreshold = 0.8\n' + '[verify]\ncommand = "python eval.py"\n' + ) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "Domain-Level OptKnob" in directive + assert "0.8" in directive + + def test_directive_without_threshold(self, tmp_path: Path): + toml = tmp_path / "no_thresh.toml" + toml.write_text( + '[task]\nname = "no-thresh"\n' + '[scoring]\nmethod = "exit_code"\n' + '[verify]\ncommand = "true"\n' + ) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "Domain-Level OptKnob" in directive + assert "No threshold configured" in directive + + class TestTaskSetupWorkflow: def test_task_setup_workflow_validates(self): from factory.workflow.definitions import task_setup_workflow From 15ac1153a13d2f59c7a188e89ad2dd4bdbfce0b8 Mon Sep 17 00:00:00 2001 From: colehurwitz Date: Wed, 9 Sep 2026 12:30:18 -0400 Subject: [PATCH 5/8] fix: remove unnecessary f-string prefixes in _task_builder.py (#1478) Co-Authored-By: Claude Opus 4.6 --- factory/cli/_task_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/factory/cli/_task_builder.py b/factory/cli/_task_builder.py index f46cf42b4..c00b52e8e 100644 --- a/factory/cli/_task_builder.py +++ b/factory/cli/_task_builder.py @@ -231,10 +231,10 @@ def _build_task_aware_directive(task_ref: str, project_path: Path) -> str: + ( f"- Current threshold: {defn.scoring.threshold}\n" if defn.scoring.threshold is not None - else f"- No threshold configured — Strategist may propose one\n" + else "- No threshold configured — Strategist may propose one\n" ) - + f"- Domain knobs should be grounded in the task's actual constraints, " - f"not hallucinated from descriptions\n" + + "- Domain knobs should be grounded in the task's actual constraints, " + "not hallucinated from descriptions\n" ) From f6759b67f1a9a51045b866cd96adbb0674bb1b42 Mon Sep 17 00:00:00 2001 From: colehurwitz Date: Wed, 9 Sep 2026 12:47:23 -0400 Subject: [PATCH 6/8] fix: use absolute path for chess-evolve.toml in test_compose.py Relative path 'benchmarks/configs/chess-evolve.toml' breaks when the working directory changes due to test ordering/isolation. Resolve the path relative to the repo root via Path(__file__). Co-Authored-By: Claude Opus 4.6 --- tests/test_compose.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_compose.py b/tests/test_compose.py index a4d84af26..690165550 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -267,11 +267,13 @@ def test_plain_object_not_protocol(self): class TestTomlTaskCapabilities: + _CHESS_TOML = Path(__file__).resolve().parent.parent / "benchmarks" / "configs" / "chess-evolve.toml" + def test_chess_evolve_toml_no_builder_required(self): """chess-evolve.toml with required_capabilities=[] should need no capabilities.""" from factory.task import TaskDefinition - defn = TaskDefinition.from_toml("benchmarks/configs/chess-evolve.toml") + defn = TaskDefinition.from_toml(self._CHESS_TOML) assert defn.constraints.required_capabilities == [] task = Task(definition=defn) caps = TaskCapabilities.from_task(task) @@ -281,7 +283,7 @@ def test_chess_evolve_toml_passes_any_workflow(self): """chess-evolve.toml should pass composition with a research-only workflow.""" from factory.task import TaskDefinition - defn = TaskDefinition.from_toml("benchmarks/configs/chess-evolve.toml") + defn = TaskDefinition.from_toml(self._CHESS_TOML) task = Task(definition=defn) wf = _make_workflow(researcher=True, name="research-only") validate_composition(wf, task) From 32ad7382ef5108876cb63074aa78efedd58c1ee7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 11 Sep 2026 16:20:37 +0000 Subject: [PATCH 7/8] feat: add DataNode OptKnob derivation rules to task-aware directive (#1478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a '### DataNode OptKnob Auto-Generation' section to _build_task_aware_directive() with mechanical derivation rules for DataNode fields: - parallelism → threshold knob, bounds [1, current, current*2, 8] - limit (when set) → threshold knob, bounds [limit//2, limit, limit*2] - max_items → threshold knob, bounds [100, 250, 500, 1000] Explicitly prohibits topology knobs for split/shuffle fields. Includes OptKnob.description for each knob (uses PR #1487's field). 8 new tests covering DataNode section presence, individual field derivation rules, kind validation, expandable flags, and topology exclusion. Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/cli/_task_builder.py | 15 ++++++ tests/test_task_aware_create.py | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/factory/cli/_task_builder.py b/factory/cli/_task_builder.py index c00b52e8e..7358b3105 100644 --- a/factory/cli/_task_builder.py +++ b/factory/cli/_task_builder.py @@ -217,6 +217,21 @@ def _build_task_aware_directive(task_ref: str, project_path: Path) -> str: f"| AgentNode.role (each agent) | model | ['haiku', 'sonnet', 'opus'] | False |\n" f"| AgentNode.timeout | threshold | [default/2, default, default*2] | False |\n" f"| AgentNode.prompt_template (when non-empty) | prompt | [current_value] | True |\n\n" + f"### DataNode OptKnob Auto-Generation\n\n" + f"When the generated workflow contains DataNode instances, " + f"derive additional threshold OptKnobs from their fields:\n\n" + f"| Node Field | OptKnob Kind | Bounds | Expandable | Description |\n" + f"|---|---|---|---|---|\n" + f"| DataNode.parallelism | threshold | " + f"[1, current_value, current_value*2, 8] | False | " + f"Number of concurrent data items processed in parallel |\n" + f"| DataNode.limit (when set, not None) | threshold | " + f"[limit//2, limit, limit*2] | True | " + f"Maximum number of data items to process per run |\n" + f"| DataNode.max_items | threshold | " + f"[100, 250, 500, 1000] | True | " + f"Safety ceiling for total data items loaded |\n\n" + f"Do NOT add topology knobs for DataNode.split or DataNode.shuffle.\n\n" f"Never auto-generate kind='topology' knobs.\n" f"Use `compose.py validate_composition()` as a post-build gate.\n\n" f"### Domain-Level OptKnob Suggestions\n\n" diff --git a/tests/test_task_aware_create.py b/tests/test_task_aware_create.py index 1608228a0..e24d8b682 100644 --- a/tests/test_task_aware_create.py +++ b/tests/test_task_aware_create.py @@ -146,6 +146,87 @@ def test_directive_without_threshold(self, tmp_path: Path): assert "No threshold configured" in directive +class TestDataNodeOptKnobs: + """Tests for DataNode OptKnob derivation rules in the task-aware directive.""" + + def _make_toml(self, tmp_path: Path) -> Path: + toml = tmp_path / "dn-task.toml" + toml.write_text( + '[task]\nname = "dn-task"\n' + '[scoring]\nmethod = "json"\nmetric_path = "score"\n' + '[verify]\ncommand = "python eval.py"\n' + ) + return toml + + def test_directive_contains_datanode_section(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "### DataNode OptKnob Auto-Generation" in directive + + def test_parallelism_knob_in_directive(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "DataNode.parallelism" in directive + assert "Number of concurrent data items processed in parallel" in directive + + def test_limit_knob_in_directive(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "DataNode.limit" in directive + assert "Maximum number of data items to process per run" in directive + # limit is conditional — "when set, not None" + assert "when set" in directive + + def test_max_items_knob_in_directive(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + assert "DataNode.max_items" in directive + assert "Safety ceiling for total data items loaded" in directive + + def test_no_topology_knobs_for_datanode(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + # split/shuffle should only appear in the prohibition instruction, not as table rows + assert "Do NOT add topology knobs for DataNode.split or DataNode.shuffle" in directive + dn_start = directive.index("### DataNode OptKnob Auto-Generation") + dn_section = directive[dn_start:directive.index("Do NOT add topology", dn_start)] + table_rows = [ln for ln in dn_section.splitlines() if ln.startswith("| DataNode.")] + for row in table_rows: + assert "split" not in row + assert "shuffle" not in row + + def test_datanode_knobs_are_threshold_kind(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + # Extract the DataNode table section + dn_start = directive.index("### DataNode OptKnob Auto-Generation") + dn_section = directive[dn_start:directive.index("Never auto-generate", dn_start)] + # All three DataNode table rows should specify 'threshold' kind + lines = [ln for ln in dn_section.splitlines() if ln.startswith("| DataNode.")] + assert len(lines) == 3 + for line in lines: + assert "threshold" in line + + def test_parallelism_not_expandable(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + dn_start = directive.index("### DataNode OptKnob Auto-Generation") + dn_section = directive[dn_start:directive.index("Never auto-generate", dn_start)] + parallelism_line = [ + ln for ln in dn_section.splitlines() if "DataNode.parallelism" in ln + ][0] + assert "| False |" in parallelism_line + + def test_limit_and_max_items_expandable(self, tmp_path: Path): + toml = self._make_toml(tmp_path) + directive = _build_task_aware_directive(str(toml), tmp_path) + dn_start = directive.index("### DataNode OptKnob Auto-Generation") + dn_section = directive[dn_start:directive.index("Never auto-generate", dn_start)] + for field in ("DataNode.limit", "DataNode.max_items"): + field_line = [ln for ln in dn_section.splitlines() if field in ln][0] + assert "| True |" in field_line + + class TestTaskSetupWorkflow: def test_task_setup_workflow_validates(self): from factory.workflow.definitions import task_setup_workflow From 33cb1b78b02617cd76f6478ffc850a654db27472 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 11 Sep 2026 16:38:41 +0000 Subject: [PATCH 8/8] fix: add deep-QA pipeline to task-setup workflow and update registry count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add _deep_qa_subgraph() to task_setup_workflow(): fork_qa → [health_checker, code_reviewer, adversarial_tester] → join_qa → gate_qa between builder and validate_task - Update registry count assertion from 16 to 17 in test_spec_generate.py - Regenerate workflow-task-setup/SKILL.md and SKILL.annotations.yaml Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/workflow/definitions.py | 29 ++- tests/test_spec_generate.py | 2 +- workflow-task-setup/SKILL.annotations.yaml | 266 +++++++++++++++++++++ workflow-task-setup/SKILL.md | 216 +++++++++++++++++ 4 files changed, 510 insertions(+), 3 deletions(-) create mode 100644 workflow-task-setup/SKILL.annotations.yaml create mode 100644 workflow-task-setup/SKILL.md diff --git a/factory/workflow/definitions.py b/factory/workflow/definitions.py index e260fc86d..21aee1ade 100644 --- a/factory/workflow/definitions.py +++ b/factory/workflow/definitions.py @@ -1231,7 +1231,8 @@ def task_setup_workflow() -> Workflow: the task needs TOML or Python, and produces a validated TaskDefinition. Fork(researcher_domain, researcher_verification) → Join → CEO gate → - Strategist → User gate → Builder → FnNode(validate) → Archivist + Strategist → User gate → Builder → deep-QA → gate_qa → + FnNode(validate) → Archivist """ nodes: dict[str, Any] = {} edges: list[Edge] = [] @@ -1328,6 +1329,25 @@ def task_setup_workflow() -> Workflow: writes={".factory/reviews/builder-latest.md"}, ) + # Deep-QA subgraph: fork_qa → [health_checker, code_reviewer, adversarial_tester] → join_qa + dq_nodes, dq_edges = _deep_qa_subgraph() + nodes.update(dq_nodes) + + nodes["gate_qa"] = GateNode( + id="gate_qa", + evaluator_type="agent", + evaluator_role=AgentRole.CEO, + gate_prompt=( + "Review QA results for the task definition. PROCEED if all checks pass. " + "RELOOP to builder (max 3 iterations) if issues found." + ), + reads={ + ".factory/reviews/health-check.md", + ".factory/reviews/code-review.md", + ".factory/reviews/adversarial-qa.md", + }, + ) + nodes["validate_task"] = FnNode( id="validate_task", command="factory task validate {task_name}", @@ -1354,7 +1374,12 @@ def task_setup_workflow() -> Workflow: Edge(source="strategist", target="gate_strategy"), Edge(source="gate_strategy", target="builder", condition=VerdictType.PROCEED), Edge(source="gate_strategy", target="strategist", condition=VerdictType.RELOOP), - Edge(source="builder", target="validate_task"), + # Builder → deep-QA → gate_qa → validate_task → archivist + Edge(source="builder", target="fork_qa"), + *dq_edges, + Edge(source="join_qa", target="gate_qa"), + Edge(source="gate_qa", target="validate_task", condition=VerdictType.PROCEED), + Edge(source="gate_qa", target="builder", condition=VerdictType.RELOOP), Edge(source="validate_task", target="archivist"), ] diff --git a/tests/test_spec_generate.py b/tests/test_spec_generate.py index 6db94c653..66a52b025 100644 --- a/tests/test_spec_generate.py +++ b/tests/test_spec_generate.py @@ -92,7 +92,7 @@ def test_register_all_includes_spec_generate(self) -> None: def test_register_all_count(self) -> None: all_wf = register_all() - assert len(all_wf) == 16 + assert len(all_wf) == 17 def test_all_workflows_validate(self) -> None: all_wf = register_all() diff --git a/workflow-task-setup/SKILL.annotations.yaml b/workflow-task-setup/SKILL.annotations.yaml new file mode 100644 index 000000000..dba5e6382 --- /dev/null +++ b/workflow-task-setup/SKILL.annotations.yaml @@ -0,0 +1,266 @@ +fork_research: + type: ForkNode + id: fork_research + targets: researcher_domain,researcher_verification + edges_out: + - target: researcher_domain + condition: null + - target: researcher_verification + condition: null +researcher_domain: + type: AgentNode + id: researcher_domain + role: researcher + blocking: 'true' + reads: [] + writes: + - .factory/strategy/research-domain.md + edges_out: + - target: join_research + condition: null + slots: + task_prompt_researcher_domain: 'Domain analysis for task setup. Study the target + repository: language, framework, test infrastructure, CI/CD setup, and existing + evaluation patterns. Identify what the project does, what its key outputs are, + and how quality is currently measured (test suites, linting, benchmarks). Document: + project purpose, tech stack, existing test commands, directory structure, and + key source files. Write findings to .factory/strategy/research-domain.md. + + Write output to: .factory/strategy/research-domain.md' + timeout_researcher_domain: '600' +researcher_verification: + type: AgentNode + id: researcher_verification + role: researcher + blocking: 'true' + reads: [] + writes: + - .factory/strategy/research-verification.md + edges_out: + - target: join_research + condition: null + slots: + task_prompt_researcher_verification: 'Verification method analysis for task setup. + Study how the target project verifies correctness: - Does it use pytest, unittest, + or another test framework? - Are there integration tests, benchmarks, or eval + scripts? - Does any test output structured JSON with scores? - Is verification + binary (pass/fail) or graded (partial credit)? Classify the verification type: + - EXECUTABLE: shell command + exit code or JSON parse → TOML task - JUDGMENTAL: + custom control flow, multi-stage, or LLM-based → Python task This classification + follows the eval_spec.py classify_eval_spec_item pattern. Write findings to + .factory/strategy/research-verification.md. + + Write output to: .factory/strategy/research-verification.md' + timeout_researcher_verification: '600' +join_research: + type: JoinNode + id: join_research + sources: researcher_domain,researcher_verification + reads: [] + writes: [] + edges_out: + - target: gate_research + condition: null +gate_research: + type: GateNode + id: gate_research + evaluator_type: agent + evaluator_role: ceo + reads: + - .factory/strategy/research-domain.md + - .factory/strategy/research-verification.md + edges_out: + - target: strategist + condition: PROCEED + - target: fork_research + condition: RELOOP + slots: + gate_prompt_gate_research: Is the domain well-documented? Is the verification + classification (EXECUTABLE vs JUDGMENTAL) supported by evidence from the codebase? + PROCEED if both researchers produced substantive findings. RELOOP if either + is shallow or missing. + max_iterations_gate_research: '3' +strategist: + type: AgentNode + id: strategist + role: strategist + blocking: 'true' + reads: + - .factory/strategy/research-domain.md + - .factory/strategy/research-verification.md + writes: + - .factory/strategy/current.md + edges_out: + - target: gate_strategy + condition: null + slots: + task_prompt_strategist: 'Draft a TaskDefinition for this project. Read ALL research + files at .factory/strategy/research-*.md. Based on the verification classification: + - If EXECUTABLE: draft a TOML task definition with [task], [instances], [setup], + [prompt], [verify], [scoring], and [constraints] sections. The verify command + should be a shell command that exits 0 on success. Choose scoring method: + ''exit_code'' for binary, ''json'' for graded. - If JUDGMENTAL: draft a Python + Task subclass skeleton with custom instances(), setup(), prompt(), and verify() + hooks. Include docstrings explaining what each hook should do for this specific + domain. Include a proposed task name (kebab-case), description, timeout, and + required capabilities. Write the complete draft to .factory/strategy/current.md. + + Read: .factory/strategy/research-domain.md, .factory/strategy/research-verification.md + + Write output to: .factory/strategy/current.md' + timeout_strategist: '600' +gate_strategy: + type: GateNode + id: gate_strategy + evaluator_type: user + reads: + - .factory/strategy/current.md + edges_out: + - target: builder + condition: PROCEED + - target: strategist + condition: RELOOP + slots: + max_iterations_gate_strategy: '3' +builder: + type: AgentNode + id: builder + role: builder + blocking: 'true' + reads: + - .factory/strategy/current.md + writes: + - .factory/reviews/builder-latest.md + edges_out: + - target: fork_qa + condition: null + slots: + task_prompt_builder: 'Write the task file from the approved specification. Read + the approved spec at .factory/strategy/current.md. If the spec describes a TOML + task: write .factory/tasks/.toml with all required sections. If the spec + describes a Python task: write .factory/tasks/.py with a Task subclass + implementing the four hooks. Ensure the task directory exists (mkdir -p .factory/tasks/). + After writing, run: factory task validate to verify the task definition + is valid. + + Read: .factory/strategy/current.md + + Write output to: .factory/reviews/builder-latest.md' + timeout_builder: '600' +fork_qa: + type: ForkNode + id: fork_qa + targets: health_checker,code_reviewer,adversarial_tester + edges_out: + - target: join_qa + condition: null +health_checker: + type: AgentNode + id: health_checker + role: health_checker + blocking: 'true' + reads: + - .factory/reviews/builder-latest.md + - .factory/strategy/current.md + writes: + - .factory/reviews/health-check.md + edges_out: [] + slots: + task_prompt_health_checker: 'Execute health_checker task for the project. + + Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md + + Write output to: .factory/reviews/health-check.md' + timeout_health_checker: '600' +code_reviewer: + type: AgentNode + id: code_reviewer + role: code_reviewer + blocking: 'true' + reads: + - .factory/reviews/builder-latest.md + - .factory/strategy/current.md + writes: + - .factory/reviews/code-review.md + edges_out: [] + slots: + task_prompt_code_reviewer: 'Execute code_reviewer task for the project. + + Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md + + Write output to: .factory/reviews/code-review.md' + timeout_code_reviewer: '900' +adversarial_tester: + type: AgentNode + id: adversarial_tester + role: adversarial_tester + blocking: 'true' + reads: + - .factory/reviews/builder-latest.md + - .factory/strategy/current.md + writes: + - .factory/reviews/adversarial-qa.md + edges_out: [] + slots: + task_prompt_adversarial_tester: 'Execute adversarial_tester task for the project. + + Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md + + Write output to: .factory/reviews/adversarial-qa.md' + timeout_adversarial_tester: '1800' +join_qa: + type: JoinNode + id: join_qa + sources: health_checker,code_reviewer,adversarial_tester + reads: + - .factory/reviews/adversarial-qa.md + - .factory/reviews/code-review.md + - .factory/reviews/health-check.md + writes: [] + edges_out: + - target: gate_qa + condition: null +gate_qa: + type: GateNode + id: gate_qa + evaluator_type: agent + evaluator_role: ceo + reads: + - .factory/reviews/adversarial-qa.md + - .factory/reviews/code-review.md + - .factory/reviews/health-check.md + edges_out: + - target: validate_task + condition: PROCEED + - target: builder + condition: RELOOP + slots: + gate_prompt_gate_qa: Review QA results for the task definition. PROCEED if all + checks pass. RELOOP to builder (max 3 iterations) if issues found. + max_iterations_gate_qa: '3' +validate_task: + type: FnNode + id: validate_task + command: factory task validate {task_name} + reads: [] + writes: [] + edges_out: + - target: archivist + condition: null +archivist: + type: AgentNode + id: archivist + role: archivist + blocking: 'false' + reads: + - .factory/reviews/builder-latest.md + writes: + - .factory/archive/task-setup.md + edges_out: [] + slots: + task_prompt_archivist: 'Archive the task setup results and task definition. + + Read: .factory/reviews/builder-latest.md + + Write output to: .factory/archive/task-setup.md' + timeout_archivist: '300' diff --git a/workflow-task-setup/SKILL.md b/workflow-task-setup/SKILL.md new file mode 100644 index 000000000..453de4e00 --- /dev/null +++ b/workflow-task-setup/SKILL.md @@ -0,0 +1,216 @@ +--- +name: workflow-task-setup +description: "Task setup mode — scaffolds Task files (.factory/tasks/.toml or .py) from a target repository. A conversational wizard that studies the repo, classifies whether the task needs TOML (shell command + exit code/JSON) or Python (custom control flow), and produces a validated TaskDefinition. Use when the user says 'set up a task', 'create an evaluation harness', or wants to define what to evaluate for the outer loop." +disable-model-invocation: true +argument-hint: " --focus 'task description'" +--- + +# Task Setup Workflow + +The user wants: **$ARGUMENTS** + +## Phase 1: Research (Parallel) + +Spawn 2 agents in parallel: + +```bash +factory agent researcher --review-tag domain --task "Domain analysis for task setup. Study the target repository: language, framework, test infrastructure, CI/CD setup, and existing evaluation patterns. Identify what the project does, what its key outputs are, and how quality is currently measured (test suites, linting, benchmarks). Document: project purpose, tech stack, existing test commands, directory structure, and key source files. Write findings to .factory/strategy/research-domain.md. +Write output to: .factory/strategy/research-domain.md" --project "$PROJECT_PATH" --timeout 600 & +``` + +```bash +factory agent researcher --review-tag verification --task "Verification method analysis for task setup. Study how the target project verifies correctness: - Does it use pytest, unittest, or another test framework? - Are there integration tests, benchmarks, or eval scripts? - Does any test output structured JSON with scores? - Is verification binary (pass/fail) or graded (partial credit)? Classify the verification type: - EXECUTABLE: shell command + exit code or JSON parse → TOML task - JUDGMENTAL: custom control flow, multi-stage, or LLM-based → Python task This classification follows the eval_spec.py classify_eval_spec_item pattern. Write findings to .factory/strategy/research-verification.md. +Write output to: .factory/strategy/research-verification.md" --project "$PROJECT_PATH" --timeout 600 & +``` + +```bash +wait +``` + +**Important:** Run ALL commands above in a **single** Bash tool call with timeout set to at least 600 seconds. + +```bash +# Artifact verification: researcher_domain +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/research-domain.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: researcher_domain: .factory/strategy/research-domain.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: researcher_domain: .factory/strategy/research-domain.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=researcher_domain" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: researcher_domain artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=researcher_domain" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" + +# Artifact verification: researcher_verification +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/research-verification.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: researcher_verification: .factory/strategy/research-verification.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: researcher_verification: .factory/strategy/research-verification.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=researcher_verification" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: researcher_verification artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=researcher_verification" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(post-barrier harness verification — DO NOT SKIP)* + +## Barrier: Research + +Wait for all parallel agents to complete: `researcher_domain`, `researcher_verification` + +### CEO Review — Research + +Apply the CEO Review Gate protocol: +1. Read the agent output for the preceding step +2. Read artifacts: `.factory/strategy/research-domain.md`, `.factory/strategy/research-verification.md` +3. Assess: Is the domain well-documented? Is the verification classification (EXECUTABLE vs JUDGMENTAL) supported by evidence from the codebase? PROCEED if both researchers produced substantive findings. RELOOP if either is shallow or missing. +4. Write verdict to `.factory/reviews/ceo-verdict-research.md` +5. **PROCEED** → continue to next step +6. **REDIRECT** → re-invoke the preceding agent with corrections (max 2) +7. **ABORT** → log failure and skip to archival + +*On RELOOP: return to `fork_research` (max 3 iterations)* + +## Phase 2: Strategist + +```bash +factory agent strategist --task "Draft a TaskDefinition for this project. Read ALL research files at .factory/strategy/research-*.md. Based on the verification classification: - If EXECUTABLE: draft a TOML task definition with [task], [instances], [setup], [prompt], [verify], [scoring], and [constraints] sections. The verify command should be a shell command that exits 0 on success. Choose scoring method: 'exit_code' for binary, 'json' for graded. - If JUDGMENTAL: draft a Python Task subclass skeleton with custom instances(), setup(), prompt(), and verify() hooks. Include docstrings explaining what each hook should do for this specific domain. Include a proposed task name (kebab-case), description, timeout, and required capabilities. Write the complete draft to .factory/strategy/current.md. +Read: .factory/strategy/research-domain.md, .factory/strategy/research-verification.md +Write output to: .factory/strategy/current.md" --project "$PROJECT_PATH" --timeout 600 +``` + +```bash +# Artifact verification: strategist +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/current.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: strategist: .factory/strategy/current.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: strategist: .factory/strategy/current.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=strategist" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: strategist artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=strategist" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +### Steering Point — Strategy (User Approval) + +**This is a USER approval gate, NOT a CEO review gate. Do NOT self-approve.** + +Present the strategy/findings to the user by summarizing key points in your output. +Then explicitly ask the user: "Do you approve this plan, or do you have feedback?" + +**You MUST wait for the user's response before proceeding.** +- The user says "approve", "yes", "looks good", or similar → proceed to next step +- The user provides feedback or corrections → re-run the previous step incorporating their feedback +- Do NOT write a verdict file and auto-proceed — this gate requires human input + +*On RELOOP: return to `strategist` (max 3 iterations)* + +## Phase 3: Builder + +```bash +factory agent builder --task "Write the task file from the approved specification. Read the approved spec at .factory/strategy/current.md. If the spec describes a TOML task: write .factory/tasks/.toml with all required sections. If the spec describes a Python task: write .factory/tasks/.py with a Task subclass implementing the four hooks. Ensure the task directory exists (mkdir -p .factory/tasks/). After writing, run: factory task validate to verify the task definition is valid. +Read: .factory/strategy/current.md +Write output to: .factory/reviews/builder-latest.md" --project "$PROJECT_PATH" --timeout 600 +``` + +```bash +# Artifact verification: builder +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/builder-latest.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: builder: .factory/reviews/builder-latest.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: builder: .factory/reviews/builder-latest.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=builder" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: builder artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=builder" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +## Phase 4: Qa (Parallel) + +Spawn 3 agents in parallel: + +```bash +factory agent health_checker --task "Execute health_checker task for the project. +Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md +Write output to: .factory/reviews/health-check.md" --project "$PROJECT_PATH" --timeout 600 & +``` + +```bash +factory agent code_reviewer --task "Execute code_reviewer task for the project. +Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md +Write output to: .factory/reviews/code-review.md" --project "$PROJECT_PATH" --timeout 900 & +``` + +```bash +factory agent adversarial_tester --task "Execute adversarial_tester task for the project. +Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md +Write output to: .factory/reviews/adversarial-qa.md" --project "$PROJECT_PATH" --timeout 1800 & +``` + +```bash +wait +``` + +**Important:** Run ALL commands above in a **single** Bash tool call with timeout set to at least 1800 seconds. + +```bash +# Artifact verification: health_checker +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/health-check.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: health_checker: .factory/reviews/health-check.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: health_checker: .factory/reviews/health-check.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=health_checker" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: health_checker artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=health_checker" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" + +# Artifact verification: code_reviewer +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/code-review.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: code_reviewer: .factory/reviews/code-review.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: code_reviewer: .factory/reviews/code-review.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=code_reviewer" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: code_reviewer artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=code_reviewer" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" + +# Artifact verification: adversarial_tester +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/adversarial-qa.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: adversarial_tester: .factory/reviews/adversarial-qa.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: adversarial_tester: .factory/reviews/adversarial-qa.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=adversarial_tester" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: adversarial_tester artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=adversarial_tester" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(post-barrier harness verification — DO NOT SKIP)* + +## Barrier: Qa + +Wait for all parallel agents to complete: `health_checker`, `code_reviewer`, `adversarial_tester` + +Read combined outputs: `.factory/reviews/adversarial-qa.md`, `.factory/reviews/code-review.md`, `.factory/reviews/health-check.md` + +### CEO Review — Qa + +Apply the CEO Review Gate protocol: +1. Read the agent output for the preceding step +2. Read artifacts: `.factory/reviews/adversarial-qa.md`, `.factory/reviews/code-review.md`, `.factory/reviews/health-check.md` +3. Assess: Review QA results for the task definition. PROCEED if all checks pass. RELOOP to builder (max 3 iterations) if issues found. +4. Write verdict to `.factory/reviews/ceo-verdict-qa.md` +5. **PROCEED** → continue to next step +6. **REDIRECT** → re-invoke the preceding agent with corrections (max 2) +7. **ABORT** → log failure and skip to archival + +*On RELOOP: return to `builder` (max 3 iterations)* + +## Step: Validate Task + +Hard validation gate — the task must pass all checks. The {task_name} placeholder is replaced by the CEO with the actual task name from the builder output. + +```bash +factory task validate {task_name} +``` + +## Phase 5: Archivist + +```bash +factory agent archivist --task "Archive the task setup results and task definition. +Read: .factory/reviews/builder-latest.md +Write output to: .factory/archive/task-setup.md" --project "$PROJECT_PATH" --timeout 300 --model haiku & +``` +*(fire-and-forget — CEO continues immediately)*