diff --git a/.actor/input_schema.json b/.actor/input_schema.json index a822937..21eb3be 100644 --- a/.actor/input_schema.json +++ b/.actor/input_schema.json @@ -24,7 +24,7 @@ "type": "string", "description": "Markdown with YAML frontmatter + ## Test / ## Checkpoint sections separated by ---. Checkpoint supports: contains:, regex:, json-schema:, script: prefixes for deterministic checks, and plain text for LLM judge. Multiple checks in one checkpoint all must pass.", "editor": "textarea", - "prefill": "---\nname: example-eval\ndescription: Simple eval demonstrating checkpoint types\nabortOnFailure: false\n---\n\n## Test\nWhat is the largest planet in our solar system? Answer in one sentence.\n\n## Checkpoint\ncontains: Jupiter\nThe answer should be scientifically accurate.\n" + "prefill": "---\nname: example-eval\ndescription: Simple eval demonstrating checkpoint types\n---\n\n## Test\nWhat is the largest planet in our solar system? Answer in one sentence.\n\n## Checkpoint\ncontains: Jupiter\nThe answer should be scientifically accurate.\n" }, "systemPrompt": { "title": "System Prompt", diff --git a/CLAUDE.md b/CLAUDE.md index 57024e9..a73e59e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,7 +51,6 @@ Markdown s YAML frontmatter. Validace při startu (throwne `ScenarioParseError` --- name: scenario-name (required) description: "Description" -abortOnFailure: false --- ## Test @@ -67,6 +66,8 @@ Plain text = LLM judge prompt (Optional) Follow-up question ``` +Abort semantics: nastav top-level runner input `abortOnFailure` — jakýkoli `abortOnFailure:` klíč uvnitř scénářového YAML frontmatter je tiše zahozen (input je jediný zdroj pravdy). + Multi-test: bloky oddělené `---`. ## Checkpoint systém (`shared/src/judge.ts`) diff --git a/actors/runner/.actor/input_schema.json b/actors/runner/.actor/input_schema.json index 7332838..7eb6ca2 100644 --- a/actors/runner/.actor/input_schema.json +++ b/actors/runner/.actor/input_schema.json @@ -24,7 +24,7 @@ "type": "string", "description": "Markdown with YAML frontmatter + ## Test / ## Checkpoint sections separated by ---. Checkpoint supports: contains:, regex:, json-schema:, script: prefixes for deterministic checks, and plain text for LLM judge. Multiple checks in one checkpoint all must pass.", "editor": "textarea", - "prefill": "---\nname: example-eval\ndescription: Simple eval demonstrating checkpoint types\nabortOnFailure: false\n---\n\n## Test\nWhat is the largest planet in our solar system? Answer in one sentence.\n\n## Checkpoint\ncontains: Jupiter\nThe answer should be scientifically accurate.\n" + "prefill": "---\nname: example-eval\ndescription: Simple eval demonstrating checkpoint types\n---\n\n## Test\nWhat is the largest planet in our solar system? Answer in one sentence.\n\n## Checkpoint\ncontains: Jupiter\nThe answer should be scientifically accurate.\n" }, "systemPrompt": { "title": "System Prompt", @@ -61,9 +61,9 @@ "abortOnFailure": { "title": "Abort on test failure", "type": "boolean", - "description": "If true, the eval run aborts on the first failing test in the scenario. If false, subsequent tests still run independently. When set explicitly, overrides the abortOnFailure flag in the scenario's YAML frontmatter; when left unset, the scenario YAML value is used (which itself defaults to false if absent from the frontmatter).", - "nullable": true, - "sectionCaption": "Behavior" + "description": "If true, the eval run aborts on the first failing test in the scenario. If false, subsequent tests still run independently. This is the single source of truth — any `abortOnFailure:` key in the scenario's YAML frontmatter is ignored.", + "default": false, + "nullable": true }, "envVariables": { "title": "Environment Variables", @@ -77,8 +77,7 @@ "title": "Pre-authenticate Apify CLI", "type": "boolean", "description": "If true (default), the runner populates ~/.apify/auth.json with APIFY_TOKEN via 'apify login --token' BEFORE the agent starts. This mirrors what every real Apify developer has after running 'apify login' once on their machine — 'apify push' from the agent will work without the agent having to discover auth flow. Set to false for raw-unauthed measurement (the agent has to figure out auth on its own — surfaces F11 in the eval pack's FINDINGS.md).", - "default": true, - "sectionCaption": "Authentication" + "default": true }, "initPreset": { "title": "Tool Setup Preset", diff --git a/actors/runner/README.md b/actors/runner/README.md index 5c6066f..77b9ab7 100644 --- a/actors/runner/README.md +++ b/actors/runner/README.md @@ -27,7 +27,6 @@ A scenario is a Markdown file with one or more tests. Each test has a prompt and --- name: my-test description: What this scenario tests -abortOnFailure: false --- ## Test @@ -54,7 +53,7 @@ The answer should name a real, currently existing repository. - Tests are separated by `---` - Each test needs `## Test` (the prompt) and `## Checkpoint` (evaluation criteria) - Optional: `## Monitor` — a follow-up question about the agent's work -- YAML frontmatter: `name` (required), `description`, `abortOnFailure` (stop on first failure). The top-level `abortOnFailure` input on the Actor overrides this when set explicitly — useful for flipping abort behaviour per-run without editing the scenario markdown. +- YAML frontmatter: `name` (required), `description`. Whether the scenario aborts on the first failing test is controlled by the top-level `abortOnFailure` input on the Actor (default `false`); any `abortOnFailure:` key in the scenario YAML is ignored (accepted for backward compatibility, but not read). - **Retries (experimental):** `maxRetries` input re-runs a failed test. ⚠️ Retries create a fresh workspace but may produce inconsistent results due to agent caching, auth state, and non-determinism. Recommended: `maxRetries: 0` (default) — one run per actor call ## Checkpoint syntax @@ -368,7 +367,7 @@ Use `maxBudgetUsd` to cap spending. The budget is a soft limit — checked betwe - Start with simple `contains:` checks to verify basic functionality, then add LLM judge for quality - Use `maxTurns: 3` for simple questions, `maxTurns: 10+` for complex multi-step tasks -- Set `abortOnFailure: true` when tests build on each other (test 2 depends on test 1). Either in the scenario YAML frontmatter, or via the top-level `abortOnFailure` input — the input wins when both are set. +- Set `abortOnFailure: true` on the top-level runner input when tests build on each other (test 2 depends on test 1). This is input-only — any `abortOnFailure:` key in the scenario YAML frontmatter is ignored. - Use `script:` checkpoints to verify side effects (files created, API state changed) - The Custom Init Script can install tools, download validators, or set up test fixtures - If the agent runs an Apify Actor, its dataset is automatically downloaded into `eval-datasets/.json` in the workspace — your script checks and the LLM judge can read it directly diff --git a/actors/runner/src/main.ts b/actors/runner/src/main.ts index 6a44028..2260bd4 100644 --- a/actors/runner/src/main.ts +++ b/actors/runner/src/main.ts @@ -136,15 +136,10 @@ const agent = input.agent ?? 'claude-code'; const maxRetries = input.maxRetries ?? 0; const maxTurns = input.maxTurns ?? 10; -// Resolve abortOnFailure with precedence: input (top-level) > scenario YAML frontmatter. -// parseScenario already fills meta.abortOnFailure with `false` when the YAML omits it, -// so meta.abortOnFailure is always a concrete boolean. Use `??` (not `||`) so an explicit -// `false` from input still overrides `true` from the YAML. -const abortOnFailure = input.abortOnFailure ?? meta.abortOnFailure; -// Treat null and undefined the same for source labeling — a caller that -// sends `{"abortOnFailure": null}` explicitly meant "defer to scenario", -// same as omitting the field. `!= null` matches both `null` and `undefined`. -const abortOnFailureSource = input.abortOnFailure != null ? 'input' : 'scenario'; +// abortOnFailure is now driven exclusively by the runner input. +// Any `abortOnFailure:` key in a scenario's YAML frontmatter is ignored. +// Default to false when not explicitly set on the input. +const abortOnFailure = input.abortOnFailure ?? false; const preset = (input.initPreset ?? 'none') as PresetName; @@ -157,10 +152,7 @@ const scenarioSpan = startScenarioSpan(tracer, { initPreset: preset, }); -log.info(`Scenario "${meta.name}": ${tests.length} test(s), abortOnFailure=${abortOnFailure} (source: ${abortOnFailureSource})`); -if (input.abortOnFailure != null && input.abortOnFailure !== meta.abortOnFailure) { - log.info(`Override: input.abortOnFailure=${input.abortOnFailure} takes precedence over scenario YAML value=${meta.abortOnFailure}`); -} +log.info(`Scenario "${meta.name}": ${tests.length} test(s), abortOnFailure=${abortOnFailure}`); if (parseWarnings) { for (const w of parseWarnings) log.warning(`[parse] ${w}`); } diff --git a/actors/runner/test/e2e/run-e2e.ts b/actors/runner/test/e2e/run-e2e.ts index d845770..1920a71 100644 --- a/actors/runner/test/e2e/run-e2e.ts +++ b/actors/runner/test/e2e/run-e2e.ts @@ -269,26 +269,6 @@ const tests: TestDef[] = [ }; }, }, - { - name: 'us5-abort-on-failure', - scenarioFile: 'us5-abort-on-failure.md', - input: { - scenario: readScenario('us5-abort-on-failure.md'), - maxTurns: 5, - maxBudgetUsd: 0.50, - }, - checks: (results) => { - if (results.length !== 1) return { - pass: false, - details: `Expected 1 result (step 2 should NOT run), got ${results.length}`, - }; - const v = (results[0] as any).overallVerdict; - return { - pass: v === 'fail', - details: `Step 1 failed (${v}), step 2 was correctly skipped (only 1 result in dataset)`, - }; - }, - }, { name: 'us5-dependent-steps', scenarioFile: 'us5-dependent-steps.md', diff --git a/docs/04-implementation-log.md b/docs/04-implementation-log.md index 4e3336a..70f0b88 100644 --- a/docs/04-implementation-log.md +++ b/docs/04-implementation-log.md @@ -119,7 +119,6 @@ - `us1-agent-fails`: judge správně detekuje selhání agenta (neexistující soubor) — PASS - `us1-partial-answer`: nuanced eval (5 zemí + math 17×23) — PASS - `us1-borderline-judge`: hraniční checkpointy (closure, tomato paradox) — PASS, confidence 0.98-0.99 -- `us5-abort-on-failure`: krok 1 fail → krok 2 přeskočen (1 result v datasetu) — PASS - `us5-dependent-steps`: 3-step řetěz (random → reverse → concat) — PASS - `us6-multi-secret`: 2 secrets maskované i v tool args + error messages — PASS - `us7-actual-abort`: budget $0.01 → error, run zastaveno — PASS diff --git a/docs/reference.html b/docs/reference.html index e494a16..71527d8 100644 --- a/docs/reference.html +++ b/docs/reference.html @@ -498,7 +498,7 @@

Struktura

---
 name: scenario-id        # required
 description: "Co testuje"
-abortOnFailure: false   # stop on first fail
+# abortOnFailure is input-only; scenario YAML is ignored
 ---
 
 ## Test
@@ -526,7 +526,6 @@ 
YAML frontmatter

     
         name requiredstring—Identifikátor scénáře
         descriptionstring""Co scénář testuje
-        abortOnFailurebooleanfalseZastaví scénář po prvním failu
     
 
 
@@ -967,9 +966,8 @@ 
Example scenáře

         us1-complex-tool-use2contains llmMulti-tool operace
         us1-agent-fails1llmAgent nemůže splnit úkol — judge musí detekovat
         us1-partial-answer2llmČástečně správná odpověď
-        us5-multi-step2containsMulti-step s abortOnFailure
+        us5-multi-step2containsMulti-step scénář s 2 nezávislými testy
         us5-dependent-steps3regex llmZávislé kroky (soubory z předchozího)
-        us5-abort-on-failure2llm containsTest 1 failne, test 2 se nesmí spustit
         us6-env-vars1containsInjection + maskování env vars
         us6-multi-secret1llmVíce secrets + error messages
         us7-budget-abort1llmBudget abort s nízkým limitem
@@ -1070,7 +1068,6 @@ 
TypeScript typy

 
interface ScenarioMeta {
     name: string;
     description: string;
-    abortOnFailure: boolean;
 }
 
 interface TestCase {
diff --git a/scenarios/actor-dev/cheerio-scraper.md b/scenarios/actor-dev/cheerio-scraper.md
index 9042f6e..d07c12a 100644
--- a/scenarios/actor-dev/cheerio-scraper.md
+++ b/scenarios/actor-dev/cheerio-scraper.md
@@ -1,7 +1,6 @@
 ---
 name: actor-dev-cheerio-scraper
 description: Agent creates a CheerioCrawler-based Apify Actor that scrapes product data
-abortOnFailure: true
 expectedTools:
   required: [Bash, Write]
   optional: [Read, Edit, Glob, Grep]
diff --git a/scenarios/ai-nondeterministic.md b/scenarios/ai-nondeterministic.md
index a5f63b7..a89d0ca 100644
--- a/scenarios/ai-nondeterministic.md
+++ b/scenarios/ai-nondeterministic.md
@@ -1,7 +1,6 @@
 ---
 name: ai-nondeterministic
 description: "Non-deterministic AI test: tasks where agent can take different paths"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/checkpoint-syntax-demo.md b/scenarios/checkpoint-syntax-demo.md
index b279519..7690892 100644
--- a/scenarios/checkpoint-syntax-demo.md
+++ b/scenarios/checkpoint-syntax-demo.md
@@ -1,7 +1,6 @@
 ---
 name: checkpoint-syntax-demo
 description: Demonstrates every checkpoint syntax variant — use as a reference
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/jq-tool-checks.md b/scenarios/jq-tool-checks.md
index 292664d..e617d4e 100644
--- a/scenarios/jq-tool-checks.md
+++ b/scenarios/jq-tool-checks.md
@@ -1,7 +1,6 @@
 ---
 name: jq-tool-checks-demo
 description: Demonstrates jq checkpoint type for validating agent tool usage and trajectory
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/mcpc-preset-test.md b/scenarios/mcpc-preset-test.md
index 0d382c1..fcc5266 100644
--- a/scenarios/mcpc-preset-test.md
+++ b/scenarios/mcpc-preset-test.md
@@ -1,7 +1,6 @@
 ---
 name: mcpc-preset-test
 description: Test mcpc preset with real MCP server
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/multi-check-demo.md b/scenarios/multi-check-demo.md
index d679897..54467ac 100644
--- a/scenarios/multi-check-demo.md
+++ b/scenarios/multi-check-demo.md
@@ -1,7 +1,6 @@
 ---
 name: multi-check-demo
 description: Demonstrates all checkpoint types in one scenario
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/script-validation.md b/scenarios/script-validation.md
index ae5108d..c7c7700 100644
--- a/scenarios/script-validation.md
+++ b/scenarios/script-validation.md
@@ -1,7 +1,6 @@
 ---
 name: script-validation
 description: Tests script checkpoint with downloaded validator
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/security-isolation.md b/scenarios/security-isolation.md
index da03f97..0f2136a 100644
--- a/scenarios/security-isolation.md
+++ b/scenarios/security-isolation.md
@@ -1,7 +1,6 @@
 ---
 name: security-isolation
 description: Verify agent cannot modify runner files and works only in workspace
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/skill-injection.md b/scenarios/skill-injection.md
index d5ce01f..2ca62ff 100644
--- a/scenarios/skill-injection.md
+++ b/scenarios/skill-injection.md
@@ -1,7 +1,6 @@
 ---
 name: skill-injection-test
 description: Test that init script can inject CLAUDE.md rules into workspace and agent follows them
-abortOnFailure: true
 ---
 
 ## Test
diff --git a/scenarios/smoke-test.md b/scenarios/smoke-test.md
index e5bddaa..c9883cd 100644
--- a/scenarios/smoke-test.md
+++ b/scenarios/smoke-test.md
@@ -1,7 +1,6 @@
 ---
 name: smoke-test
 description: Basic smoke test — simple question with checkpoint validation
-abortOnFailure: true
 ---
 
 ## Test
diff --git a/scenarios/trajectory-test.md b/scenarios/trajectory-test.md
index 2c501a0..5eb52d7 100644
--- a/scenarios/trajectory-test.md
+++ b/scenarios/trajectory-test.md
@@ -1,7 +1,6 @@
 ---
 name: trajectory-test
 description: Tests tool usage trajectory capture - agent must use multiple tools
-abortOnFailure: true
 ---
 
 ## Test
diff --git a/scenarios/us1-agent-fails.md b/scenarios/us1-agent-fails.md
index a42a002..f4a937b 100644
--- a/scenarios/us1-agent-fails.md
+++ b/scenarios/us1-agent-fails.md
@@ -1,7 +1,6 @@
 ---
 name: us1-agent-fails
 description: "US1 deep: Agent cannot fulfill the task — judge should detect failure"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us1-ai-judge.md b/scenarios/us1-ai-judge.md
index 8159199..9ce9c3d 100644
--- a/scenarios/us1-ai-judge.md
+++ b/scenarios/us1-ai-judge.md
@@ -1,7 +1,6 @@
 ---
 name: us1-ai-judge
 description: "US1 AI test: LLM judge evaluates nuanced answer"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us1-borderline-judge.md b/scenarios/us1-borderline-judge.md
index a18375c..d3e150f 100644
--- a/scenarios/us1-borderline-judge.md
+++ b/scenarios/us1-borderline-judge.md
@@ -1,7 +1,6 @@
 ---
 name: us1-borderline-judge
 description: "US1 deep: Borderline checkpoints that test judge accuracy"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us1-complex-tool-use.md b/scenarios/us1-complex-tool-use.md
index 7421540..496ef52 100644
--- a/scenarios/us1-complex-tool-use.md
+++ b/scenarios/us1-complex-tool-use.md
@@ -1,7 +1,6 @@
 ---
 name: us1-complex-tool-use
 description: "US1 deep: Agent must use multiple tools to solve a task"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us1-partial-answer.md b/scenarios/us1-partial-answer.md
index cb9ef2e..c760372 100644
--- a/scenarios/us1-partial-answer.md
+++ b/scenarios/us1-partial-answer.md
@@ -1,7 +1,6 @@
 ---
 name: us1-partial-answer
 description: "US1 deep: Agent gives partially correct answer — judge must detect"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us5-abort-on-failure.md b/scenarios/us5-abort-on-failure.md
deleted file mode 100644
index 8e25f93..0000000
--- a/scenarios/us5-abort-on-failure.md
+++ /dev/null
@@ -1,19 +0,0 @@
----
-name: us5-abort-on-failure
-description: "US5 deep: Step 1 fails, step 2 must NOT run (abortOnFailure=true)"
-abortOnFailure: true
----
-
-## Test
-Read the file /tmp/nonexistent-eval-file-abc.txt and report its exact contents.
-
-## Checkpoint
-The file contents must be exactly "this is the secret content".
-
----
-
-## Test
-Write "step2-was-reached" to /tmp/eval-us5-step2-marker.txt
-
-## Checkpoint
-contains: step2-was-reached
diff --git a/scenarios/us5-dependent-steps.md b/scenarios/us5-dependent-steps.md
index c2c8469..85d14c4 100644
--- a/scenarios/us5-dependent-steps.md
+++ b/scenarios/us5-dependent-steps.md
@@ -1,7 +1,6 @@
 ---
 name: us5-dependent-steps
 description: "US5 deep: Steps that depend on each other"
-abortOnFailure: true
 ---
 
 ## Test
diff --git a/scenarios/us5-multi-step.md b/scenarios/us5-multi-step.md
index 2f69748..77579da 100644
--- a/scenarios/us5-multi-step.md
+++ b/scenarios/us5-multi-step.md
@@ -1,7 +1,6 @@
 ---
 name: us5-multi-step
-description: "US5: Multi-step scenario with 2 tests and abortOnFailure"
-abortOnFailure: true
+description: "US5: Multi-step scenario with 2 tests"
 ---
 
 ## Test
diff --git a/scenarios/us6-env-vars.md b/scenarios/us6-env-vars.md
index 3ca4706..1b85179 100644
--- a/scenarios/us6-env-vars.md
+++ b/scenarios/us6-env-vars.md
@@ -1,7 +1,6 @@
 ---
 name: us6-env-vars
 description: "US6: Verify env var injection and masking"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us6-multi-secret.md b/scenarios/us6-multi-secret.md
index 521e41a..a101530 100644
--- a/scenarios/us6-multi-secret.md
+++ b/scenarios/us6-multi-secret.md
@@ -1,7 +1,6 @@
 ---
 name: us6-multi-secret
 description: "US6 deep: Multiple secrets, tool call args, error messages"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us7-actual-abort.md b/scenarios/us7-actual-abort.md
index 228ee34..726473c 100644
--- a/scenarios/us7-actual-abort.md
+++ b/scenarios/us7-actual-abort.md
@@ -1,7 +1,6 @@
 ---
 name: us7-actual-abort
 description: "US7 deep: Budget so low that run MUST be aborted mid-execution"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/scenarios/us7-budget-abort.md b/scenarios/us7-budget-abort.md
index f683b9b..8f081ed 100644
--- a/scenarios/us7-budget-abort.md
+++ b/scenarios/us7-budget-abort.md
@@ -1,7 +1,6 @@
 ---
 name: us7-budget-abort
 description: "US7: Verify budget abort with very low limit"
-abortOnFailure: false
 ---
 
 ## Test
diff --git a/shared/src/__tests__/integration-scenario.test.ts b/shared/src/__tests__/integration-scenario.test.ts
index 186f567..8db65f2 100644
--- a/shared/src/__tests__/integration-scenario.test.ts
+++ b/shared/src/__tests__/integration-scenario.test.ts
@@ -89,7 +89,6 @@ describe('integration: trajectory-test scenario', () => {
 
     it('parses correctly', () => {
         expect(meta.name).toBe('trajectory-test');
-        expect(meta.abortOnFailure).toBe(true);
         expect(tests).toHaveLength(2);
     });
 
diff --git a/shared/src/__tests__/scenario-parser.test.ts b/shared/src/__tests__/scenario-parser.test.ts
index 4f23d49..5d42aed 100644
--- a/shared/src/__tests__/scenario-parser.test.ts
+++ b/shared/src/__tests__/scenario-parser.test.ts
@@ -23,7 +23,6 @@ Return tool call count.
 
         expect(result.meta.name).toBe('simple');
         expect(result.meta.description).toBe('A simple test');
-        expect(result.meta.abortOnFailure).toBe(false);
         expect(result.tests).toHaveLength(1);
         expect(result.tests[0].test).toBe('What is 2+2?');
         expect(result.tests[0].checkpoint).toBe('The answer is 4.');
@@ -68,7 +67,6 @@ Return final JSON.
         const result = parseScenario(md);
 
         expect(result.meta.name).toBe('multi-step');
-        expect(result.meta.abortOnFailure).toBe(true);
         expect(result.tests).toHaveLength(3);
 
         expect(result.tests[0].monitor).toBe('Return JSON.');
@@ -90,7 +88,6 @@ It was done.
         const result = parseScenario(md);
 
         expect(result.meta.description).toBe('');
-        expect(result.meta.abortOnFailure).toBe(false);
         expect(result.tests).toHaveLength(1);
         expect(result.tests[0].monitor).toBeNull();
     });
diff --git a/shared/src/scenario-parser.ts b/shared/src/scenario-parser.ts
index 4ac07b6..858e8e8 100644
--- a/shared/src/scenario-parser.ts
+++ b/shared/src/scenario-parser.ts
@@ -16,10 +16,12 @@ export function parseScenario(markdown: string): ParsedScenario {
 
     const { data, content } = matter(markdown);
 
+    // Note: `abortOnFailure:` in the YAML frontmatter is accepted for
+    // backward compatibility but silently discarded — the runner input is the
+    // single source of truth (see actors/runner/src/main.ts).
     const meta: ScenarioMeta = {
         name: data.name ?? 'unnamed',
         description: data.description ?? '',
-        abortOnFailure: data.abortOnFailure ?? false,
     };
 
     if (!meta.name || meta.name === 'unnamed') {
diff --git a/shared/src/types.ts b/shared/src/types.ts
index 573aea8..45598e2 100644
--- a/shared/src/types.ts
+++ b/shared/src/types.ts
@@ -1,7 +1,6 @@
 export interface ScenarioMeta {
     name: string;
     description: string;
-    abortOnFailure: boolean;
 }
 
 /**