From 56125e7bbb58f65d04d04c0d55b762e5f87f2059 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Sun, 30 Aug 2026 23:06:45 +0000 Subject: [PATCH 1/4] feat: add FableLoom autopilot self-improvement --- .../fableloom/LoomEditorialAutomation.jsx | 40 ++++ .../LoomEditorialAutomation.test.jsx | 41 ++++ client/src/services/README.md | 2 +- client/src/services/apiFableLoom.test.js | 4 +- data.reference/prompts/stage-config.json | 7 + .../fableloom-editorial-self-improve.md | 83 ++++++++ docs/features/fableloom.md | 7 + ...-fableloom-editorial-self-improve-stage.js | 5 + ...eloom-editorial-self-improve-stage.test.js | 12 ++ server/lib/fableLoomValidation.js | 4 + .../lib/promptStageCallSites.generated.json | 3 + server/routes/fableLoom.test.js | 10 +- server/services/fableLoom/README.md | 1 + .../services/fableLoom/editorialAutopilot.js | 97 ++++++--- .../fableLoom/editorialAutopilot.test.js | 53 ++++- .../fableLoom/editorialSelfImprove.js | 172 ++++++++++++++++ .../fableLoom/editorialSelfImprove.test.js | 185 ++++++++++++++++++ server/services/fableLoom/index.js | 8 + 18 files changed, 702 insertions(+), 32 deletions(-) create mode 100644 data.reference/prompts/stages/fableloom-editorial-self-improve.md create mode 100644 scripts/migrations/322-fableloom-editorial-self-improve-stage.js create mode 100644 scripts/migrations/322-fableloom-editorial-self-improve-stage.test.js create mode 100644 server/services/fableLoom/editorialSelfImprove.js create mode 100644 server/services/fableLoom/editorialSelfImprove.test.js diff --git a/client/src/components/fableloom/LoomEditorialAutomation.jsx b/client/src/components/fableloom/LoomEditorialAutomation.jsx index 8434ef86b..b33a76067 100644 --- a/client/src/components/fableloom/LoomEditorialAutomation.jsx +++ b/client/src/components/fableloom/LoomEditorialAutomation.jsx @@ -81,6 +81,7 @@ function FindingLink({ loom, finding }) { export default function LoomEditorialAutomation({ loom, dirty, onLoomUpdate }) { const [route, setRoute] = useState({ providerId: '', model: '', effort: '' }); const [maxRounds, setMaxRounds] = useState(3); + const [selfImprove, setSelfImprove] = useState(false); const [result, setResult] = useState(null); const [autopilotRun, setAutopilotRun] = useState(null); const handledTerminalRunRef = useRef(null); @@ -165,6 +166,7 @@ export default function LoomEditorialAutomation({ loom, dirty, onLoomUpdate }) { const run = await startLoomEditorialAutopilot(loom.id, { ...routeBody, maxRounds, + ...(selfImprove ? { selfImprove: true } : {}), }, { silent: true }); handledTerminalRunRef.current = null; setAutopilotRun(run); @@ -262,6 +264,25 @@ export default function LoomEditorialAutomation({ loom, dirty, onLoomUpdate }) {

) : null} +
+ +
+
) : null} diff --git a/client/src/components/fableloom/LoomEditorialAutomation.test.jsx b/client/src/components/fableloom/LoomEditorialAutomation.test.jsx index 6239d3e15..7b8458cbc 100644 --- a/client/src/components/fableloom/LoomEditorialAutomation.test.jsx +++ b/client/src/components/fableloom/LoomEditorialAutomation.test.jsx @@ -127,6 +127,47 @@ describe('LoomEditorialAutomation', () => { expect(screen.getAllByText(/Round 1: evaluating and remediating/).length).toBeGreaterThan(0); }); + it('opts a run into approval-gated FableLoom workflow diagnosis', async () => { + const user = userEvent.setup(); + api.startLoomEditorialAutopilot.mockResolvedValue({ + id: 'editorial-run-1', loomId: 'loom-1', status: 'running', round: 0, maxRounds: 3, + message: 'Starting FableLoom editorial autopilot…', rounds: [], residualFindings: [], + }); + renderPanel(); + + const toggle = await screen.findByLabelText(/improve fableloom itself/i); + expect(toggle).not.toBeChecked(); + await user.click(toggle); + expect(screen.getByText(/queues a deduplicated worktree \+ PR CoS task in the approval queue/i)).toBeInTheDocument(); + await user.click(screen.getByRole('button', { name: 'Start editor autopilot' })); + + await waitFor(() => expect(api.startLoomEditorialAutopilot).toHaveBeenCalledWith( + 'loom-1', { maxRounds: 3, selfImprove: true }, { silent: true }, + )); + }); + + it('links a filed workflow diagnosis to its CoS approval task', async () => { + api.getLoom.mockResolvedValue(loom); + api.getLoomEditorialAutopilotStatus.mockResolvedValue({ + run: { + id: 'editorial-run-1', loomId: 'loom-1', status: 'paused', round: 2, maxRounds: 3, + pauseReason: 'plateau', message: 'Editorial autopilot paused.', rounds: [], + residualFindings: [], + selfImprove: { + verdict: 'pipeline', area: 'prompt', title: 'Tighten the remediation contract', + taskId: 'sys-example', filed: true, duplicate: false, + }, + }, + }); + renderPanel(); + + expect(await screen.findByText(/Queued a FableLoom improvement \(prompt\)/)).toBeInTheDocument(); + expect(screen.getByRole('link', { name: 'Review CoS task' })).toHaveAttribute( + 'href', + '/cos/tasks?task=sys-example&source=internal', + ); + }); + it('blocks every mutating AI action while the series plan has unsaved edits', async () => { renderPanel({ dirty: true }); diff --git a/client/src/services/README.md b/client/src/services/README.md index 97c7caeb8..8d0ec359b 100644 --- a/client/src/services/README.md +++ b/client/src/services/README.md @@ -103,7 +103,7 @@ toasts on throw). **Custom catch ⇒ `silent: true`** — otherwise toasts fire | `apiMediaJobs.js` | Media generation job tracking + `refineMediaPrompt` / `promptFromMedia` (vision reverse-prompt). | | `apiCreativeDirector.js` | Creative Director (video production). | | `apiCreativeCommission.js` | Creative Commissions (Autonomous Creation Engine — standing recurring briefs). | -| `apiFableLoom.js` | FableLoom branching narratives — loom/episode/scene-node/transition CRUD, deterministic graph validation, and the AI lanes (weave, branch, review, play turns, per-episode scene reformat). | +| `apiFableLoom.js` | FableLoom branching narratives — loom/episode/scene-node/transition CRUD, deterministic graph validation, AI authoring lanes, and the bounded editorial/playthrough autopilot lifecycle. | | `apiGames.js` | Game studio records, managed-app binding, reusable sprite/music bindings, deterministic asset-bundle compilation/integrity preflight, and AI feedback history. | | `apiMusicVideo.js` | Music Video projects + scene board + audio analysis. | | `apiSprites.js` | Sprite Manager records, asset library, production-set import (#2895), reference workflow: create/generate/lock (#2896), directional walk and per-track generation/approval, animation-type definition CRUD (#3153), trim/postprocess, and per-run source-frame listing for the Loop Trimmer's re-derive (#2980), and animation render-provider readiness (#4876). | diff --git a/client/src/services/apiFableLoom.test.js b/client/src/services/apiFableLoom.test.js index 83d156a6f..2ea59c852 100644 --- a/client/src/services/apiFableLoom.test.js +++ b/client/src/services/apiFableLoom.test.js @@ -70,9 +70,9 @@ describe('apiFableLoom', () => { method: 'POST', body: JSON.stringify({ aiReview: true }), }); - await api.startLoomEditorialAutopilot('loom-1', { maxRounds: 3 }); + await api.startLoomEditorialAutopilot('loom-1', { maxRounds: 3, selfImprove: true }); expect(request).toHaveBeenCalledWith('/fableloom/loom-1/editorial/autopilot/start', { - method: 'POST', body: JSON.stringify({ maxRounds: 3 }), + method: 'POST', body: JSON.stringify({ maxRounds: 3, selfImprove: true }), }); await api.getLoomEditorialAutopilotStatus('loom-1', { silent: true }); diff --git a/data.reference/prompts/stage-config.json b/data.reference/prompts/stage-config.json index 7ee03049f..7ee8d4838 100644 --- a/data.reference/prompts/stage-config.json +++ b/data.reference/prompts/stage-config.json @@ -1080,6 +1080,13 @@ "returnsJson": true, "variables": [] }, + "fableloom-editorial-self-improve": { + "name": "FableLoom — Editorial Autopilot Self-Improvement", + "description": "Opt-in, content-free post-mortem for a paused or failed FableLoom editor/reviewer loop. Distinguishes unfinished story work from an inefficient, broken, or incomplete PortOS workflow and returns a bounded software diagnosis; a confident PortOS verdict queues a deduplicated, approval-gated CoS task.", + "model": "heavy", + "returnsJson": true, + "variables": [] + }, "fableloom-review-playthroughs": { "name": "FableLoom — Review Playthrough Variations", "description": "Reviews deterministically enumerated playthrough variations for mechanics, causal branch coherence, audience agency, continuity, pacing, ending payoff, canon, and ship-quality storytelling.", diff --git a/data.reference/prompts/stages/fableloom-editorial-self-improve.md b/data.reference/prompts/stages/fableloom-editorial-self-improve.md new file mode 100644 index 000000000..a0c0c4575 --- /dev/null +++ b/data.reference/prompts/stages/fableloom-editorial-self-improve.md @@ -0,0 +1,83 @@ +# FableLoom Editorial Autopilot — Self-Improvement Diagnosis + +You are a **systems engineer** doing a post-mortem on PortOS's automated FableLoom editor/reviewer loop. Decide whether the run's trouble came from the **story** (the interactive series still needs editorial work, while the automation behaved correctly) or from **PortOS** (its workflow, prompt, validation, runner, configuration, or UI is broken, wasteful, or unable to apply the repair it claims to perform). + +You are NOT reviewing the story. You receive no story prose, titles, record ids, or finding text. Do not invent or request them. Your output may become a CoS coding-task brief and a public pull request, so it must describe **software behavior only**. + +## The workflow + +Each bounded round performs these steps in order: + +1. `evaluate-remediate` — inspect the complete existing series and return a safe sparse patch. It may repair series-plan fields, episode outlines, existing scene metadata, and existing transition labels/targets, but it must preserve episode, scene, and transition membership and ids. PortOS rejects invalid output and graph/outline/playthrough/continuity regressions before saving. +2. `playthrough-review` — deterministically enumerate bounded path variations, run structural and continuity diagnostics, and ask an independent story editor for a quality verdict. +3. If findings remain, translate them into guidance for the next remediation pass. Stop when the story passes, the same findings survive a no-change pass (`plateau`), the selected round limit is reached, the user cancels, or a step throws. + +The post-mortem is opt-in and runs only after a pause or failure. A confident `pipeline` verdict (PortOS is at fault) queues a deduplicated, worktree-isolated, PR-opening CoS task. The task waits for human approval before an agent starts. Reporting `content` or `none` is useful and files nothing. + +## The run + +- **Outcome:** `{{outcome}}` +- **Reason:** `{{outcomeReason}}` +- **Step active when the run stopped:** `{{currentStep}}` +- **Error code:** `{{errorCode}}` +- **Round:** {{round}} / {{maxRounds}} +- **Playthrough path cap:** {{maxPaths}} + +Content-free round telemetry: + +```json +{{telemetryJson}} +``` + +Each round reports only counters and booleans: + +- `remediation.changed` / `changeCount` — whether a sparse patch actually changed the loom and how many change notes the stage returned. +- `before` / `after` — deterministic blocker counts around the accepted patch. +- `evaluationFindingCount` — how many issues the remediation stage itself identified. +- `diagnosticsPassed` / `diagnosticFindingCount` — whether non-narrative editorial diagnostics cleared. +- `deterministicPassed` / `deterministicComplete` / `stats` — path enumeration health and coverage. +- `reviewPassed` / `qualityScore` / finding counts by category and severity — the independent quality verdict, without any story text. + +## What counts as a PORTOS problem + +Return `pipeline` only when the counters, terminal reason, or error code support a concrete software change. Strong cases include: + +1. **An ineffective loop step.** Remediation repeatedly reports no applicable change while its own findings remain, accepted edits do not reduce the relevant deterministic blockers, or next-round guidance cannot target what the review produced. +2. **A broken output contract or validator.** A stage returns unusable data, claims edits that do not apply, produces a shape its consumer cannot read, or a safe intended repair is rejected for a contract PortOS should support. +3. **A missing or late gate.** Expensive full-series editing discovers a structural condition that a cheaper outline/graph preflight could have surfaced before expansion. +4. **A runner or lifecycle defect.** The wrong provider route is used, cancellation/status handling is inconsistent, a failure is swallowed, or a terminal run is left active. +5. **A bad bound or missing control.** A round/path limit makes the workflow unable to do its job, or the editor needs a specific user-facing option that does not exist. +6. **A wasteful ordering.** The workflow performs costly review before required deterministic evidence is available, or repeats work that an earlier retained result should have made unnecessary. + +## What is NOT a PortOS problem + +- A story genuinely needs another authored choice, consequence, scene, or ending, and the loop correctly pauses instead of changing graph membership → `content`. +- The round limit is reached while distinct story findings continue to improve from round to round → `content`. +- A plateau alone, without counters showing an automation mismatch → `content` or `none`; do not assume every hard editorial problem is a software bug. +- A one-off provider or network failure with no software evidence → `none`. +- Any theory that requires story text or source inspection not present here → `none` with low confidence. + +## Output contract + +Return ONLY valid JSON matching this shape — no prose, no markdown fence, no commentary. + +```json +{ + "verdict": "pipeline", + "confidence": 0.0, + "area": "pipeline-step", + "title": "string — imperative, one line, names the software defect", + "problem": "string — what is wrong in PortOS and which counters/status values show it", + "evidence": ["string — a specific counter, status, reason, or error code from the telemetry above"], + "proposedChange": "string — the smallest concrete source/prompt/check/config/UI change that fixes it", + "risks": "string — behavior that must keep working" +} +``` + +- `verdict` must be `pipeline` (PortOS is at fault), `content` (the story needs work), or `none`. +- `confidence` is 0–1. Diagnoses below 0.6 are discarded; a guess should be discarded. +- `area` must be `editorial-check`, `pipeline-step`, `prompt`, `runner`, `config`, or `ui`. +- `evidence` has at most 8 entries and may cite only the supplied content-free telemetry. +- `proposedChange` must be actionable without access to this run or its story. + +When the story needs work or the evidence is inconclusive, return the same shape with `verdict` set to `content` or `none` and empty change fields. diff --git a/docs/features/fableloom.md b/docs/features/fableloom.md index a2e87d333..58d81dd64 100644 --- a/docs/features/fableloom.md +++ b/docs/features/fableloom.md @@ -98,6 +98,13 @@ scene-local clothes cannot silently reappear between episodes. are client-side state (restart is free; nothing persists server-side). - **Story settings drawer** — audience role and communication medium, scene format (plus the rewrite pass), and the narrator's provider/model/effort pin. +- **AI editor, reviewer & playtest** — a whole-series remediation pass and a + bounded autopilot that alternates safe edits with deterministic + narrative + path review. Its optional **Improve FableLoom itself** post-mortem runs only + after a pause or failure, sends content-free counters (never story records) + through one budget-gated diagnosis, and queues a deduplicated PortOS CoS task + when the workflow rather than the story is at fault. The task remains in the + approval queue; the per-run checkbox does not grant unattended source edits. - **Series detail page** (`/pipeline/series/:seriesId`) — a "Branching narratives" card lists the looms linked to that series (counts + a link into the editor) and spawns a new one pre-linked to the series and its universe. diff --git a/scripts/migrations/322-fableloom-editorial-self-improve-stage.js b/scripts/migrations/322-fableloom-editorial-self-improve-stage.js new file mode 100644 index 000000000..71ae6f881 --- /dev/null +++ b/scripts/migrations/322-fableloom-editorial-self-improve-stage.js @@ -0,0 +1,5 @@ +/** Seed the FableLoom editorial-autopilot self-improvement stage. */ + +import { makeSeedMigration } from './_seedStageHelpers.js'; + +export default makeSeedMigration('fableloom-editorial-self-improve'); diff --git a/scripts/migrations/322-fableloom-editorial-self-improve-stage.test.js b/scripts/migrations/322-fableloom-editorial-self-improve-stage.test.js new file mode 100644 index 000000000..387a96e36 --- /dev/null +++ b/scripts/migrations/322-fableloom-editorial-self-improve-stage.test.js @@ -0,0 +1,12 @@ +import { describe } from 'vitest'; + +import migration from './322-fableloom-editorial-self-improve-stage.js'; +import { runSeedStageMigrationTests } from './_seedStageTestHelpers.js'; + +describe('migration 322 — seed the FableLoom editorial self-improvement stage', () => { + runSeedStageMigrationTests({ + migration, + stages: ['fableloom-editorial-self-improve'], + prefix: 'migration-322-', + }); +}); diff --git a/server/lib/fableLoomValidation.js b/server/lib/fableLoomValidation.js index d40e4ac08..c915302d4 100644 --- a/server/lib/fableLoomValidation.js +++ b/server/lib/fableLoomValidation.js @@ -358,6 +358,10 @@ export const playthroughReviewSchema = z.object({ export const editorialAutopilotStartSchema = z.object({ maxRounds: z.number().int().min(1).max(LOOM_LIMITS.EDITORIAL_AUTOPILOT_ROUNDS_MAX).optional(), maxPaths: z.number().int().min(1).max(FABLELOOM_PLAYTEST_LIMITS.MAX_PATHS).optional(), + // Opt-in post-mortem over content-free run counters. A confident PortOS + // verdict queues an approval-gated CoS task; healthy/canceled runs spend + // nothing, and the story itself never crosses into the task brief. + selfImprove: z.boolean().optional(), ...llmPickFields, }); diff --git a/server/lib/promptStageCallSites.generated.json b/server/lib/promptStageCallSites.generated.json index 5ba62f6c8..2d015aee5 100644 --- a/server/lib/promptStageCallSites.generated.json +++ b/server/lib/promptStageCallSites.generated.json @@ -37,6 +37,9 @@ "fableloom-editorial-remediate": [ "server/services/fableLoom/editorial.js" ], + "fableloom-editorial-self-improve": [ + "server/services/fableLoom/editorialSelfImprove.js" + ], "fableloom-feedback-episode": [ "server/services/fableLoom/weave.js" ], diff --git a/server/routes/fableLoom.test.js b/server/routes/fableLoom.test.js index e7f844e3c..78e5aaf0b 100644 --- a/server/routes/fableLoom.test.js +++ b/server/routes/fableLoom.test.js @@ -199,12 +199,18 @@ describe('FableLoom routes', () => { fableLoom.startFableLoomEditorialAutopilot.mockResolvedValueOnce(running); const started = await request(makeApp()) .post('/api/fableloom/loom-1/editorial/autopilot/start') - .send({ maxRounds: 3, maxPaths: 128, providerId: 'writer' }); + .send({ maxRounds: 3, maxPaths: 128, providerId: 'writer', selfImprove: true }); expect(started.status).toBe(202); expect(fableLoom.startFableLoomEditorialAutopilot).toHaveBeenCalledWith('loom-1', { - maxRounds: 3, maxPaths: 128, providerId: 'writer', + maxRounds: 3, maxPaths: 128, providerId: 'writer', selfImprove: true, }); + const invalid = await request(makeApp()) + .post('/api/fableloom/loom-1/editorial/autopilot/start') + .send({ selfImprove: 'yes' }); + expect(invalid.status).toBe(400); + expect(fableLoom.startFableLoomEditorialAutopilot).toHaveBeenCalledTimes(1); + fableLoom.getLoom.mockResolvedValueOnce({ id: 'loom-1' }); fableLoom.getLatestFableLoomEditorialAutopilot.mockReturnValueOnce(running); const status = await request(makeApp()) diff --git a/server/services/fableLoom/README.md b/server/services/fableLoom/README.md index 4dffde1c0..427b4cc11 100644 --- a/server/services/fableLoom/README.md +++ b/server/services/fableLoom/README.md @@ -12,6 +12,7 @@ intent to a transition and moves them through the graph until an ending. | `weave.js` | AI ops via `runStagedLLM`: `generateSeriesPlan` (full arc / plot-point / side-quest scaffold), `generateEpisodeOutline` + `validateEpisodeOutline` + `reviewEpisodeOutline` (log-line beat planning before teleplay expansion), `weaveEpisode` (single-camera-cut graph with automatic cuts and looping decisions), `branchNode` (grow paths), `feedbackEpisode` (apply a conversational sparse patch to one episode), `reviewEpisode` + `reviewSeriesTeleplay` (episode or complete-series critique with deterministic analysis), `playTurn` (reader intent → transition; tapped/automatic paths resolve with NO LLM call), `reformatEpisodeScenes` (rewrite ONE episode's scenes into another format; the loom's format pin lands only once every episode is converted). | | `editorial.js` | Whole-series AI evaluate-and-remediate pass plus deterministic/AI playthrough review. Preserves episode/scene/path membership and IDs, validates generated outlines, rejects graph regressions, and applies story-aware convergence sources. | | `editorialAutopilot.js` | User-triggered bounded editor/reviewer loop: remediate, exercise every bounded branch variation, judge story quality, then complete, pause on residuals/plateau, fail, or cancel cooperatively. | +| `editorialSelfImprove.js` | Opt-in, budget-gated post-mortem for paused/failed editorial-autopilot runs. Sends only content-free counters to a diagnostic stage and queues a deduplicated, approval-gated PortOS CoS task for confident workflow defects. | | `formats.js` | Scene formats (`prose` / `teleplay`) and the prompt contracts each generative stage renders for them. | | `hostedSession.js` | Scoped QR-hosted play session lifecycle, HTTPS readiness preflight, token hashing, live voice gate revalidation, and half-duplex turn taking (#5383). | | `production.js` | Episodic production orchestration: batch planning, DAG generation, cancellable batch runs, and user-triggered episodic continuity review (#5384). | diff --git a/server/services/fableLoom/editorialAutopilot.js b/server/services/fableLoom/editorialAutopilot.js index 4e99d7b35..cb969b16f 100644 --- a/server/services/fableLoom/editorialAutopilot.js +++ b/server/services/fableLoom/editorialAutopilot.js @@ -17,6 +17,10 @@ import { evaluateAndRemediateFableLoom, reviewFableLoomPlaythroughs, } from './editorial.js'; +import { + runFableLoomEditorialSelfImprove, + shouldDiagnoseFableLoomEditorial, +} from './editorialSelfImprove.js'; export const FABLELOOM_EDITORIAL_AUTOPILOT_LIMITS = Object.freeze({ DEFAULT_ROUNDS: 3, @@ -110,13 +114,49 @@ const finishCanceled = (run) => touch(run, { completedAt: nowIso(), }); -const finishFailed = (run, error) => touch(run, { - status: 'failed', - currentStep: null, - message: errorMessage(error), - error: errorMessage(error), - completedAt: nowIso(), -}); +const terminalDiagnosis = async (run, outcome, { reason = null, error = null } = {}) => { + if (!shouldDiagnoseFableLoomEditorial(run, outcome)) return null; + const sourceStep = run.currentStep; + touch(run, { + currentStep: 'self-improve', + message: 'Diagnosing whether the editorial automation itself should improve…', + }); + return runFableLoomEditorialSelfImprove(run, { + outcome, + reason, + sourceStep, + errorCode: error?.code || null, + }).catch((diagnosisError) => { + console.log(`⚠️ FableLoom self-improve diagnosis failed: ${diagnosisError.message}`); + return null; + }); +}; + +const finishPaused = async (run, pauseReason, message) => { + const selfImprove = await terminalDiagnosis(run, 'paused', { reason: pauseReason }); + if (run.cancelRequested) return finishCanceled(run); + return touch(run, { + status: 'paused', + pauseReason, + currentStep: null, + message, + selfImprove, + completedAt: nowIso(), + }); +}; + +const finishFailed = async (run, error) => { + const selfImprove = await terminalDiagnosis(run, 'failed', { reason: 'run-error', error }); + if (run.cancelRequested) return finishCanceled(run); + return touch(run, { + status: 'failed', + currentStep: null, + message: errorMessage(error), + error: errorMessage(error), + selfImprove, + completedAt: nowIso(), + }); +}; async function runRound(run, guidance) { const round = run.round + 1; @@ -174,22 +214,18 @@ async function runRound(run, guidance) { const plateau = !remediation.changed && signature === run.previousFindingSignature; run.previousFindingSignature = signature; if (plateau) { - return touch(run, { - status: 'paused', - pauseReason: 'plateau', - currentStep: null, - message: 'Editorial autopilot paused because another safe pass produced no changes and the same findings remained.', - completedAt: nowIso(), - }); + return finishPaused( + run, + 'plateau', + 'Editorial autopilot paused because another safe pass produced no changes and the same findings remained.', + ); } if (round >= run.maxRounds) { - return touch(run, { - status: 'paused', - pauseReason: 'round-limit', - currentStep: null, - message: `Editorial autopilot reached its ${run.maxRounds}-round limit with review findings still open.`, - completedAt: nowIso(), - }); + return finishPaused( + run, + 'round-limit', + `Editorial autopilot reached its ${run.maxRounds}-round limit with review findings still open.`, + ); } touch(run, { @@ -200,7 +236,7 @@ async function runRound(run, guidance) { /** Start and detach a bounded editor/reviewer run. */ export async function startFableLoomEditorialAutopilot(loomId, { - maxRounds, maxPaths, providerId, model, effort, + maxRounds, maxPaths, providerId, model, effort, selfImprove, } = {}) { cleanStaleRuns(); await requireLoomForRun(loomId); @@ -236,6 +272,8 @@ export async function startFableLoomEditorialAutopilot(loomId, { completedAt: null, cancelRequested: false, pauseReason: null, + selfImproveEnabled: selfImprove === true, + selfImprove: null, message: 'Starting FableLoom editorial autopilot…', error: null, rounds: [], @@ -247,9 +285,18 @@ export async function startFableLoomEditorialAutopilot(loomId, { }; runs.set(run.id, run); latestRunByLoom.set(loomId, run.id); - void runRound(run, '').catch((error) => ( - run.cancelRequested ? finishCanceled(run) : finishFailed(run, error) - )); + void runRound(run, '') + .catch((error) => (run.cancelRequested ? finishCanceled(run) : finishFailed(run, error))) + .catch((error) => { + console.error(`❌ FableLoom editorial autopilot terminal handling failed: ${error.message}`); + touch(run, { + status: 'failed', + currentStep: null, + message: errorMessage(error), + error: errorMessage(error), + completedAt: nowIso(), + }); + }); return run; } diff --git a/server/services/fableLoom/editorialAutopilot.test.js b/server/services/fableLoom/editorialAutopilot.test.js index 92fe7ae3c..0b2ef8801 100644 --- a/server/services/fableLoom/editorialAutopilot.test.js +++ b/server/services/fableLoom/editorialAutopilot.test.js @@ -3,12 +3,22 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; const getLoomMock = vi.hoisted(() => vi.fn(async (id) => ({ id }))); const remediateMock = vi.hoisted(() => vi.fn()); const playtestMock = vi.hoisted(() => vi.fn()); +const selfImproveMock = vi.hoisted(() => vi.fn(async () => ({ + verdict: 'pipeline', area: 'prompt', title: 'Tighten the remediation contract', + taskId: 'sys-example', filed: true, duplicate: false, +}))); vi.mock('./records.js', () => ({ getLoom: getLoomMock })); vi.mock('./editorial.js', () => ({ evaluateAndRemediateFableLoom: remediateMock, reviewFableLoomPlaythroughs: playtestMock, })); +vi.mock('./editorialSelfImprove.js', () => ({ + shouldDiagnoseFableLoomEditorial: (run, outcome) => ( + run?.selfImproveEnabled === true && ['paused', 'failed'].includes(outcome) + ), + runFableLoomEditorialSelfImprove: (...args) => selfImproveMock(...args), +})); const { _resetFableLoomEditorialAutopilots, @@ -62,6 +72,7 @@ beforeEach(() => { getLoomMock.mockClear().mockImplementation(async (id) => ({ id })); remediateMock.mockReset(); playtestMock.mockReset(); + selfImproveMock.mockClear(); }); describe('FableLoom editorial autopilot', () => { @@ -82,6 +93,7 @@ describe('FableLoom editorial autopilot', () => { expect(playtestMock).toHaveBeenCalledWith('loom-example', { providerId: 'writer', model: 'large', effort: 'high', aiReview: true, }); + expect(selfImproveMock).not.toHaveBeenCalled(); }); it('pauses on a plateau after the same finding survives a no-change pass', async () => { @@ -110,10 +122,47 @@ describe('FableLoom editorial autopilot', () => { findings: [{ severity: 'low', category: 'pacing', problem: 'The second route rushes its turn.' }], })); - const started = await startFableLoomEditorialAutopilot('loom-example', { maxRounds: 1 }); + const started = await startFableLoomEditorialAutopilot('loom-example', { + maxRounds: 1, + selfImprove: true, + }); + const finished = await waitForTerminal(started.id); + + expect(finished).toMatchObject({ + status: 'paused', + pauseReason: 'round-limit', + round: 1, + selfImproveEnabled: true, + selfImprove: { verdict: 'pipeline', taskId: 'sys-example', filed: true }, + }); + expect(selfImproveMock).toHaveBeenCalledWith(expect.objectContaining({ id: started.id }), { + outcome: 'paused', + reason: 'round-limit', + sourceStep: 'playthrough-review', + errorCode: null, + }); + }); + + it('keeps provider failures terminal while best-effort self-improvement diagnoses the failed step', async () => { + const providerError = Object.assign(new Error('Provider returned an unusable patch'), { + code: 'AI_RESPONSE_INVALID', + }); + remediateMock.mockRejectedValueOnce(providerError); + + const started = await startFableLoomEditorialAutopilot('loom-example', { selfImprove: true }); const finished = await waitForTerminal(started.id); - expect(finished).toMatchObject({ status: 'paused', pauseReason: 'round-limit', round: 1 }); + expect(finished).toMatchObject({ + status: 'failed', + error: providerError.message, + selfImprove: { taskId: 'sys-example' }, + }); + expect(selfImproveMock).toHaveBeenCalledWith(expect.any(Object), { + outcome: 'failed', + reason: 'run-error', + sourceStep: 'evaluate-remediate', + errorCode: 'AI_RESPONSE_INVALID', + }); }); it('keeps diagnostics-only blockers actionable in the residual findings', async () => { diff --git a/server/services/fableLoom/editorialSelfImprove.js b/server/services/fableLoom/editorialSelfImprove.js new file mode 100644 index 000000000..eaaf8ca3a --- /dev/null +++ b/server/services/fableLoom/editorialSelfImprove.js @@ -0,0 +1,172 @@ +/** + * FableLoom editorial-autopilot self-improvement (opt-in post-mortem). + * + * The editor/reviewer loop judges the STORY. This pass judges the AUTOMATION: + * after a paused or failed run, it reads a compact, content-free account of + * what each round did and asks whether PortOS itself is inefficient, broken, + * or missing a useful control. A confident PortOS verdict files one deduped, + * approval-gated CoS task; story problems file nothing. + * + * The task can lead to a public PR, so neither the prompt variables nor the + * task brief contain loom names, record ids, scene text, findings, or provider + * output. Only bounded counters, status vocabulary, and error codes cross this + * boundary. + */ + +import * as cosTaskStore from '../cosTaskStore.js'; +import { getDomainBudgetStatus, recordDomainUsage } from '../domainUsage.js'; +import { runStagedLLM } from '../stageRunner.js'; +import { + buildDiagnosisTask, + isActionableDiagnosis, + shapeDiagnosis, +} from '../pipeline/seriesAutopilot/diagnosisCore.js'; + +const SELF_IMPROVE_STAGE = 'fableloom-editorial-self-improve'; + +export const FABLELOOM_EDITORIAL_SELF_IMPROVE_AREAS = Object.freeze([ + 'editorial-check', 'pipeline-step', 'prompt', 'runner', 'config', 'ui', +]); + +export const FABLELOOM_EDITORIAL_SELF_IMPROVE_MIN_CONFIDENCE = 0.6; + +const countBy = (items, key) => { + const list = Array.isArray(items) ? items : []; + return Object.fromEntries( + [...new Set(list.map((item) => item?.[key]).filter(Boolean))] + .sort() + .map((value) => [value, list.filter((item) => item?.[key] === value).length]), + ); +}; + +const numericStats = (stats) => Object.fromEntries( + Object.entries(stats || {}).filter(([, value]) => Number.isFinite(value)), +); + +const safeToken = (value, fallback) => { + if (typeof value !== 'string' || !value.trim()) return fallback; + const token = value.trim(); + return /^[a-zA-Z0-9_.:-]{1,80}$/.test(token) ? token : fallback; +}; + +const compactRound = (round) => ({ + round: round.round, + remediation: { + changed: round.changed === true, + changeCount: Array.isArray(round.changes) ? round.changes.length : 0, + before: numericStats(round.before), + after: numericStats(round.after), + evaluationFindingCount: round.evaluation?.findings?.length || 0, + }, + playthrough: { + passed: round.passed === true, + diagnosticsPassed: round.diagnostics?.passed === true, + diagnosticFindingCount: round.diagnostics?.findings?.length || 0, + deterministicPassed: round.deterministic?.passed === true, + deterministicComplete: round.deterministic?.complete === true, + stats: numericStats(round.deterministic?.stats), + reviewPassed: round.review?.passed === true, + qualityScore: Number.isFinite(round.review?.qualityScore) ? round.review.qualityScore : null, + reviewFindingCount: round.review?.findings?.length || 0, + findingCategories: countBy(round.review?.findings, 'category'), + findingSeverities: countBy(round.review?.findings, 'severity'), + }, +}); + +/** A clean completion and a user cancellation do not warrant a diagnosis call. */ +export const shouldDiagnoseFableLoomEditorial = (run, outcome) => ( + run?.selfImproveEnabled === true && ['paused', 'failed'].includes(outcome) +); + +/** + * Content-free evidence sent to the diagnosis stage. Exported so the privacy + * boundary and the useful counters can be pinned in focused tests. + */ +export function buildFableLoomEditorialTelemetry(run, { + outcome, + reason = null, + sourceStep = null, + errorCode = null, +} = {}) { + return { + outcome: safeToken(outcome, 'unknown'), + reason: safeToken(reason, 'none'), + sourceStep: safeToken(sourceStep, 'unknown'), + errorCode: safeToken(errorCode, 'none'), + round: Number.isInteger(run?.round) ? run.round : 0, + maxRounds: Number.isInteger(run?.maxRounds) ? run.maxRounds : 0, + maxPaths: Number.isInteger(run?.maxPaths) ? run.maxPaths : 'default', + rounds: (Array.isArray(run?.rounds) ? run.rounds : []).map(compactRound), + }; +} + +export function buildFableLoomEditorialSelfImproveTask({ diagnosis, telemetry }) { + return buildDiagnosisTask({ + diagnosis, + descriptionPrefix: 'FableLoom editorial self-improvement', + leadLine: `FableLoom Editorial Autopilot diagnosed a PortOS automation defect: ${diagnosis.title}`, + tailLines: [ + `Diagnosed from a content-free autopilot summary after a \`${telemetry.outcome}\` run at round ${telemetry.round}/${telemetry.maxRounds} (step: \`${telemetry.sourceStep}\`, reason: \`${telemetry.reason}\`, error code: \`${telemetry.errorCode}\`).`, + 'Confirm the defect in the source before changing anything: this brief is one LLM\'s read of bounded run counters, not a reproduction.', + ], + approvalRequired: true, + }); +} + +const routeOptions = (run) => ({ + ...(run?.route?.providerId ? { providerOverride: run.route.providerId } : {}), + ...(run?.route?.model ? { modelOverride: run.route.model } : {}), + ...(run?.route?.effort ? { effortOverride: run.route.effort } : {}), +}); + +/** Run one best-effort terminal diagnosis and file a task for PortOS defects. */ +export async function runFableLoomEditorialSelfImprove(run, context = {}) { + if (!shouldDiagnoseFableLoomEditorial(run, context.outcome)) return null; + + const budget = await getDomainBudgetStatus('cos'); + if (!budget.withinBudget) return null; + + const telemetry = buildFableLoomEditorialTelemetry(run, context); + const { content } = await runStagedLLM(SELF_IMPROVE_STAGE, { + outcome: telemetry.outcome, + outcomeReason: telemetry.reason, + currentStep: telemetry.sourceStep, + errorCode: telemetry.errorCode, + round: telemetry.round, + maxRounds: telemetry.maxRounds, + maxPaths: telemetry.maxPaths, + telemetryJson: JSON.stringify(telemetry.rounds, null, 2), + }, { + ...routeOptions(run), + returnsJson: true, + source: SELF_IMPROVE_STAGE, + }); + await recordDomainUsage('cos', { actions: 1 }); + + // Cancellation remains cooperative even if it arrived while this final LLM + // call was running: do not file new work after the user asked the run to stop. + if (run.cancelRequested) return null; + + const diagnosis = shapeDiagnosis(content, FABLELOOM_EDITORIAL_SELF_IMPROVE_AREAS); + if (!isActionableDiagnosis(diagnosis, FABLELOOM_EDITORIAL_SELF_IMPROVE_MIN_CONFIDENCE)) { + console.log(`🔧 FableLoom self-improve: nothing filed (verdict=${diagnosis?.verdict || 'unreadable'} confidence=${diagnosis?.confidence ?? '—'})`); + return null; + } + + const task = buildFableLoomEditorialSelfImproveTask({ diagnosis, telemetry }); + const result = await cosTaskStore.addTask(task, 'internal') + .catch((error) => { + console.log(`⚠️ FableLoom self-improve task filing failed: ${error.message}`); + return null; + }); + const filed = !!result && !result.duplicate; + console.log(`🔧 FableLoom self-improve: area=${diagnosis.area} ${result ? (filed ? `filed ${result.id}` : 'duplicate of an open task') : 'filing failed'}`); + return { + verdict: diagnosis.verdict, + area: diagnosis.area, + title: diagnosis.title, + taskId: result?.id || null, + filed, + duplicate: !!result?.duplicate, + }; +} diff --git a/server/services/fableLoom/editorialSelfImprove.test.js b/server/services/fableLoom/editorialSelfImprove.test.js new file mode 100644 index 000000000..c2ff3f7d7 --- /dev/null +++ b/server/services/fableLoom/editorialSelfImprove.test.js @@ -0,0 +1,185 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const addTask = vi.hoisted(() => vi.fn(async () => ({ id: 'sys-example' }))); +const getDomainBudgetStatus = vi.hoisted(() => vi.fn(async () => ({ withinBudget: true }))); +const recordDomainUsage = vi.hoisted(() => vi.fn(async () => {})); +const runStagedLLM = vi.hoisted(() => vi.fn()); + +vi.mock('../cosTaskStore.js', () => ({ addTask: (...args) => addTask(...args) })); +vi.mock('../domainUsage.js', () => ({ + getDomainBudgetStatus: (...args) => getDomainBudgetStatus(...args), + recordDomainUsage: (...args) => recordDomainUsage(...args), +})); +vi.mock('../stageRunner.js', () => ({ runStagedLLM: (...args) => runStagedLLM(...args) })); + +const { + FABLELOOM_EDITORIAL_SELF_IMPROVE_MIN_CONFIDENCE, + buildFableLoomEditorialSelfImproveTask, + buildFableLoomEditorialTelemetry, + runFableLoomEditorialSelfImprove, + shouldDiagnoseFableLoomEditorial, +} = await import('./editorialSelfImprove.js'); +const { PORTOS_APP_ID } = await import('../../lib/appIdentity.js'); + +const goodDiagnosis = { + verdict: 'pipeline', + confidence: 0.9, + area: 'prompt', + title: 'Make remediation return applicable transition edits', + problem: 'The remediation stage reports findings but repeatedly produces no applicable patch.', + evidence: ['round 2 remediation.changed=false while evaluationFindingCount=3'], + proposedChange: 'Tighten the remediation prompt output contract around existing transition ids.', + risks: 'Preserve graph membership and reject unknown ids.', +}; + +const makeRun = (overrides = {}) => ({ + selfImproveEnabled: true, + cancelRequested: false, + currentStep: 'playthrough-review', + round: 2, + maxRounds: 3, + maxPaths: 64, + route: { providerId: 'writer', model: 'large', effort: 'high' }, + rounds: [{ + round: 1, + changed: false, + changes: [], + before: { graphErrors: 1, outlineErrors: 0 }, + after: { graphErrors: 1, outlineErrors: 0 }, + evaluation: { + summary: 'PRIVATE STORY SUMMARY', + findings: [{ problem: 'PRIVATE STORY FINDING' }], + }, + diagnostics: { + passed: false, + findings: [{ problem: 'PRIVATE DIAGNOSTIC FINDING' }], + }, + deterministic: { + passed: true, + complete: true, + stats: { variationCount: 3, transitionCount: 6, visitedTransitionCount: 6 }, + }, + review: { + passed: false, + qualityScore: 6.5, + summary: 'PRIVATE REVIEW SUMMARY', + findings: [{ severity: 'medium', category: 'choice', problem: 'PRIVATE REVIEW FINDING' }], + }, + passed: false, + }], + ...overrides, +}); + +beforeEach(() => { + vi.clearAllMocks(); + getDomainBudgetStatus.mockResolvedValue({ withinBudget: true }); + runStagedLLM.mockResolvedValue({ content: { ...goodDiagnosis } }); +}); + +describe('FableLoom editorial self-improvement', () => { + it('diagnoses only opted-in paused or failed runs', () => { + const run = makeRun(); + expect(shouldDiagnoseFableLoomEditorial(run, 'paused')).toBe(true); + expect(shouldDiagnoseFableLoomEditorial(run, 'failed')).toBe(true); + expect(shouldDiagnoseFableLoomEditorial(run, 'completed')).toBe(false); + expect(shouldDiagnoseFableLoomEditorial(run, 'canceled')).toBe(false); + expect(shouldDiagnoseFableLoomEditorial({ ...run, selfImproveEnabled: false }, 'paused')).toBe(false); + }); + + it('reduces the run to counters and sanitized status tokens without story data', () => { + const telemetry = buildFableLoomEditorialTelemetry(makeRun(), { + outcome: 'paused', + reason: 'round limit with private words', + sourceStep: 'playthrough review', + errorCode: '/project', + }); + const encoded = JSON.stringify(telemetry); + + expect(telemetry).toMatchObject({ + outcome: 'paused', + reason: 'none', + sourceStep: 'unknown', + errorCode: 'none', + rounds: [{ + remediation: { changed: false, evaluationFindingCount: 1 }, + playthrough: { + diagnosticFindingCount: 1, + reviewFindingCount: 1, + findingCategories: { choice: 1 }, + }, + }], + }); + expect(encoded).not.toContain('PRIVATE'); + }); + + it('queues one approval-gated PortOS task for a confident automation verdict', async () => { + const run = makeRun(); + const summary = await runFableLoomEditorialSelfImprove(run, { + outcome: 'paused', reason: 'plateau', sourceStep: 'playthrough-review', + }); + + expect(runStagedLLM).toHaveBeenCalledWith( + 'fableloom-editorial-self-improve', + expect.objectContaining({ + outcome: 'paused', outcomeReason: 'plateau', currentStep: 'playthrough-review', + telemetryJson: expect.any(String), + }), + { + providerOverride: 'writer', modelOverride: 'large', effortOverride: 'high', + returnsJson: true, source: 'fableloom-editorial-self-improve', + }, + ); + expect(recordDomainUsage).toHaveBeenCalledWith('cos', { actions: 1 }); + expect(addTask).toHaveBeenCalledWith(expect.objectContaining({ + app: PORTOS_APP_ID, + approvalRequired: true, + useWorktree: true, + openPR: true, + prCompletion: 'review-then-merge', + }), 'internal'); + expect(summary).toEqual({ + verdict: 'pipeline', area: 'prompt', title: goodDiagnosis.title, + taskId: 'sys-example', filed: true, duplicate: false, + }); + }); + + it('skips spent budgets and low-confidence or content diagnoses', async () => { + getDomainBudgetStatus.mockResolvedValueOnce({ withinBudget: false }); + await expect(runFableLoomEditorialSelfImprove(makeRun(), { outcome: 'paused' })).resolves.toBeNull(); + expect(runStagedLLM).not.toHaveBeenCalled(); + + runStagedLLM.mockResolvedValueOnce({ + content: { ...goodDiagnosis, confidence: FABLELOOM_EDITORIAL_SELF_IMPROVE_MIN_CONFIDENCE - 0.01 }, + }); + await expect(runFableLoomEditorialSelfImprove(makeRun(), { outcome: 'failed' })).resolves.toBeNull(); + + runStagedLLM.mockResolvedValueOnce({ content: { ...goodDiagnosis, verdict: 'content' } }); + await expect(runFableLoomEditorialSelfImprove(makeRun(), { outcome: 'paused' })).resolves.toBeNull(); + expect(addTask).not.toHaveBeenCalled(); + }); + + it('does not file new work when cancellation arrives during diagnosis', async () => { + const run = makeRun(); + runStagedLLM.mockImplementationOnce(async () => { + run.cancelRequested = true; + return { content: { ...goodDiagnosis } }; + }); + + await expect(runFableLoomEditorialSelfImprove(run, { outcome: 'paused' })).resolves.toBeNull(); + expect(recordDomainUsage).toHaveBeenCalledWith('cos', { actions: 1 }); + expect(addTask).not.toHaveBeenCalled(); + }); + + it('keeps the task headline stable per defect and excludes run identity', () => { + const telemetry = buildFableLoomEditorialTelemetry(makeRun(), { + outcome: 'paused', reason: 'plateau', sourceStep: 'playthrough-review', + }); + const task = buildFableLoomEditorialSelfImproveTask({ diagnosis: goodDiagnosis, telemetry }); + + expect(task.description).not.toContain('\n'); + expect(task.description).toContain('prompt/make-remediation-return-applicable-transition'); + expect(task.context).not.toContain('loom-'); + expect(task.context).not.toContain('episode-'); + expect(task.context).toContain(goodDiagnosis.proposedChange); + }); +}); diff --git a/server/services/fableLoom/index.js b/server/services/fableLoom/index.js index fc118e8f4..a3a8363d3 100644 --- a/server/services/fableLoom/index.js +++ b/server/services/fableLoom/index.js @@ -60,6 +60,14 @@ export { publicFableLoomEditorialAutopilot, startFableLoomEditorialAutopilot, } from './editorialAutopilot.js'; +export { + FABLELOOM_EDITORIAL_SELF_IMPROVE_AREAS, + FABLELOOM_EDITORIAL_SELF_IMPROVE_MIN_CONFIDENCE, + buildFableLoomEditorialSelfImproveTask, + buildFableLoomEditorialTelemetry, + runFableLoomEditorialSelfImprove, + shouldDiagnoseFableLoomEditorial, +} from './editorialSelfImprove.js'; export { _resetFableLoomBackend, isValidLoomId, From 06685dcb08db3ff1b7f3b730df9a13d4789cf442 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Sun, 30 Aug 2026 23:14:42 +0000 Subject: [PATCH 2/4] fix: address review (claude): preserve diagnosis terminal facts --- .../services/fableLoom/editorialAutopilot.js | 10 ++-- .../fableLoom/editorialAutopilot.test.js | 50 +++++++++++++++++++ .../fableLoom/editorialSelfImprove.js | 6 +-- .../fableLoom/editorialSelfImprove.test.js | 2 +- 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/server/services/fableLoom/editorialAutopilot.js b/server/services/fableLoom/editorialAutopilot.js index cb969b16f..ea8e48203 100644 --- a/server/services/fableLoom/editorialAutopilot.js +++ b/server/services/fableLoom/editorialAutopilot.js @@ -107,11 +107,12 @@ const routeOptions = (run) => ({ ...(run.route.effort ? { effort: run.route.effort } : {}), }); -const finishCanceled = (run) => touch(run, { +const finishCanceled = (run, terminalFacts = {}) => touch(run, { status: 'canceled', currentStep: null, message: 'Editorial autopilot canceled after the active AI step finished.', completedAt: nowIso(), + ...terminalFacts, }); const terminalDiagnosis = async (run, outcome, { reason = null, error = null } = {}) => { @@ -134,7 +135,7 @@ const terminalDiagnosis = async (run, outcome, { reason = null, error = null } = const finishPaused = async (run, pauseReason, message) => { const selfImprove = await terminalDiagnosis(run, 'paused', { reason: pauseReason }); - if (run.cancelRequested) return finishCanceled(run); + if (run.cancelRequested) return finishCanceled(run, { pauseReason, selfImprove }); return touch(run, { status: 'paused', pauseReason, @@ -147,7 +148,10 @@ const finishPaused = async (run, pauseReason, message) => { const finishFailed = async (run, error) => { const selfImprove = await terminalDiagnosis(run, 'failed', { reason: 'run-error', error }); - if (run.cancelRequested) return finishCanceled(run); + if (run.cancelRequested) return finishCanceled(run, { + error: errorMessage(error), + selfImprove, + }); return touch(run, { status: 'failed', currentStep: null, diff --git a/server/services/fableLoom/editorialAutopilot.test.js b/server/services/fableLoom/editorialAutopilot.test.js index 0b2ef8801..487b02e57 100644 --- a/server/services/fableLoom/editorialAutopilot.test.js +++ b/server/services/fableLoom/editorialAutopilot.test.js @@ -165,6 +165,56 @@ describe('FableLoom editorial autopilot', () => { }); }); + it('preserves a known failure when cancellation lands during diagnosis', async () => { + let finishDiagnosis; + selfImproveMock.mockImplementationOnce(() => new Promise((resolve) => { + finishDiagnosis = resolve; + })); + const providerError = Object.assign(new Error('Provider returned an unusable patch'), { + code: 'AI_RESPONSE_INVALID', + }); + remediateMock.mockRejectedValueOnce(providerError); + + const started = await startFableLoomEditorialAutopilot('loom-example', { selfImprove: true }); + await vi.waitFor(() => expect(selfImproveMock).toHaveBeenCalledTimes(1)); + expect(cancelFableLoomEditorialAutopilot(started.id).status).toBe('canceling'); + finishDiagnosis(null); + const finished = await waitForTerminal(started.id); + + expect(finished).toMatchObject({ + status: 'canceled', + error: providerError.message, + selfImprove: null, + }); + }); + + it('preserves a known pause when cancellation lands during diagnosis', async () => { + let finishDiagnosis; + selfImproveMock.mockImplementationOnce(() => new Promise((resolve) => { + finishDiagnosis = resolve; + })); + remediateMock.mockResolvedValueOnce(remediation(true)); + playtestMock.mockResolvedValueOnce(playtest({ + passed: false, + findings: [{ severity: 'low', category: 'pacing', problem: 'One route rushes its turn.' }], + })); + + const started = await startFableLoomEditorialAutopilot('loom-example', { + maxRounds: 1, + selfImprove: true, + }); + await vi.waitFor(() => expect(selfImproveMock).toHaveBeenCalledTimes(1)); + expect(cancelFableLoomEditorialAutopilot(started.id).status).toBe('canceling'); + finishDiagnosis(null); + const finished = await waitForTerminal(started.id); + + expect(finished).toMatchObject({ + status: 'canceled', + pauseReason: 'round-limit', + selfImprove: null, + }); + }); + it('keeps diagnostics-only blockers actionable in the residual findings', async () => { const diagnosticFinding = { severity: 'high', diff --git a/server/services/fableLoom/editorialSelfImprove.js b/server/services/fableLoom/editorialSelfImprove.js index eaaf8ca3a..dc2ad180d 100644 --- a/server/services/fableLoom/editorialSelfImprove.js +++ b/server/services/fableLoom/editorialSelfImprove.js @@ -114,9 +114,9 @@ export function buildFableLoomEditorialSelfImproveTask({ diagnosis, telemetry }) } const routeOptions = (run) => ({ - ...(run?.route?.providerId ? { providerOverride: run.route.providerId } : {}), - ...(run?.route?.model ? { modelOverride: run.route.model } : {}), - ...(run?.route?.effort ? { effortOverride: run.route.effort } : {}), + ...(run?.route?.providerId ? { providerDefault: run.route.providerId } : {}), + ...(run?.route?.model ? { modelDefault: run.route.model } : {}), + ...(run?.route?.effort ? { effortDefault: run.route.effort } : {}), }); /** Run one best-effort terminal diagnosis and file a task for PortOS defects. */ diff --git a/server/services/fableLoom/editorialSelfImprove.test.js b/server/services/fableLoom/editorialSelfImprove.test.js index c2ff3f7d7..d50268928 100644 --- a/server/services/fableLoom/editorialSelfImprove.test.js +++ b/server/services/fableLoom/editorialSelfImprove.test.js @@ -125,7 +125,7 @@ describe('FableLoom editorial self-improvement', () => { telemetryJson: expect.any(String), }), { - providerOverride: 'writer', modelOverride: 'large', effortOverride: 'high', + providerDefault: 'writer', modelDefault: 'large', effortDefault: 'high', returnsJson: true, source: 'fableloom-editorial-self-improve', }, ); From 1a06b17b1a70e1829de468c8b4bc44f6280e465f Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Sun, 30 Aug 2026 23:21:08 +0000 Subject: [PATCH 3/4] fix: refresh socket event catalog line references --- server/lib/socketEventCatalog.generated.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/lib/socketEventCatalog.generated.json b/server/lib/socketEventCatalog.generated.json index 20c95681f..c818e1fe0 100644 --- a/server/lib/socketEventCatalog.generated.json +++ b/server/lib/socketEventCatalog.generated.json @@ -2261,7 +2261,7 @@ { "direction": "server-to-client", "source": "client/src/components/fableloom/LoomPlayPanel.jsx", - "line": 114 + "line": 125 } ] }, @@ -2300,7 +2300,7 @@ { "direction": "server-to-client", "source": "client/src/components/fableloom/LoomPlayPanel.jsx", - "line": 145 + "line": 156 } ] }, @@ -2339,7 +2339,7 @@ { "direction": "server-to-client", "source": "client/src/components/fableloom/LoomPlayPanel.jsx", - "line": 137 + "line": 148 } ] }, @@ -2365,7 +2365,7 @@ { "direction": "server-to-client", "source": "client/src/components/fableloom/LoomPlayPanel.jsx", - "line": 118 + "line": 129 } ] }, @@ -2396,7 +2396,7 @@ { "direction": "server-to-client", "source": "client/src/components/fableloom/LoomPlayPanel.jsx", - "line": 122 + "line": 133 } ] }, @@ -2409,7 +2409,7 @@ { "direction": "server-to-client", "source": "client/src/components/fableloom/LoomPlayPanel.jsx", - "line": 126 + "line": 137 } ] }, From 6165f5cafac0fc614967cafbc0dfafca3403dcc9 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Sun, 30 Aug 2026 23:31:51 +0000 Subject: [PATCH 4/4] fix: address review (claude): preserve diagnosis results --- .../fableloom/LoomEditorialAutomation.jsx | 2 +- .../fableloom/LoomEditorialAutomation.test.jsx | 16 ++++++++++++++++ server/services/fableLoom/editorialAutopilot.js | 3 ++- .../fableLoom/editorialAutopilot.test.js | 2 ++ .../services/fableLoom/editorialSelfImprove.js | 11 ++++------- .../fableLoom/editorialSelfImprove.test.js | 2 +- 6 files changed, 26 insertions(+), 10 deletions(-) diff --git a/client/src/components/fableloom/LoomEditorialAutomation.jsx b/client/src/components/fableloom/LoomEditorialAutomation.jsx index b33a76067..29324d6d5 100644 --- a/client/src/components/fableloom/LoomEditorialAutomation.jsx +++ b/client/src/components/fableloom/LoomEditorialAutomation.jsx @@ -196,7 +196,7 @@ export default function LoomEditorialAutomation({ loom, dirty, onLoomUpdate }) { const findings = result?.type === 'autopilot' || shownRun?.status ? shownRun?.residualFindings || [] : review?.findings?.length ? review.findings : evaluation?.findings || []; - const summary = review?.summary || evaluation?.summary || shownRun?.message; + const summary = shownRun?.message || review?.summary || evaluation?.summary; const passed = result?.type === 'playtest' ? response?.passed : shownRun?.status === 'completed' || diagnostics?.passed; diff --git a/client/src/components/fableloom/LoomEditorialAutomation.test.jsx b/client/src/components/fableloom/LoomEditorialAutomation.test.jsx index 7b8458cbc..00facc3bd 100644 --- a/client/src/components/fableloom/LoomEditorialAutomation.test.jsx +++ b/client/src/components/fableloom/LoomEditorialAutomation.test.jsx @@ -168,6 +168,22 @@ describe('LoomEditorialAutomation', () => { ); }); + it('keeps the decided pause explanation visible when cancellation lands during diagnosis', async () => { + api.getLoomEditorialAutopilotStatus.mockResolvedValue({ + run: { + id: 'editorial-run-1', loomId: 'loom-1', status: 'canceled', round: 1, maxRounds: 1, + pauseReason: 'round-limit', + message: 'Editorial autopilot reached its 1-round limit with review findings still open.', + rounds: [], residualFindings: [], + lastReview: { summary: 'The story review still has open findings.', findings: [] }, + }, + }); + renderPanel(); + + expect(await screen.findByText(/reached its 1-round limit with review findings still open/i)) + .toBeInTheDocument(); + }); + it('blocks every mutating AI action while the series plan has unsaved edits', async () => { renderPanel({ dirty: true }); diff --git a/server/services/fableLoom/editorialAutopilot.js b/server/services/fableLoom/editorialAutopilot.js index ea8e48203..124a79295 100644 --- a/server/services/fableLoom/editorialAutopilot.js +++ b/server/services/fableLoom/editorialAutopilot.js @@ -135,7 +135,7 @@ const terminalDiagnosis = async (run, outcome, { reason = null, error = null } = const finishPaused = async (run, pauseReason, message) => { const selfImprove = await terminalDiagnosis(run, 'paused', { reason: pauseReason }); - if (run.cancelRequested) return finishCanceled(run, { pauseReason, selfImprove }); + if (run.cancelRequested) return finishCanceled(run, { pauseReason, message, selfImprove }); return touch(run, { status: 'paused', pauseReason, @@ -150,6 +150,7 @@ const finishFailed = async (run, error) => { const selfImprove = await terminalDiagnosis(run, 'failed', { reason: 'run-error', error }); if (run.cancelRequested) return finishCanceled(run, { error: errorMessage(error), + message: errorMessage(error), selfImprove, }); return touch(run, { diff --git a/server/services/fableLoom/editorialAutopilot.test.js b/server/services/fableLoom/editorialAutopilot.test.js index 487b02e57..7057b2bb9 100644 --- a/server/services/fableLoom/editorialAutopilot.test.js +++ b/server/services/fableLoom/editorialAutopilot.test.js @@ -184,6 +184,7 @@ describe('FableLoom editorial autopilot', () => { expect(finished).toMatchObject({ status: 'canceled', error: providerError.message, + message: providerError.message, selfImprove: null, }); }); @@ -211,6 +212,7 @@ describe('FableLoom editorial autopilot', () => { expect(finished).toMatchObject({ status: 'canceled', pauseReason: 'round-limit', + message: 'Editorial autopilot reached its 1-round limit with review findings still open.', selfImprove: null, }); }); diff --git a/server/services/fableLoom/editorialSelfImprove.js b/server/services/fableLoom/editorialSelfImprove.js index dc2ad180d..12283b8be 100644 --- a/server/services/fableLoom/editorialSelfImprove.js +++ b/server/services/fableLoom/editorialSelfImprove.js @@ -113,12 +113,6 @@ export function buildFableLoomEditorialSelfImproveTask({ diagnosis, telemetry }) }); } -const routeOptions = (run) => ({ - ...(run?.route?.providerId ? { providerDefault: run.route.providerId } : {}), - ...(run?.route?.model ? { modelDefault: run.route.model } : {}), - ...(run?.route?.effort ? { effortDefault: run.route.effort } : {}), -}); - /** Run one best-effort terminal diagnosis and file a task for PortOS defects. */ export async function runFableLoomEditorialSelfImprove(run, context = {}) { if (!shouldDiagnoseFableLoomEditorial(run, context.outcome)) return null; @@ -137,7 +131,10 @@ export async function runFableLoomEditorialSelfImprove(run, context = {}) { maxPaths: telemetry.maxPaths, telemetryJson: JSON.stringify(telemetry.rounds, null, 2), }, { - ...routeOptions(run), + // Keep the selected provider when available, but let this software-focused + // stage resolve its own heavy model instead of inheriting a story-writing + // model (modelDefault outranks stage tiers in stageRunner). + ...(run?.route?.providerId ? { providerDefault: run.route.providerId } : {}), returnsJson: true, source: SELF_IMPROVE_STAGE, }); diff --git a/server/services/fableLoom/editorialSelfImprove.test.js b/server/services/fableLoom/editorialSelfImprove.test.js index d50268928..285c099b2 100644 --- a/server/services/fableLoom/editorialSelfImprove.test.js +++ b/server/services/fableLoom/editorialSelfImprove.test.js @@ -125,7 +125,7 @@ describe('FableLoom editorial self-improvement', () => { telemetryJson: expect.any(String), }), { - providerDefault: 'writer', modelDefault: 'large', effortDefault: 'high', + providerDefault: 'writer', returnsJson: true, source: 'fableloom-editorial-self-improve', }, );