From 4e6ffb60cacac9a8005129a4dc9f34ad61374a38 Mon Sep 17 00:00:00 2001 From: zebster-cmd Date: Tue, 14 Jul 2026 18:46:28 +0200 Subject: [PATCH 1/3] docs(copilot): openspec setup + requesty reasoning-leak change Initialize OpenSpec (spec-driven) and gitignore .claude/ (local Claude Code state). Add the change 'fix-requesty-reasoning-leak' (proposal, design, specs, tasks) for capability copilot-reasoning-separation: isolate model reasoning ("thinking") from content and artifact text. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 3 + .../.openspec.yaml | 2 + .../fix-requesty-reasoning-leak/design.md | 56 +++++++++++++++++++ .../fix-requesty-reasoning-leak/proposal.md | 31 ++++++++++ .../copilot-reasoning-separation/spec.md | 55 ++++++++++++++++++ .../fix-requesty-reasoning-leak/tasks.md | 35 ++++++++++++ openspec/config.yaml | 20 +++++++ 7 files changed, 202 insertions(+) create mode 100644 openspec/changes/fix-requesty-reasoning-leak/.openspec.yaml create mode 100644 openspec/changes/fix-requesty-reasoning-leak/design.md create mode 100644 openspec/changes/fix-requesty-reasoning-leak/proposal.md create mode 100644 openspec/changes/fix-requesty-reasoning-leak/specs/copilot-reasoning-separation/spec.md create mode 100644 openspec/changes/fix-requesty-reasoning-leak/tasks.md create mode 100644 openspec/config.yaml diff --git a/.gitignore b/.gitignore index 4741fee4f5445..6431f32082bc6 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,6 @@ storageState.json /.playwright-browsers/ **/.vitest-attachments/ /blocksuite/framework/std/src/__tests__/gfx/__screenshots__/ + +# Claude Code local state (worktrees, session data, regenerable opsx tooling) +.claude/ diff --git a/openspec/changes/fix-requesty-reasoning-leak/.openspec.yaml b/openspec/changes/fix-requesty-reasoning-leak/.openspec.yaml new file mode 100644 index 0000000000000..64105fc96f1f5 --- /dev/null +++ b/openspec/changes/fix-requesty-reasoning-leak/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-14 diff --git a/openspec/changes/fix-requesty-reasoning-leak/design.md b/openspec/changes/fix-requesty-reasoning-leak/design.md new file mode 100644 index 0000000000000..b8b51fea556a5 --- /dev/null +++ b/openspec/changes/fix-requesty-reasoning-leak/design.md @@ -0,0 +1,56 @@ +## Context + +The copilot streaming pipeline turns provider SSE into a union of stream parts. Reasoning has a first-class channel already: the native `llm_adapter` emits `LlmToolLoopStreamEvent`s (`text_delta` / `reasoning_delta`, `native.ts:708-712`), which `native-adapter.ts` maps to `text-delta` / `reasoning-delta` (streamText ~213-238) and `{type:'text-delta'}` / `{type:'reasoning'}` (streamObject ~343-354). `TextStreamParser` styles reasoning as callouts (`utils.ts:275-277`), and `StreamObjectParser.mergeContent` already keeps only `text-delta` for the UI object path (`utils.ts:365-372`). + +This machinery only works when the provider delivers reasoning as a **separate field**. GLM via Requesty (`RequestyProvider extends OpenAIProvider`, `resolveModelBackendKind() = 'openai_chat'`) instead embeds reasoning inline as `…` inside `content`, so it arrives as `text_delta` and is treated as content. A repo-wide search for ``, `reasoning_content` returns zero matches — nothing separates inline reasoning today. Separately, the non-streaming text path deliberately folds reasoning into content: `extractTextResponse` filters `type === 'text' || type === 'reasoning'` (`native-execution-engine.ts:59-64`) and `adapter.text()` concatenates every chunk including reasoning (`native-adapter.ts:164-174`). Both feed `code-artifact.ts:37-55`, which consumes the raw string. + +Constraint: the SSE decode itself lives in the external compiled Rust `llm_adapter` crate and is out of reach. The fix must live in the in-repo TypeScript adapter/runtime layer. + +## Goals / Non-Goals + +**Goals:** + +- Strip inline `…` from the content stream and re-route it to the reasoning channel, in both streamText and streamObject paths, robust to tags split across chunks. +- Ensure non-streaming tool/artifact text (`extractTextResponse`, `adapter.text()`) contains content only, never reasoning. +- Keep reasoning available on the reasoning channel for UI display. + +**Non-Goals:** + +- Modifying the external Rust `affine_doc_loader` / `llm_adapter` crates. +- Changing how natively-separated reasoning (Anthropic/Gemini/OpenAI) is produced. +- Broad prompt-output sanitization beyond reasoning isolation. + +## Decisions + +**Decision 1 — Split inline `` in the TS adapter layer, not at the SSE decode.** +The SSE decode is compiled and unmodifiable. Implement a small stateful tag-splitter applied to `text_delta` text as it is turned into stream parts in `native-adapter.ts`. It maintains an `insideThink` flag and a small carry buffer for a possibly-partial boundary tag, emitting text outside think as `text-delta`/`{type:'text-delta'}` and text inside think as `reasoning-delta`/`{type:'reasoning'}`. + +- _Alternative considered:_ strip in `TextStreamParser.parse` `'text-delta'` case (`utils.ts:145-158`). Rejected as the single point because the parser is content-formatting-oriented and the object path (`StreamObjectParser`) would still need the split; doing it once at the adapter boundary covers both `streamText` and `streamObject`. +- _Alternative considered:_ strip only in `code-artifact.ts`. Rejected — a band-aid that leaves raw `` leaking into chat and other tools. + +**Decision 2 — Stateful splitter tolerant of chunk boundaries.** +Tags can split across chunks (``, or body across many deltas). The splitter buffers a trailing partial that could be the start of ``/`` and only commits text once it is known not to be a tag boundary. This satisfies the "split across chunks" scenario. + +**Decision 3 — Fix the non-streaming text path independently.** +Even with Decision 1, `adapter.text()` accumulates reasoning-delta chunks and `extractTextResponse` keeps reasoning parts. Change `extractTextResponse` to filter `type === 'text'` only, and make `text()` skip reasoning-delta chunks. Both are required; Decision 1 alone does not stop the artifact leak for models that separate reasoning natively. + +**Decision 4 — Requesty GLM reasoning behavior flag is optional and secondary.** +Adding a `reasoning_supported` behavior flag to the GLM variant in `model_registry.rs` could make Requesty return separated reasoning where supported, but requires a native rebuild and depends on provider behavior. The `` splitter must exist regardless as the safety net, so the flag is deferred/optional and evaluated after the splitter lands. + +## Risks / Trade-offs + +- **False positives — legitimate `` in user content (e.g. a doc about HTML/XML).** → Scope the splitter to the reasoning-prone path and only treat `` at the start of a reasoning segment; keep it conservative (exact tag match, not arbitrary angle-bracket content). Covered by the "content without think tags is unchanged" scenario. +- **Chunk-boundary bugs dropping or duplicating characters.** → Unit tests that feed the same content split at every offset and assert content + reasoning reassemble exactly. +- **Nested or malformed tags (`` with no close).** → On stream end, flush any open think buffer to the reasoning channel (never to content). +- **Other tools relying on reasoning-in-content.** → Grep confirms none do; `mergeContent` already excludes reasoning, so aligning `text()`/`extractTextResponse` matches existing intent. + +## Migration Plan + +- Backend-only, no schema/data migration. Ships via the existing CI image build → redeploy loop used for prior copilot changes. +- If Decision 4's model-registry flag is included, it requires a native rebuild (same as prior tool-wiring changes). +- Rollback: revert the adapter/runtime changes; no persisted state is affected. + +## Resolved Decisions + +- **Splitter scope:** applied **universally** but conservatively — it reacts only to literal ``/`` tags, so it is a no-op for providers that separate reasoning natively (Claude/Gemini/OpenAI). No per-provider gating; it serves as a general safety net. +- **Tag set:** handle **`` only** (the confirmed GLM convention). Other conventions (``, ``) are deliberately not stripped, to avoid eating legitimate content that references those tags; expand only if another model is shown to leak. diff --git a/openspec/changes/fix-requesty-reasoning-leak/proposal.md b/openspec/changes/fix-requesty-reasoning-leak/proposal.md new file mode 100644 index 0000000000000..ef5c84ce04739 --- /dev/null +++ b/openspec/changes/fix-requesty-reasoning-leak/proposal.md @@ -0,0 +1,31 @@ +## Why + +Models served through the Requesty router (notably GLM 5.2 / `sference/glm-5.2`) emit their chain-of-thought **inline** as `…` tags inside the normal content stream rather than in a separate reasoning field. The copilot layer has no mechanism to detect or separate inline reasoning, so the thinking text is treated as ordinary content — accumulated verbatim and baked into generated HTML code artifacts, where it is plainly visible in the preview. A second, latent defect means even _correctly_ separated reasoning (e.g. Claude) is folded back into the content string that the code-artifact tool consumes. The result is leaked reasoning in user-facing output. + +## What Changes + +- Inline `…` segments in the assistant content stream are detected in the TypeScript adapter layer and re-routed onto the existing **reasoning** channel (`reasoning-delta` / `{ type: 'reasoning' }`) instead of being emitted as content, for both the `streamText` and `streamObject` paths. +- The non-streaming text/tool path stops folding reasoning into returned content: `extractTextResponse` drops `type === 'reasoning'` parts, and `adapter.text()` skips reasoning-delta chunks. Tool-driven prompts (e.g. Code Artifact) receive content only. +- As a result, `code_artifact` output no longer contains `` tags or reasoning text, regardless of whether the model separates reasoning natively or emits it inline. +- (Optional, evaluated in design) Tag the Requesty GLM model variant with a reasoning behavior flag so the request asks for separated reasoning where the provider supports it — but inline `` stripping remains as an always-on safety net. + +## Capabilities + +### New Capabilities + +- `copilot-reasoning-separation`: Reasoning/"thinking" output — whether delivered natively as a separate field or inline as `…` tags — is isolated onto the reasoning channel and excluded from assistant content and from tool/artifact text. + +### Modified Capabilities + + + +## Impact + +- **Code (backend copilot layer only):** + - `packages/backend/server/src/plugins/copilot/runtime/tool/native-adapter.ts` — `streamText` (~213-238), `streamObject` (~343-354), and `text()` accumulation (~164-174). + - `packages/backend/server/src/plugins/copilot/providers/utils.ts` — `TextStreamParser` `'text-delta'` handling (~145-158); reference behavior in `StreamObjectParser.mergeContent` (~365-372). + - `packages/backend/server/src/plugins/copilot/runtime/native-execution-engine.ts` — `extractTextResponse` (59-64). + - (Optional) `packages/backend/native/src/llm/core/model_registry.rs` — Requesty GLM variant behavior flag (requires native rebuild). +- **No changes** to the external Rust `affine_doc_loader` or `llm_adapter` crates (the SSE decode is compiled and out of scope). +- **Tests:** server-side ava suite under `packages/backend/server/src/__tests__/copilot/` (CI Node 22). +- **Behavior:** user-facing output (chat + artifacts) no longer leaks reasoning; reasoning remains available on its own channel for display as callouts. diff --git a/openspec/changes/fix-requesty-reasoning-leak/specs/copilot-reasoning-separation/spec.md b/openspec/changes/fix-requesty-reasoning-leak/specs/copilot-reasoning-separation/spec.md new file mode 100644 index 0000000000000..714b5c4f14b69 --- /dev/null +++ b/openspec/changes/fix-requesty-reasoning-leak/specs/copilot-reasoning-separation/spec.md @@ -0,0 +1,55 @@ +## ADDED Requirements + +### Requirement: Inline reasoning tags are separated from content + +The system SHALL detect reasoning delivered inline as `…` segments within the assistant content stream and route that reasoning onto the reasoning channel, excluding it from assistant content. This SHALL apply to both the streaming text path and the streaming object path. + +#### Scenario: Inline think block in a streamed text response + +- **WHEN** a provider (e.g. GLM via Requesty) streams content containing `my reasoningvisible answer` +- **THEN** `my reasoning` is emitted on the reasoning channel (as `reasoning-delta`) +- **AND** only `visible answer` is emitted as content +- **AND** the accumulated content string contains no `` or `` markers and no reasoning text + +#### Scenario: Think block split across multiple stream chunks + +- **WHEN** the `` open tag, reasoning body, and `` close tag arrive in separate stream chunks +- **THEN** the reasoning body is still routed to the reasoning channel in full +- **AND** no partial tag fragment leaks into content + +#### Scenario: Content without think tags is unchanged + +- **WHEN** a provider streams content that contains no `` tags +- **THEN** all content is emitted unchanged as content +- **AND** no reasoning-delta is produced from the content + +### Requirement: Tool and artifact text excludes reasoning + +The system SHALL exclude reasoning parts from the text returned to tool-driven, non-streaming prompt executions, so that generated artifacts contain only content. + +#### Scenario: Code artifact from a reasoning model + +- **WHEN** a `code_artifact` prompt is executed against a model whose response includes reasoning (either separated natively or inline `` tags) +- **THEN** the returned artifact HTML contains no reasoning text and no ``/`` markers + +#### Scenario: Non-streaming text extraction drops reasoning parts + +- **WHEN** `extractTextResponse` processes a message whose `content` parts include both `text` and `reasoning` parts +- **THEN** only `text` parts are concatenated into the returned string +- **AND** `reasoning` parts are omitted + +#### Scenario: Accumulated text() skips reasoning chunks + +- **WHEN** `adapter.text()` accumulates a stream that yields both text-delta and reasoning-delta chunks +- **THEN** the returned string contains only the text-delta content +- **AND** reasoning-delta content is omitted + +### Requirement: Reasoning remains available on its own channel + +The system SHALL continue to surface reasoning on the reasoning channel for display purposes; separation MUST NOT discard reasoning entirely. + +#### Scenario: Reasoning still emitted for display + +- **WHEN** a response contains reasoning (native or inline) +- **THEN** the reasoning is available as reasoning-delta / `{ type: 'reasoning' }` events for the UI to render (e.g. as a callout) +- **AND** it is not present in the content channel diff --git a/openspec/changes/fix-requesty-reasoning-leak/tasks.md b/openspec/changes/fix-requesty-reasoning-leak/tasks.md new file mode 100644 index 0000000000000..316459025db16 --- /dev/null +++ b/openspec/changes/fix-requesty-reasoning-leak/tasks.md @@ -0,0 +1,35 @@ +## 1. Inline reasoning splitter (TDD) + +- [x] 1.1 Add a failing unit test: a helper that, given a sequence of content chunks containing `…`, yields separated `{ content, reasoning }` streams — assert reasoning routed out, content clean, no tag markers left +- [x] 1.2 Add a failing test for tags split across chunk boundaries (feed the same input split at every offset; content + reasoning must reassemble exactly) +- [x] 1.3 Add a failing test for an unterminated `` (flush open buffer to reasoning on stream end, never to content) +- [x] 1.4 Add a failing test that content with no `` tags passes through unchanged +- [x] 1.5 Implement the stateful `` splitter (insideThink flag + partial-boundary carry buffer) to make 1.1–1.4 pass + +## 2. Wire the splitter into the adapter paths + +- [x] 2.1 Apply the splitter in `native-adapter.ts` `streamText` (~213-238): inside-think text → `reasoning-delta`, outside → `text-delta` +- [x] 2.2 Apply the splitter in `native-adapter.ts` `streamObject` (~343-354): inside-think → `{type:'reasoning'}`, outside → `{type:'text-delta'}` +- [x] 2.3 Add a test proving a GLM-style inline `` stream produces reasoning on its channel and clean content on both stream paths + +## 3. Purge reasoning from the non-streaming text/tool path + +- [x] 3.1 Add a failing test: `extractTextResponse` (`native-execution-engine.ts:59-64`) drops `type === 'reasoning'` parts, keeps only `text` +- [x] 3.2 Change `extractTextResponse` filter to `type === 'text'` only +- [x] 3.3 Add a failing test: `adapter.text()` (`native-adapter.ts:164-174`) skips reasoning-delta chunks +- [x] 3.4 Update `adapter.text()` accumulation to skip reasoning-delta chunks + +## 4. End-to-end artifact assertion + +- [x] 4.1 Add a test that a `code_artifact` prompt over a stream containing inline `` tags returns HTML with no ``/`` markers and no reasoning text +- [x] 4.2 Add a test for the same over a natively-separated-reasoning response (reasoning parts present) → artifact still content-only + +## 5. Optional: Requesty GLM reasoning flag + +- [x] 5.1 Evaluate adding a `reasoning_supported`/behavior flag to the Requesty GLM variant in `model_registry.rs` — **DEFERRED**. The inline `` splitter already fully fixes the symptom and is provider-agnostic. Adding the flag requires a native (Rust) rebuild and depends on whether Requesty forwards a reasoning-separation param for GLM and whether GLM honors it — an optimization that needs testing against the live Requesty API, not required for the fix. The splitter stays as the always-on safety net regardless. +- [x] 5.2 N/A — flag deferred (see 5.1); no native rebuild performed. + +## 6. Verify + +- [~] 6.1 Copilot ava suite: **cannot run locally** — ava fails to bootstrap its prelude under local Node 24 (`ERR_MODULE_NOT_FOUND` resolving `src/prelude.ts`), the exact "runs in CI on Node 22, not locally" constraint noted in design. Verified equivalent behavior via standalone `tsx` execution: **58 splitter assertions + 13 integration assertions (streamText/streamObject/text/extractTextResponse/code_artifact) all green**, plus a clean `tsc --noEmit` on all changed files and the spec. The ava specs themselves run in CI on Node 22. +- [x] 6.2 Manual GLM 5.2-via-Requesty artifact check — confirmed on the live deployment: HTML artifact preview shows no thinking blocks. diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 0000000000000..392946c67c03e --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,20 @@ +schema: spec-driven + +# Project context (optional) +# This is shown to AI when creating artifacts. +# Add your tech stack, conventions, style guides, domain knowledge, etc. +# Example: +# context: | +# Tech stack: TypeScript, React, Node.js +# We use conventional commits +# Domain: e-commerce platform + +# Per-artifact rules (optional) +# Add custom rules for specific artifacts. +# Example: +# rules: +# proposal: +# - Keep proposals under 500 words +# - Always include a "Non-goals" section +# tasks: +# - Break tasks into chunks of max 2 hours From 44f1a620486ee3bd220827ac145c873abd038938 Mon Sep 17 00:00:00 2001 From: zebster-cmd Date: Tue, 14 Jul 2026 18:47:43 +0200 Subject: [PATCH 2/3] feat(copilot): separate inline reasoning from content + artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ThinkTagSplitter (stateful, chunk-boundary-tolerant) + stripThinkTags. Wire it into the native adapter: streamText/streamObject route inline … onto the reasoning channel; text() excludes reasoning (emitReasoning:false). extractTextResponse now keeps only text parts and strips inline from them, so code_artifact HTML is reasoning-free whether the model separates reasoning natively or emits it inline (e.g. GLM via Requesty). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plugins/copilot/providers/reasoning.ts | 127 ++++++++++++++++++ .../runtime/native-execution-engine.ts | 9 +- .../copilot/runtime/tool/native-adapter.ts | 76 ++++++++--- 3 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 packages/backend/server/src/plugins/copilot/providers/reasoning.ts diff --git a/packages/backend/server/src/plugins/copilot/providers/reasoning.ts b/packages/backend/server/src/plugins/copilot/providers/reasoning.ts new file mode 100644 index 0000000000000..6f272ceb0d9bd --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/providers/reasoning.ts @@ -0,0 +1,127 @@ +/** + * Inline reasoning ("thinking") separation. + * + * Some providers — notably GLM served through the Requesty router — emit their + * chain-of-thought inline as `…` segments inside the normal + * content stream rather than in a separate reasoning field. Left untouched, that + * reasoning is treated as content: accumulated verbatim and baked into generated + * artifacts. This module splits inline `` segments out of content so they + * can be routed onto the reasoning channel (or dropped from tool/artifact text). + * + * The splitter is deliberately conservative: it reacts only to the literal + * `` / `` tags, so it is a no-op for providers that separate + * reasoning natively (they never emit these tags in content). + */ + +const OPEN_TAG = ''; +const CLOSE_TAG = ''; + +export interface ReasoningSegment { + kind: 'text' | 'reasoning'; + text: string; +} + +/** + * Length of the longest suffix of `s` that is a *proper* prefix of `tag` + * (i.e. shorter than the full tag). Used to hold back a trailing fragment that + * might be the start of a boundary tag continued in the next chunk. + */ +function partialTagSuffixLength(s: string, tag: string): number { + const max = Math.min(s.length, tag.length - 1); + for (let k = max; k > 0; k--) { + if (tag.startsWith(s.slice(s.length - k))) { + return k; + } + } + return 0; +} + +function pushSegment( + out: ReasoningSegment[], + kind: ReasoningSegment['kind'], + text: string +) { + if (!text) return; + const last = out[out.length - 1]; + if (last && last.kind === kind) { + last.text += text; + } else { + out.push({ kind, text }); + } +} + +/** + * Stateful, chunk-boundary-tolerant splitter. Feed streamed chunks through + * `push()`; call `flush()` once at end-of-stream to drain any held fragment. + */ +export class ThinkTagSplitter { + #inside = false; + // A trailing fragment of the previous chunk that could be the start of the + // next boundary tag (OPEN when outside, CLOSE when inside). + #buffer = ''; + + push(chunk: string): ReasoningSegment[] { + const out: ReasoningSegment[] = []; + let data = this.#buffer + chunk; + this.#buffer = ''; + + let i = 0; + while (i < data.length) { + const tag = this.#inside ? CLOSE_TAG : OPEN_TAG; + const idx = data.indexOf(tag, i); + const kind: ReasoningSegment['kind'] = this.#inside + ? 'reasoning' + : 'text'; + + if (idx === -1) { + // No complete boundary tag remains. Emit everything except a trailing + // fragment that might be the start of the tag (continued next chunk). + const rest = data.slice(i); + const hold = partialTagSuffixLength(rest, tag); + pushSegment(out, kind, rest.slice(0, rest.length - hold)); + this.#buffer = rest.slice(rest.length - hold); + break; + } + + pushSegment(out, kind, data.slice(i, idx)); + this.#inside = !this.#inside; + i = idx + tag.length; + } + + return out; + } + + /** + * Drain any buffered fragment at end-of-stream. An unterminated `` + * (we are still `inside`) flushes its remainder to the reasoning channel — + * never to content. A dangling partial `` outside a block is real + * content and is flushed as text. + */ + flush(): ReasoningSegment[] { + const out: ReasoningSegment[] = []; + if (this.#buffer) { + pushSegment(out, this.#inside ? 'reasoning' : 'text', this.#buffer); + this.#buffer = ''; + } + return out; + } +} + +/** + * Split a complete string into its content and reasoning parts. Convenience for + * the non-streaming path (a fully-assembled message part). + */ +export function stripThinkTags(fullText: string): { + content: string; + reasoning: string; +} { + const splitter = new ThinkTagSplitter(); + const segments = [...splitter.push(fullText), ...splitter.flush()]; + let content = ''; + let reasoning = ''; + for (const seg of segments) { + if (seg.kind === 'text') content += seg.text; + else reasoning += seg.text; + } + return { content, reasoning }; +} diff --git a/packages/backend/server/src/plugins/copilot/runtime/native-execution-engine.ts b/packages/backend/server/src/plugins/copilot/runtime/native-execution-engine.ts index f67dbd6ab79b8..d5e30ba965b31 100644 --- a/packages/backend/server/src/plugins/copilot/runtime/native-execution-engine.ts +++ b/packages/backend/server/src/plugins/copilot/runtime/native-execution-engine.ts @@ -14,6 +14,7 @@ import { parseNativeStructuredOutput, } from '../../../native'; import { type ByokFeatureKind, ByokService } from '../byok'; +import { stripThinkTags } from '../providers/reasoning'; import { type StreamObject } from '../providers/types'; import { CopilotExecutionMetrics } from './execution-metrics'; import { @@ -56,10 +57,12 @@ function resolveAbortSignal( : signalOrOptions?.signal; } -function extractTextResponse(response: LlmDispatchResponse) { +export function extractTextResponse(response: LlmDispatchResponse) { + // Content only: drop separated reasoning parts, and strip any inline + // … reasoning embedded in text parts (e.g. GLM via Requesty). return response.message.content - .filter(part => part.type === 'text' || part.type === 'reasoning') - .map(part => part.text) + .filter(part => part.type === 'text') + .map(part => stripThinkTags(part.text).content) .join('') .trim(); } diff --git a/packages/backend/server/src/plugins/copilot/runtime/tool/native-adapter.ts b/packages/backend/server/src/plugins/copilot/runtime/tool/native-adapter.ts index c196a1836b761..2da43a9f6de06 100644 --- a/packages/backend/server/src/plugins/copilot/runtime/tool/native-adapter.ts +++ b/packages/backend/server/src/plugins/copilot/runtime/tool/native-adapter.ts @@ -2,6 +2,10 @@ import { Logger } from '@nestjs/common'; import type { LlmRequest, LlmToolLoopStreamEvent } from '../../../../native'; import type { NodeTextMiddleware } from '../../config'; +import { + type ReasoningSegment, + ThinkTagSplitter, +} from '../../providers/reasoning'; import type { PromptMessage, StreamObject } from '../../providers/types'; import { CitationFootnoteFormatter, @@ -167,7 +171,10 @@ export class NativeProviderAdapter { messages?: PromptMessage[] ) { let output = ''; - for await (const chunk of this.streamText(request, signal, messages)) { + // Tool/value text (e.g. code_artifact) is content only — never reasoning. + for await (const chunk of this.streamText(request, signal, messages, { + emitReasoning: false, + })) { output += chunk; } return output.trim(); @@ -176,13 +183,43 @@ export class NativeProviderAdapter { async *streamText( request: LlmRequest, signal?: AbortSignal, - messages?: PromptMessage[] + messages?: PromptMessage[], + options: { emitReasoning?: boolean } = {} ): AsyncIterableIterator { + const emitReasoning = options.emitReasoning ?? true; const textParser = this.#enableCallout ? new TextStreamParser() : null; const citationFormatter = this.#enableCitationFootnote ? new CitationFootnoteFormatter() : null; + // Separates inline … reasoning out of the content stream so + // it is routed onto the reasoning channel (or dropped when emitReasoning is + // false). No-op for providers that separate reasoning natively. + const thinkSplitter = new ThinkTagSplitter(); let streamPartId = 0; + const routeSegments = function* ( + segments: ReasoningSegment[] + ): Generator { + for (const segment of segments) { + if (segment.kind === 'reasoning') { + if (!emitReasoning) continue; + yield textParser + ? textParser.parse({ + type: 'reasoning-delta', + id: String(streamPartId++), + text: segment.text, + }) + : segment.text; + } else { + yield textParser + ? textParser.parse({ + type: 'text-delta', + id: String(streamPartId++), + text: segment.text, + }) + : segment.text; + } + } + }; const usageState: { model?: string; usage?: Extract['usage']; @@ -212,18 +249,11 @@ export class NativeProviderAdapter { } case 'text_delta': { const textEvent = event as unknown as { text: string }; - if (textParser) { - yield textParser.parse({ - type: 'text-delta', - id: String(streamPartId++), - text: textEvent.text, - }); - } else { - yield textEvent.text; - } + yield* routeSegments(thinkSplitter.push(textEvent.text)); break; } case 'reasoning_delta': { + if (!emitReasoning) break; const reasoningEvent = event as unknown as { text: string }; if (textParser) { yield textParser.parse({ @@ -280,6 +310,8 @@ export class NativeProviderAdapter { { type: 'done' } >; usageState.usage = doneEvent.usage ?? usageState.usage; + // Drain any buffered inline-reasoning fragment before the tails. + yield* routeSegments(thinkSplitter.flush()); const footnotes = textParser?.end() ?? ''; const citations = citationFormatter?.end() ?? ''; const tails = [citations, footnotes].filter(Boolean).join('\n'); @@ -311,8 +343,23 @@ export class NativeProviderAdapter { const citationFormatter = this.#enableCitationFootnote ? new CitationFootnoteFormatter() : null; + const thinkSplitter = new ThinkTagSplitter(); const fallbackAttachmentFootnotes = new Map(); let hasFootnoteReference = false; + const routeSegments = function* ( + segments: ReasoningSegment[] + ): Generator { + for (const segment of segments) { + if (segment.kind === 'reasoning') { + yield { type: 'reasoning', textDelta: segment.text }; + } else { + if (segment.text.includes('[^')) { + hasFootnoteReference = true; + } + yield { type: 'text-delta', textDelta: segment.text }; + } + } + }; const usageState: { model?: string; usage?: Extract['usage']; @@ -342,10 +389,7 @@ export class NativeProviderAdapter { } case 'text_delta': { const textEvent = event as unknown as { text: string }; - if (textEvent.text.includes('[^')) { - hasFootnoteReference = true; - } - yield { type: 'text-delta', textDelta: textEvent.text }; + yield* routeSegments(thinkSplitter.push(textEvent.text)); break; } case 'reasoning_delta': { @@ -394,6 +438,8 @@ export class NativeProviderAdapter { { type: 'done' } >; usageState.usage = doneEvent.usage ?? usageState.usage; + // Drain any buffered inline-reasoning fragment before the tails. + yield* routeSegments(thinkSplitter.flush()); const citations = citationFormatter?.end() ?? ''; if (citations) { hasFootnoteReference = true; From a42de50cf8bdf0c5a2f880530ed45dfda73b47ca Mon Sep 17 00:00:00 2001 From: zebster-cmd Date: Tue, 14 Jul 2026 18:49:46 +0200 Subject: [PATCH 3/3] test(copilot): reasoning separation unit + integration specs Splitter: basic/multiple blocks, split-at-every-offset invariance, unterminated/partial tags, think-free passthrough. Integration: streamText/streamObject route inline to the reasoning channel with clean content, text() excludes reasoning, extractTextResponse drops reasoning + strips inline tags, and code_artifact HTML is reasoning-free. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../copilot/reasoning-separation.spec.ts | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 packages/backend/server/src/__tests__/copilot/reasoning-separation.spec.ts diff --git a/packages/backend/server/src/__tests__/copilot/reasoning-separation.spec.ts b/packages/backend/server/src/__tests__/copilot/reasoning-separation.spec.ts new file mode 100644 index 0000000000000..a5e353ca73c5e --- /dev/null +++ b/packages/backend/server/src/__tests__/copilot/reasoning-separation.spec.ts @@ -0,0 +1,240 @@ +import test from 'ava'; + +import { + type ReasoningSegment, + stripThinkTags, + ThinkTagSplitter, +} from '../../plugins/copilot/providers/reasoning'; +import { extractTextResponse } from '../../plugins/copilot/runtime/native-execution-engine'; +import { NativeProviderAdapter } from '../../plugins/copilot/runtime/tool/native-adapter'; +import { createCodeArtifactTool } from '../../plugins/copilot/tools/code-artifact'; + +/** Build a fake native dispatch that yields a fixed list of runtime events. */ +function dispatchOf(events: Array>) { + return async function* () { + for (const event of events) { + yield event as any; + } + } as any; +} + +const GLM_STREAM = [ + { type: 'text_delta', text: 'Vis' }, + { type: 'text_delta', text: 'ible secret' }, + { type: 'text_delta', text: ' reasoning tail' }, + { type: 'done' }, +]; + +/** + * Feed a sequence of chunks through a fresh splitter and collapse the emitted + * segments into `{ content, reasoning }` totals (order preserved per channel). + */ +function collect(chunks: string[]): { content: string; reasoning: string } { + const splitter = new ThinkTagSplitter(); + const segments: ReasoningSegment[] = []; + for (const chunk of chunks) { + segments.push(...splitter.push(chunk)); + } + segments.push(...splitter.flush()); + + let content = ''; + let reasoning = ''; + for (const seg of segments) { + if (seg.kind === 'text') content += seg.text; + else reasoning += seg.text; + } + return { content, reasoning }; +} + +// 1.1 — basic separation over a single chunk +test('separates reasoning from content and leaves no tag markers', t => { + const { content, reasoning } = collect(['foobarbaz']); + t.is(content, 'foobaz'); + t.is(reasoning, 'bar'); + t.false(content.includes('')); + t.false(content.includes('')); +}); + +// 1.1 — multiple think blocks in one stream +test('separates multiple blocks', t => { + const { content, reasoning } = collect([ + 'aonebtwoc', + ]); + t.is(content, 'abc'); + t.is(reasoning, 'onetwo'); +}); + +// 1.2 — tags/content split across chunk boundaries at every offset +test('is invariant to how the stream is chunked (split at every offset)', t => { + const whole = 'intro hidden reasoning answer text'; + const expected = collect([whole]); + t.is(expected.content, 'intro answer text'); + t.is(expected.reasoning, 'hidden reasoning'); + + for (let i = 1; i < whole.length; i++) { + const chunks = [whole.slice(0, i), whole.slice(i)]; + const got = collect(chunks); + t.deepEqual( + got, + expected, + `mismatch when split at offset ${i} (${JSON.stringify(chunks)})` + ); + } + + // also split into single characters + const perChar = collect(whole.split('')); + t.deepEqual(perChar, expected); +}); + +// 1.3 — unterminated flushes to reasoning, never to content +test('unterminated routes the remainder to reasoning, content stays clean', t => { + const { content, reasoning } = collect(['visibledangling reasoning']); + t.is(content, 'visible'); + t.is(reasoning, 'dangling reasoning'); + t.false(content.includes('')); +}); + +// 1.3 — a dangling partial close tag at end-of-stream is not leaked to content +test('a trailing partial close tag is flushed to the reasoning channel', t => { + const { content, reasoning } = collect(['xy { + const { content, reasoning } = collect(['hello tags is unchanged and produces no reasoning', t => { + const { content, reasoning } = collect([ + 'plain content, ', + 'no reasoning here.', + ]); + t.is(content, 'plain content, no reasoning here.'); + t.is(reasoning, ''); +}); + +// stripThinkTags convenience over a complete string +test('stripThinkTags splits a whole string into content and reasoning', t => { + const { content, reasoning } = stripThinkTags( + 'internal

Title

' + ); + t.is(content, '

Title

'); + t.is(reasoning, 'internal'); +}); + +test('stripThinkTags leaves think-free text intact', t => { + const { content, reasoning } = stripThinkTags('

No reasoning

'); + t.is(content, '

No reasoning

'); + t.is(reasoning, ''); +}); + +// 2.3 — GLM-style inline stream through streamObject: reasoning on its channel, +// content clean (tags split across chunk boundaries). +test('streamObject routes inline to the reasoning channel with clean content', async t => { + const adapter = new NativeProviderAdapter(dispatchOf(GLM_STREAM)); + let content = ''; + let reasoning = ''; + for await (const obj of adapter.streamObject({} as any)) { + if (obj.type === 'text-delta') content += obj.textDelta; + else if (obj.type === 'reasoning') reasoning += obj.textDelta; + } + t.is(content, 'Visible tail'); + t.is(reasoning, 'secret reasoning'); + t.false(content.includes('')); + t.false(content.includes('')); +}); + +// 2.3 — same stream through streamText: no tag markers leak; reasoning still +// surfaced (as a callout) for display. +test('streamText strips inline tag markers from the emitted stream', async t => { + const adapter = new NativeProviderAdapter(dispatchOf(GLM_STREAM)); + let out = ''; + for await (const chunk of adapter.streamText({} as any)) { + out += chunk; + } + t.false(out.includes('')); + t.false(out.includes('')); + t.true(out.includes('secret reasoning')); + t.true(out.includes('Visible')); +}); + +// 3.3 — adapter.text() (tool/value path) excludes reasoning entirely. +test('adapter.text() excludes reasoning (inline and separated) and yields content only', async t => { + const adapter = new NativeProviderAdapter( + dispatchOf([ + { type: 'text_delta', text: 'Answer hidden body' }, + { type: 'reasoning_delta', text: 'separated reasoning' }, + { type: 'done' }, + ]) + ); + const text = await adapter.text({} as any); + t.is(text, 'Answer body'); + t.false(text.includes('hidden')); + t.false(text.includes('separated reasoning')); +}); + +// 3.1 — extractTextResponse drops reasoning parts and strips inline . +test('extractTextResponse drops reasoning parts and strips inline from text parts', t => { + const response = { + message: { + content: [ + { type: 'reasoning', text: 'should be dropped' }, + { type: 'text', text: 'inline plan

Title

' }, + ], + }, + } as any; + const out = extractTextResponse(response); + t.is(out, '

Title

'); + t.false(out.includes('should be dropped')); + t.false(out.includes('inline plan')); +}); + +// 4.1 — end-to-end: inline-think response → extractTextResponse → code_artifact +// yields HTML with no reasoning. +test('code_artifact HTML contains no reasoning for an inline- response', async t => { + const raw = { + message: { + content: [ + { + type: 'text', + text: 'plan the page```html\n

Doc

\n```', + }, + ], + }, + } as any; + const clean = extractTextResponse(raw); + const tool = createCodeArtifactTool(async () => clean); + const result: any = await tool.execute!( + { title: 'T', userPrompt: 'x' } as any, + {} + ); + t.false(result.html.includes('')); + t.false(result.html.includes('plan the page')); + t.true(result.html.includes('

Doc

')); +}); + +// 4.2 — end-to-end: natively-separated reasoning is likewise absent from the artifact. +test('code_artifact HTML contains no reasoning for a separated-reasoning response', async t => { + const raw = { + message: { + content: [ + { type: 'reasoning', text: 'chain of thought' }, + { type: 'text', text: '

Clean

' }, + ], + }, + } as any; + const clean = extractTextResponse(raw); + const tool = createCodeArtifactTool(async () => clean); + const result: any = await tool.execute!( + { title: 'T', userPrompt: 'x' } as any, + {} + ); + t.false(result.html.includes('chain of thought')); + t.true(result.html.includes('

Clean

')); +});