feat(reconciliation): implement live Vertex AI recovery advisor and Subgraph MCP client

.agent/context/20260908T232500Z-vertex-ai-recovery-advisor.md

@@ -0,0 +1,56 @@
+# Session Context: Vertex AI Recovery Advisor and Subgraph MCP Client
+
+## Date/time
+
+- UTC: 2026-09-08T23:25:00Z
+
+## User goal
+
+Implement the live Recovery Agent (Vertex AI Gemini) and Subgraph MCP Client for candidate recovery, wire them into worker composition, verify fail-closed invariants, and execute a live recovery drill.
+
+## Original prompt/request
+
+"we can not add graph mcp without recovery agent. we need to do that first than everything else"
+
+## Assumptions
+
+- Google Cloud ADC or dynamic token retrieval provides authentication for Vertex AI REST API in `europe-west1`.
+- The Graph Gateway deployment endpoint provides fallback candidate discovery when no local MCP server process is running.
+- Invariants: external indexer absence, lag, or model errors must NEVER grant settlement permissions (`settlementPermission: 'NEVER'`). All settlements are decided solely by the OneShot deterministic safety core and verified against authoritative Arc receipts.
+- Secrets remain in Secret Manager / environment, never committed or exposed in logs.
+
+## Plan
+
+1. Implement `VertexAiRecoveryAdvisor` in `packages/reconciliation/src/vertex-advisor.ts` conforming to `RecoveryAdvisorPort` and `validateAndNormalizeRecommendation`.
+2. Implement `LiveSubgraphMcpRecoveryPort` in `packages/reconciliation/src/subgraph-mcp-client.ts` supporting both MCP JSON-RPC and direct Gateway deployment endpoints.
+3. Export both implementations from `@oneshot/reconciliation`.
+4. Add unit test suites for `VertexAiRecoveryAdvisor` and `LiveSubgraphMcpRecoveryPort`.
+5. Wire both into `apps/worker` composition and add integration tests in `p4-composition.test.ts`.
+6. Execute live recovery drill with real Vertex AI `gemini-2.5-flash` on GCP and real Arc Testnet receipt.
+7. Run repository validation checks and FreePi Gate A / Gate B.
+
+## Key decisions
+
+- Vertex AI REST `generateContent` with JSON response mode (`application/json`) is used for low overhead and no heavy SDK dependencies.
+- Subgraph MCP client wraps Gateway responses into the standard MCP tool content envelope, ensuring unified downstream normalization through `normalizeSubgraphMcpTrace`.
+- Safety core maintains total authority over settlement decisions: LLM advice is strictly advisory (`settlementPermission: 'NEVER'`).
+
+## Files/components touched
+
+- `packages/reconciliation/src/vertex-advisor.ts`: Vertex AI Gemini recovery advisor implementation.
+- `packages/reconciliation/src/subgraph-mcp-client.ts`: Live Subgraph MCP / Gateway recovery client.
+- `packages/reconciliation/src/index.ts`: Public exports for both clients.
+- `packages/reconciliation/test/vertex-advisor.test.ts`: Comprehensive tests for Vertex AI advisor (valid recommendation, HTTP error fallback, invalid JSON, prompt injection defense, fabricated ID rejection).
+- `packages/reconciliation/test/subgraph-mcp-client.test.ts`: Tests for MCP JSON-RPC and Gateway modes.
+- `apps/worker/test/p4-composition.test.ts`: End-to-end composition test verifying `ProductionRecoveryService` with `LiveSubgraphMcpRecoveryPort` and `VertexAiRecoveryAdvisor` converging an `UNKNOWN` intent to `COMMITTED` with zero external submissions.
+
+## Commands/checks
+
+- `pnpm --filter @oneshot/reconciliation test`: PASS (83 tests).
+- `pnpm --filter @oneshot/worker test`: PASS (27 tests).
+- `pnpm test`: PASS (56 test files, 878 tests).
+- `pnpm lint`: PASS (0 errors, 0 warnings).
+- `pnpm typecheck`: PASS.
+- Prettier targeted check: PASS.
+- `git diff --check`: PASS.
+- Live Drill (`scratch/live-recovery-agent-drill.mjs`): PASS. Real Gemini 2.5 Flash on Vertex AI responded in 5.3s, evaluated UNKNOWN intent, advised RECONCILE, verified on-chain against Arc Testnet receipt, marked COMMITTED with 0 duplicate broadcasts.

apps/worker/test/p4-composition.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 import type { Pool } from 'pg';
 import {
   TRANSFER_EVENT_TOPIC,
@@ -20,7 +20,9 @@ import {
 } from '@oneshot/privy-adapter';
 import {
   createRecoverySimulatorComposition,
+  LiveSubgraphMcpRecoveryPort,
   RecoveryService,
+  VertexAiRecoveryAdvisor,
   type DetailedRecoveryView,
 } from '@oneshot/reconciliation';
 import { composeWorker, createProductionRecoveryService } from '../src/composition.js';
@@ -521,4 +523,147 @@ describe('Gate P4: Backend Convergence and Adapter Replacement', () => {
     expect(stored).not.toBeNull();
     expect(stored?.externalSubmissionCount).toBe(0);
   });
+
+  it('composes ProductionRecoveryService with LiveSubgraphMcpRecoveryPort and VertexAiRecoveryAdvisor, converging UNKNOWN intent to COMMITTED with zero external submissions', async () => {
+    let ledgerState: IntentResponse['state'] = 'UNKNOWN';
+    let ledgerVersion = 3;
+    let completedWith: SettlementResult | null = null;
+
+    const mockIntent: IntentResponse = {
+      ...sampleRequest,
+      payload_fingerprint: requestFingerprint,
+      get state() {
+        return ledgerState;
+      },
+      get version() {
+        return ledgerVersion;
+      },
+      attempts: [
+        {
+          attempt_id: 'att-live-1',
+          stage: 'UNKNOWN',
+          created_at: new Date().toISOString(),
+        },
+      ],
+      evidence: [],
+    };
+
+    const mockLedger = {
+      getIntent: async () => mockIntent,
+      appendEvidence: async () => {},
+      completeSubmission: async (
+        _id: string,
+        _att: string,
+        res: SettlementResult,
+      ): Promise<CompleteSubmissionResult> => {
+        completedWith = res;
+        ledgerState = res.kind === 'CONFIRMED' ? 'COMMITTED' : 'FAILED_SAFE';
+        ledgerVersion += 1;
+        return { completed: true, state: ledgerState, version: ledgerVersion };
+      },
+    } as unknown as IntentLedger;
+
+    const candidateRecord = {
+      id: 'cand-live-1',
+      transactionHash: realTxHash,
+      logIndex: '0',
+      blockNumber: '999123',
+      blockHash: realBlockHash,
+      blockTimestamp: '1788786010',
+      network: 'eip155:5042002',
+      tokenContract: sampleConfig.usdcContract,
+      sender: realSender,
+      recipient: sampleRequest.recipient,
+      amountAtomic: sampleRequest.amount_atomic,
+      memoId: null,
+    };
+
+    const graphQlBody = {
+      data: {
+        settlementCandidates: [candidateRecord],
+        _meta: {
+          deployment: recoveryLocalState.mcpPolicy.manifestCid,
+          hasIndexingErrors: false,
+          block: {
+            number: 999125,
+            hash: realBlockHash,
+            timestamp: '1788786020',
+          },
+        },
+      },
+    };
+
+    const mockGraphFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      json: async () => graphQlBody,
+    } as Response);
+
+    const subgraphMcp = new LiveSubgraphMcpRecoveryPort({
+      graphApiKey: 'test-graph-key',
+      getChainHead: async () => ({
+        blockNumber: '999126',
+        observedAt: '2026-09-08T22:00:00.000Z',
+      }),
+      fetchFn: mockGraphFetch as unknown as typeof fetch,
+      now: () => '2026-09-08T22:00:00.000Z',
+    });
+
+    const mockVertexFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      json: async () => ({
+        candidates: [
+          {
+            content: {
+              parts: [
+                {
+                  text: JSON.stringify({
+                    action: 'RECONCILE',
+                    decisionId: 'dec-live-1',
+                    reason: 'Discovered matching candidate via Subgraph MCP',
+                    referencedEvidenceIds: ['thegraph:cand-live-1'],
+                  }),
+                },
+              ],
+            },
+          },
+        ],
+      }),
+    } as Response);
+
+    const advisor = new VertexAiRecoveryAdvisor({
+      projectId: 'oneshot-508002',
+      getAuthToken: () => 'mock-vertex-token',
+      fetchFn: mockVertexFetch as unknown as typeof fetch,
+      now: () => '2026-09-08T22:00:00.000Z',
+    });
+
+    const recoveryService = createProductionRecoveryService(mockLedger, {
+      localState: recoveryLocalState,
+      bridge: {
+        receiptSource: {
+          getReceipt: async () => realReceipt,
+        },
+        defaultArcTxHash: realTxHash,
+      },
+      subgraphMcp,
+      advisor,
+    });
+
+    const mockPool = {} as unknown as Pool;
+    const worker = composeWorker(mockPool, mockLedger, {
+      profile: 'simulator',
+      recoveryService,
+    });
+
+    const eventId = 'reconcile:live-drill:1';
+    await executeReconcileIntent(mockIntent.business_intent_id, worker.options, eventId);
+
+    expect(mockGraphFetch).toHaveBeenCalled();
+    expect(mockVertexFetch).toHaveBeenCalled();
+    expect(completedWith).not.toBeNull();
+    expect(completedWith?.kind).toBe('CONFIRMED');
+    expect(ledgerState).toBe('COMMITTED');
+  });
 });

packages/reconciliation/src/index.ts

@@ -43,3 +43,5 @@ export * from './qualification.js';
 export * from './disabled-ports.js';
 export * from './chaos/index.js';
 export * from './types.js';
+export * from './vertex-advisor.js';
+export * from './subgraph-mcp-client.js';

packages/reconciliation/src/subgraph-mcp-client.ts

@@ -0,0 +1,138 @@
+import { buildMcpToolArguments } from './query.js';
+import {
+  MCP_TOOL_NAME,
+  type IndexLookupOutcome,
+  type IndexLookupRequest,
+  type SubgraphMcpPolicy,
+  type SubgraphMcpTrace,
+} from './types.js';
+import { normalizeSubgraphMcpTrace } from './validation.js';
+import type { SubgraphMcpRecoveryPort } from './service.js';
+
+export interface LiveSubgraphMcpRecoveryPortOptions {
+  readonly mcpEndpoint?: string | undefined;
+  readonly graphGatewayBaseUrl?: string | undefined;
+  readonly graphApiKey?: string | undefined;
+  readonly getChainHead?:
+    (() => Promise<{ blockNumber: string; observedAt: string } | null>) | undefined;
+  readonly fetchFn?: typeof fetch | undefined;
+  readonly now?: (() => string) | undefined;
+}
+
+export class LiveSubgraphMcpRecoveryPort implements SubgraphMcpRecoveryPort {
+  private readonly fetch: typeof fetch;
+  private readonly now: () => string;
+
+  constructor(private readonly options: LiveSubgraphMcpRecoveryPortOptions = {}) {
+    this.fetch = options.fetchFn ?? fetch;
+    this.now = options.now ?? (() => new Date().toISOString());
+  }
+
+  async lookup(
+    request: IndexLookupRequest,
+    policy: SubgraphMcpPolicy,
+  ): Promise<IndexLookupOutcome> {
+    const toolArgs = buildMcpToolArguments(request, policy);
+    const retrievedAt = this.now();
+
+    let chainHead: { blockNumber: string; observedAt: string } | null = null;
+    if (this.options.getChainHead) {
+      try {
+        chainHead = await this.options.getChainHead();
+      } catch {
+        chainHead = null;
+      }
+    }
+
+    let rawResult: unknown;
+
+    if (this.options.mcpEndpoint) {
+      // 1. Query via MCP JSON-RPC protocol
+      const callId = `mcp-${this.now()}`;
+      const rpcPayload = {
+        jsonrpc: '2.0',
+        id: callId,
+        method: 'tools/call',
+        params: {
+          name: MCP_TOOL_NAME,
+          arguments: toolArgs,
+        },
+      };
+
+      const res = await this.fetch(this.options.mcpEndpoint, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          ...(this.options.graphApiKey
+            ? { Authorization: `Bearer ${this.options.graphApiKey}` }
+            : {}),
+        },
+        body: JSON.stringify(rpcPayload),
+      });
+
+      if (!res.ok) {
+        throw new Error(`Subgraph MCP server returned HTTP ${res.status}`);
+      }
+
+      const rpcResponse = (await res.json()) as {
+        result?: unknown;
+        error?: unknown;
+      };
+
+      if (rpcResponse.error) {
+        throw new Error(`Subgraph MCP error: ${JSON.stringify(rpcResponse.error)}`);
+      }
+
+      rawResult = rpcResponse.result;
+    } else {
+      // 2. Query Gateway deployment endpoint directly, formatted as MCP result payload
+      const gatewayBase =
+        this.options.graphGatewayBaseUrl ?? 'https://gateway-arbitrum.network.thegraph.com/api';
+      const apiKeyPart = this.options.graphApiKey ? `/${this.options.graphApiKey}` : '';
+      const endpoint = `${gatewayBase}${apiKeyPart}/deployments/id/${policy.deploymentId}`;
+
+      const res = await this.fetch(endpoint, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          query: toolArgs.query,
+          variables: toolArgs.variables,
+        }),
+      });
+
+      if (!res.ok) {
+        throw new Error(`The Graph Gateway returned HTTP ${res.status}`);
+      }
+
+      const gatewayResponse = (await res.json()) as {
+        data?: unknown;
+        errors?: unknown;
+      };
+
+      rawResult = {
+        content: [
+          {
+            type: 'text',
+            text: JSON.stringify(gatewayResponse),
+          },
+        ],
+        isError: false,
+      };
+    }
+
+    const trace: SubgraphMcpTrace = {
+      callId: `call-${retrievedAt}`,
+      serverName: policy.serverName,
+      serverVersion: policy.serverVersion,
+      toolName: MCP_TOOL_NAME,
+      arguments: toolArgs,
+      result: rawResult,
+      retrievedAt,
+      chainHead,
+    };
+
+    return normalizeSubgraphMcpTrace(request, policy, trace);
+  }
+}