diff --git a/.gitignore b/.gitignore index 39e4e8f..0007915 100644 --- a/.gitignore +++ b/.gitignore @@ -425,4 +425,11 @@ FodyWeavers.xsd /temp-schema.json /data/framework/benchmarks/ /data/framework/exec-sessions.json +/data/framework/logs/ +/data/framework/audit/ +/data/framework/metrics/ +/data/framework/requests/ +/data/framework/self-healing/ +/data/framework/incidents/open.json +/data/framework/traces/ /data/framework/traces/warm-failures/ diff --git a/benchmarks/PhantomApi.Benchmarks/RuntimeBenchmarks.cs b/benchmarks/PhantomApi.Benchmarks/RuntimeBenchmarks.cs index a94bad6..c164189 100644 --- a/benchmarks/PhantomApi.Benchmarks/RuntimeBenchmarks.cs +++ b/benchmarks/PhantomApi.Benchmarks/RuntimeBenchmarks.cs @@ -1,5 +1,6 @@ using System.Text.Json; using BenchmarkDotNet.Attributes; +using Json.Schema; [MemoryDiagnoser] public class InstructionBundleCompilerBenchmarks @@ -59,7 +60,8 @@ public int GetOrLoad_CacheHit() public class EndpointMetadataBenchmarks { private string _loginMarkdown = null!; - private JsonElement _loginContract; + private JsonSchema _loginSchema = null!; + private JsonElement _sampleResponse; private string _repoRoot = null!; [GlobalSetup] @@ -68,17 +70,33 @@ public void Setup() _repoRoot = BenchmarkFixture.RepoRoot; var endpointPath = Path.Combine(_repoRoot, "instructions", "apps", "task-board", "endpoints", "auth", "login.md"); _loginMarkdown = File.ReadAllText(endpointPath); - using var contractDocument = JsonDocument.Parse(""" + _loginSchema = JsonSchema.FromText(""" { - "sessionId": "session_123", - "user": { - "id": "user_1", - "name": "Taylor" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "token": { "type": "string" }, + "userId": { "type": "integer" }, + "fullName": { "type": "string" }, + "expiresAt": { "type": "string" }, + "error": { "type": "string" } }, - "roles": ["admin", "editor"] + "required": ["ok", "token", "userId", "fullName", "expiresAt", "error"], + "additionalProperties": false } """); - _loginContract = contractDocument.RootElement.Clone(); + using var responseDocument = JsonDocument.Parse(""" + { + "ok": true, + "token": "session_123", + "userId": 10, + "fullName": "Taylor Example", + "expiresAt": "2026-03-15T12:00:00Z", + "error": "" + } + """); + _sampleResponse = responseDocument.RootElement.Clone(); } [Benchmark(Baseline = true)] @@ -88,9 +106,10 @@ public string ParseEndpointFrontmatter() } [Benchmark] - public string NormalizeResponseExampleToSchema() + public bool ValidateResponseAgainstSchema() { - return JsonSchemaUtilities.NormalizeToSchemaJson(_loginContract); + var result = _loginSchema.Evaluate(_sampleResponse); + return result.IsValid; } [Benchmark] diff --git a/data/framework/audit/security.jsonl b/data/framework/audit/security.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/data/framework/audit/security.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/data/framework/incidents/open.json b/data/framework/incidents/open.json deleted file mode 100644 index 67af33d..0000000 --- a/data/framework/incidents/open.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "openIncidents": [] -} diff --git a/data/framework/metrics/counters.json b/data/framework/metrics/counters.json deleted file mode 100644 index 032dedb..0000000 --- a/data/framework/metrics/counters.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "totalRequests": 0, - "successfulRequests": 0, - "failedRequests": 0, - "rateLimitFailures": 0, - "authFailures": 0, - "storageFailures": 0, - "repairAttempts": 0, - "repairSuccesses": 0, - "repairFailures": 0, - "instructionRepairAttempts": 0, - "instructionRepairSuccesses": 0, - "instructionRepairFailures": 0, - "validationLoopFailures": 0, - "rollbackCount": 0 -} diff --git a/data/framework/requests/ledger.jsonl b/data/framework/requests/ledger.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/data/framework/requests/ledger.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/data/framework/self-healing/diagnoses.jsonl b/data/framework/self-healing/diagnoses.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/data/framework/self-healing/diagnoses.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/data/framework/self-healing/patches.jsonl b/data/framework/self-healing/patches.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/data/framework/self-healing/patches.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/data/framework/self-healing/rollbacks.jsonl b/data/framework/self-healing/rollbacks.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/data/framework/self-healing/rollbacks.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/data/framework/self-healing/validations.jsonl b/data/framework/self-healing/validations.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/data/framework/self-healing/validations.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/data/framework/traces/events.jsonl b/data/framework/traces/events.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/data/framework/traces/events.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/instructions/apps/bank-api/endpoints/auth/login.md b/instructions/apps/bank-api/endpoints/auth/login.md index 167c0fb..f095dca 100644 --- a/instructions/apps/bank-api/endpoints/auth/login.md +++ b/instructions/apps/bank-api/endpoints/auth/login.md @@ -32,12 +32,18 @@ Behavior rules: ```json { - "ok": true, - "token": "session_123", - "userId": 1, - "fullName": "Ada Lovelace", - "accountNumber": "HU100000000000000000000001", - "expiresAt": "2026-03-15T12:00:00Z", - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "token": { "type": "string" }, + "userId": { "type": "integer" }, + "fullName": { "type": "string" }, + "accountNumber": { "type": "string" }, + "expiresAt": { "type": "string" }, + "error": { "type": "string" } + }, + "required": ["ok", "token", "userId", "fullName", "accountNumber", "expiresAt", "error"], + "additionalProperties": false } ``` diff --git a/instructions/apps/bank-api/endpoints/bank/deposit.md b/instructions/apps/bank-api/endpoints/bank/deposit.md index 849cbc8..5e17d3c 100644 --- a/instructions/apps/bank-api/endpoints/bank/deposit.md +++ b/instructions/apps/bank-api/endpoints/bank/deposit.md @@ -26,12 +26,18 @@ Behavior rules: ```json { - "ok": true, - "userId": 1, - "accountNumber": "HU100000000000000000000001", - "amount": 200, - "balance": 1700, - "message": "Deposit completed.", - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "userId": { "type": "integer" }, + "accountNumber": { "type": "string" }, + "amount": { "type": "number" }, + "balance": { "type": "number" }, + "message": { "type": "string" }, + "error": { "type": "string" } + }, + "required": ["ok", "userId", "accountNumber", "amount", "balance", "message", "error"], + "additionalProperties": false } ``` diff --git a/instructions/apps/bank-api/endpoints/bank/get-balance.md b/instructions/apps/bank-api/endpoints/bank/get-balance.md index 79e69af..00d3760 100644 --- a/instructions/apps/bank-api/endpoints/bank/get-balance.md +++ b/instructions/apps/bank-api/endpoints/bank/get-balance.md @@ -24,11 +24,17 @@ Behavior rules: ```json { - "ok": true, - "userId": 1, - "accountNumber": "HU100000000000000000000001", - "currency": "HUF", - "balance": 1500, - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "userId": { "type": "integer" }, + "accountNumber": { "type": "string" }, + "currency": { "type": "string" }, + "balance": { "type": "number" }, + "error": { "type": "string" } + }, + "required": ["ok", "userId", "accountNumber", "currency", "balance", "error"], + "additionalProperties": false } ``` diff --git a/instructions/apps/bank-api/endpoints/bank/transfer.md b/instructions/apps/bank-api/endpoints/bank/transfer.md index c8605fa..8843b9a 100644 --- a/instructions/apps/bank-api/endpoints/bank/transfer.md +++ b/instructions/apps/bank-api/endpoints/bank/transfer.md @@ -30,13 +30,19 @@ Behavior rules: ```json { - "ok": true, - "userId": 1, - "sourceAccountNumber": "HU100000000000000000000001", - "targetAccountNumber": "HU100000000000000000000002", - "amount": 200, - "sourceBalance": 1300, - "message": "Transfer completed.", - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "userId": { "type": "integer" }, + "sourceAccountNumber": { "type": "string" }, + "targetAccountNumber": { "type": "string" }, + "amount": { "type": "number" }, + "sourceBalance": { "type": "number" }, + "message": { "type": "string" }, + "error": { "type": "string" } + }, + "required": ["ok", "userId", "sourceAccountNumber", "targetAccountNumber", "amount", "sourceBalance", "message", "error"], + "additionalProperties": false } ``` diff --git a/instructions/apps/bank-api/endpoints/bank/withdraw.md b/instructions/apps/bank-api/endpoints/bank/withdraw.md index edce115..db838e5 100644 --- a/instructions/apps/bank-api/endpoints/bank/withdraw.md +++ b/instructions/apps/bank-api/endpoints/bank/withdraw.md @@ -27,12 +27,18 @@ Behavior rules: ```json { - "ok": true, - "userId": 1, - "accountNumber": "HU100000000000000000000001", - "amount": 200, - "balance": 1300, - "message": "Withdrawal completed.", - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "userId": { "type": "integer" }, + "accountNumber": { "type": "string" }, + "amount": { "type": "number" }, + "balance": { "type": "number" }, + "message": { "type": "string" }, + "error": { "type": "string" } + }, + "required": ["ok", "userId", "accountNumber", "amount", "balance", "message", "error"], + "additionalProperties": false } ``` diff --git a/instructions/apps/task-board/endpoints/auth/login.md b/instructions/apps/task-board/endpoints/auth/login.md index cba7aac..26f0044 100644 --- a/instructions/apps/task-board/endpoints/auth/login.md +++ b/instructions/apps/task-board/endpoints/auth/login.md @@ -32,11 +32,17 @@ Behavior rules: ```json { - "ok": true, - "token": "session_123", - "userId": 10, - "fullName": "Taylor Example", - "expiresAt": "2026-03-15T12:00:00Z", - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "token": { "type": "string" }, + "userId": { "type": "integer" }, + "fullName": { "type": "string" }, + "expiresAt": { "type": "string" }, + "error": { "type": "string" } + }, + "required": ["ok", "token", "userId", "fullName", "expiresAt", "error"], + "additionalProperties": false } ``` diff --git a/instructions/apps/task-board/endpoints/tasks/create.md b/instructions/apps/task-board/endpoints/tasks/create.md index a121900..19978bd 100644 --- a/instructions/apps/task-board/endpoints/tasks/create.md +++ b/instructions/apps/task-board/endpoints/tasks/create.md @@ -27,12 +27,18 @@ Behavior rules: ```json { - "ok": true, - "taskId": 101, - "userId": 10, - "title": "Prepare backlog", - "description": "Collect the next iteration items.", - "status": "open", - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "taskId": { "type": "integer" }, + "userId": { "type": "integer" }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "status": { "type": "string" }, + "error": { "type": "string" } + }, + "required": ["ok", "taskId", "userId", "title", "description", "status", "error"], + "additionalProperties": false } ``` diff --git a/instructions/apps/task-board/endpoints/tasks/list.md b/instructions/apps/task-board/endpoints/tasks/list.md index 035acb2..f02dcd9 100644 --- a/instructions/apps/task-board/endpoints/tasks/list.md +++ b/instructions/apps/task-board/endpoints/tasks/list.md @@ -23,16 +23,28 @@ Behavior rules: ```json { - "ok": true, - "userId": 10, - "tasks": [ - { - "taskId": 100, - "title": "Prepare backlog", - "description": "Collect the next iteration items.", - "status": "open" - } - ], - "error": "" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ok": { "type": "boolean" }, + "userId": { "type": "integer" }, + "tasks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "taskId": { "type": "integer" }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "status": { "type": "string" } + }, + "required": ["taskId", "title", "description", "status"], + "additionalProperties": false + } + }, + "error": { "type": "string" } + }, + "required": ["ok", "userId", "tasks", "error"], + "additionalProperties": false } ``` diff --git a/instructions/framework/contract-discipline.md b/instructions/framework/contract-discipline.md index 24977be..ea61640 100644 --- a/instructions/framework/contract-discipline.md +++ b/instructions/framework/contract-discipline.md @@ -3,11 +3,12 @@ Response contract rules: - the first `json` code block in the selected endpoint file is the authoritative response contract +- the first `json` code block must be valid JSON Schema, not an example payload - return exactly the same property set as the contract - do not add extra properties - keep property types aligned with the contract - use the contract's error-oriented fields when something fails -- contract literal values are examples for shape and type unless the endpoint explicitly states otherwise +- do not rely on contract example inference at runtime; the schema itself is the contract Framework error contract rules: diff --git a/src/PhantomApi/CodexAppServerClient.cs b/src/PhantomApi/CodexAppServerClient.cs index e851285..13dfb4b 100644 --- a/src/PhantomApi/CodexAppServerClient.cs +++ b/src/PhantomApi/CodexAppServerClient.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.Text; using System.Text.Json; +using RuntimeProfile = (string Model, string ReasoningEffort, string? ServiceTier); sealed class CodexAppServerClient : IAsyncDisposable { @@ -440,28 +441,7 @@ private async Task StartTurnAsync( if (!string.IsNullOrWhiteSpace(outputSchemaJson)) { using var outputSchema = JsonDocument.Parse(outputSchemaJson); - if (JsonSchemaUtilities.LooksLikeJsonSchema(outputSchema.RootElement)) - { - turnParams["outputSchema"] = outputSchema.RootElement.Clone(); - } - else - { - using var derivedSchema = JsonDocument.Parse(JsonSchemaUtilities.NormalizeToSchemaJson(outputSchema.RootElement)); - turnParams["outputSchema"] = derivedSchema.RootElement.Clone(); - _ = _traceLogger.WriteEventAsync( - correlationId, - appId, - endpoint, - "appserver.output-schema.derived", - "info", - null, - detail: "Derived JSON Schema from response contract example.", - metadata: new() - { - ["method"] = "turn.start", - ["reason"] = "contract-example" - }); - } + turnParams["outputSchema"] = outputSchema.RootElement.Clone(); } using var turnCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -693,63 +673,74 @@ private void ProcessMessageLine(string line) return; } - var codexEventType = method.StartsWith("codex/event/", StringComparison.Ordinal) - ? method["codex/event/".Length..] - : null; - if (!string.IsNullOrWhiteSpace(codexEventType)) + if (TryResolveSemanticEventType(root, method, out var semanticEventType)) { - if (codexEventType is "task_complete" or "task/complete") - { - HandleTaskComplete(root); - } - else if (codexEventType is "agent_message") - { - HandleAgentMessageEvent(root); - } - else if (codexEventType is "agent_message_delta" or "agent_message_content_delta") - { - HandleAgentMessageDelta(root); - } - else if (codexEventType is "raw_response_item") - { - HandleRawResponseItem(root); - } - + DispatchSemanticEvent(semanticEventType, root); return; } + } + + private static bool TryResolveSemanticEventType(JsonElement message, string method, out string semanticEventType) + { + semanticEventType = string.Empty; - if (method == "event") + if (method is "task/complete" or "task_complete") { - if (root.TryGetProperty("params", out var eventParamsElement) && - eventParamsElement.ValueKind == JsonValueKind.Object && - eventParamsElement.TryGetProperty("type", out var eventTypeElement) && - eventTypeElement.ValueKind == JsonValueKind.String) - { - var eventType = eventTypeElement.GetString(); - if (eventType is "task_complete" or "task/complete") - { - HandleTaskComplete(root); - } - else if (eventType is "agent_message") - { - HandleAgentMessageEvent(root); - } - else if (eventType is "agent_message_delta" or "agent_message_content_delta") - { - HandleAgentMessageDelta(root); - } - else if (eventType is "raw_response_item") - { - HandleRawResponseItem(root); - } - } + semanticEventType = "task_complete"; + return true; + } - return; + if (method.StartsWith("codex/event/", StringComparison.Ordinal)) + { + return TryNormalizeSemanticEventType(method["codex/event/".Length..], out semanticEventType); } - if (method is "task/complete" or "task_complete") + if (!string.Equals(method, "event", StringComparison.Ordinal)) + { + return false; + } + + if (!message.TryGetProperty("params", out var eventParamsElement) || + eventParamsElement.ValueKind != JsonValueKind.Object || + !eventParamsElement.TryGetProperty("type", out var eventTypeElement) || + eventTypeElement.ValueKind != JsonValueKind.String) + { + return false; + } + + return TryNormalizeSemanticEventType(eventTypeElement.GetString(), out semanticEventType); + } + + private static bool TryNormalizeSemanticEventType(string? rawEventType, out string semanticEventType) + { + semanticEventType = rawEventType switch { - HandleTaskComplete(root); + "task/complete" or "task_complete" => "task_complete", + "agent_message" => "agent_message", + "agent_message_delta" or "agent_message_content_delta" => "agent_message_delta", + "raw_response_item" => "raw_response_item", + _ => string.Empty + }; + + return !string.IsNullOrWhiteSpace(semanticEventType); + } + + private void DispatchSemanticEvent(string semanticEventType, JsonElement message) + { + switch (semanticEventType) + { + case "task_complete": + HandleTaskComplete(message); + break; + case "agent_message": + HandleAgentMessageEvent(message); + break; + case "agent_message_delta": + HandleAgentMessageDelta(message); + break; + case "raw_response_item": + HandleRawResponseItem(message); + break; } } diff --git a/src/PhantomApi/CodexCliExecutionResult.cs b/src/PhantomApi/CodexCliExecutionResult.cs deleted file mode 100644 index 365ce41..0000000 --- a/src/PhantomApi/CodexCliExecutionResult.cs +++ /dev/null @@ -1 +0,0 @@ -sealed record CodexCliExecutionResult(string RawResponse, string? SessionId); diff --git a/src/PhantomApi/EndpointContractCache.cs b/src/PhantomApi/EndpointContractCache.cs index 93fb1d4..b71e823 100644 --- a/src/PhantomApi/EndpointContractCache.cs +++ b/src/PhantomApi/EndpointContractCache.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Text.Json; using System.Text.RegularExpressions; +using Json.Schema; sealed class EndpointContractCache { @@ -31,17 +32,21 @@ public ResolvedEndpointContract GetOrLoad(string appId, string endpoint) } using var responseContractDocument = JsonDocument.Parse(responseContractJson!); - var responseContract = responseContractDocument.RootElement.Clone(); - var outputSchemaWasDerived = !JsonSchemaUtilities.LooksLikeJsonSchema(responseContract); - var outputSchemaJson = JsonSchemaUtilities.NormalizeToSchemaJson(responseContract); + var outputSchemaElement = responseContractDocument.RootElement.Clone(); + if (!LooksLikeJsonSchema(outputSchemaElement)) + { + throw new InvalidOperationException( + $"The first json code block in {sourcePath.Replace('\\', '/')} must be valid JSON Schema."); + } + + var outputSchemaJson = outputSchemaElement.GetRawText(); + var outputSchema = JsonSchema.FromText(outputSchemaJson); var contract = new ResolvedEndpointContract( routeKey, sourcePath, - responseContractJson!, - responseContract, outputSchemaJson, - outputSchemaWasDerived, + outputSchema, CacheHit: false); _cache[routeKey] = new CachedContract(fingerprint, contract); @@ -103,14 +108,28 @@ private static string BuildRouteKey(string appId, string endpoint) return $"{appId}:{endpoint}"; } + private static bool LooksLikeJsonSchema(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return false; + } + + return element.TryGetProperty("$schema", out _) + || element.TryGetProperty("type", out _) + || element.TryGetProperty("properties", out _) + || element.TryGetProperty("items", out _) + || element.TryGetProperty("required", out _) + || element.TryGetProperty("$defs", out _) + || element.TryGetProperty("definitions", out _); + } + private sealed record CachedContract(string Fingerprint, ResolvedEndpointContract Contract); } sealed record ResolvedEndpointContract( string RouteKey, string SourcePath, - string ResponseContractJson, - JsonElement ResponseContract, string OutputSchemaJson, - bool OutputSchemaWasDerived, + JsonSchema OutputSchema, bool CacheHit); diff --git a/src/PhantomApi/JsonSchemaUtilities.cs b/src/PhantomApi/JsonSchemaUtilities.cs deleted file mode 100644 index f43674f..0000000 --- a/src/PhantomApi/JsonSchemaUtilities.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.Text.Json; - -static class JsonSchemaUtilities -{ - public static bool LooksLikeJsonSchema(JsonElement element) - { - if (element.ValueKind != JsonValueKind.Object) - { - return false; - } - - return element.TryGetProperty("$schema", out _) - || element.TryGetProperty("type", out _) - || element.TryGetProperty("properties", out _) - || element.TryGetProperty("items", out _) - || element.TryGetProperty("required", out _) - || element.TryGetProperty("$defs", out _) - || element.TryGetProperty("definitions", out _); - } - - public static string NormalizeToSchemaJson(JsonElement element) - { - return LooksLikeJsonSchema(element) - ? element.GetRawText() - : JsonSerializer.Serialize(BuildJsonSchemaFromExample(element)); - } - - public static object BuildJsonSchemaFromExample(JsonElement element) - { - return element.ValueKind switch - { - JsonValueKind.Object => BuildObjectSchema(element), - JsonValueKind.Array => BuildArraySchema(element), - JsonValueKind.String => new Dictionary { ["type"] = "string" }, - JsonValueKind.Number => new Dictionary { ["type"] = element.TryGetInt64(out _) ? "integer" : "number" }, - JsonValueKind.True or JsonValueKind.False => new Dictionary { ["type"] = "boolean" }, - JsonValueKind.Null => new Dictionary { ["type"] = "null" }, - _ => new Dictionary() - }; - } - - private static Dictionary BuildObjectSchema(JsonElement element) - { - var properties = new Dictionary(StringComparer.Ordinal); - var required = new List(); - foreach (var property in element.EnumerateObject()) - { - properties[property.Name] = BuildJsonSchemaFromExample(property.Value); - required.Add(property.Name); - } - - return new Dictionary - { - ["type"] = "object", - ["properties"] = properties, - ["required"] = required, - ["additionalProperties"] = false - }; - } - - private static Dictionary BuildArraySchema(JsonElement element) - { - object itemsSchema; - if (element.GetArrayLength() == 0) - { - itemsSchema = new Dictionary(); - } - else - { - itemsSchema = BuildJsonSchemaFromExample(element.EnumerateArray().First()); - } - - return new Dictionary - { - ["type"] = "array", - ["items"] = itemsSchema - }; - } -} diff --git a/src/PhantomApi/PhantomApi.csproj b/src/PhantomApi/PhantomApi.csproj index 0daaf99..aa0c469 100644 --- a/src/PhantomApi/PhantomApi.csproj +++ b/src/PhantomApi/PhantomApi.csproj @@ -7,5 +7,6 @@ + diff --git a/src/PhantomApi/PhantomStartupHostedService.cs b/src/PhantomApi/PhantomStartupHostedService.cs deleted file mode 100644 index 5ca1e44..0000000 --- a/src/PhantomApi/PhantomStartupHostedService.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Microsoft.Extensions.Hosting; - -sealed class PhantomStartupHostedService : BackgroundService -{ - private readonly PhantomStartupWork _startupWork; - - public PhantomStartupHostedService(PhantomStartupWork startupWork) - { - _startupWork = startupWork; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - await _startupWork.PrimeWarmRuntimeAsync(stoppingToken); - await _startupWork.WarmConfiguredEndpointsAsync(stoppingToken); - } - - public override async Task StopAsync(CancellationToken cancellationToken) - { - await base.StopAsync(cancellationToken); - - if (_startupWork.AppServerClient is not null) - { - await _startupWork.AppServerClient.DisposeAsync(); - } - } -} diff --git a/src/PhantomApi/PhantomStartupWork.cs b/src/PhantomApi/PhantomStartupWork.cs deleted file mode 100644 index cb39922..0000000 --- a/src/PhantomApi/PhantomStartupWork.cs +++ /dev/null @@ -1,4 +0,0 @@ -sealed record PhantomStartupWork( - CodexAppServerClient? AppServerClient, - Func PrimeWarmRuntimeAsync, - Func WarmConfiguredEndpointsAsync); diff --git a/src/PhantomApi/Program.cs b/src/PhantomApi/Program.cs index 35e32f7..d985c79 100644 --- a/src/PhantomApi/Program.cs +++ b/src/PhantomApi/Program.cs @@ -3,6 +3,9 @@ using System.Text; using System.Text.Json; using System.Text.RegularExpressions; +using Json.Schema; +using CodexCliExecutionResult = (string RawResponse, string? SessionId); +using RuntimeProfile = (string Model, string ReasoningEffort, string? ServiceTier); var builder = WebApplication.CreateBuilder(args); var repoRoot = RepoRootLocator.Resolve( @@ -62,7 +65,7 @@ if (string.IsNullOrWhiteSpace(phantomOptions.CliArgumentsTemplate)) { - phantomOptions.CliArgumentsTemplate = BuildCliArgumentsTemplate(phantomOptions, defaultProfile); + phantomOptions.CliArgumentsTemplate = CodexCliExecutor.BuildCliArgumentsTemplate(defaultProfile); } if (phantomOptions.CliTimeoutSeconds <= 0) @@ -85,19 +88,21 @@ ? new CodexAppServerClient(phantomOptions, repoRoot, traceLogger) : null; -builder.Services.AddSingleton(new PhantomStartupWork( - appServerClient, - PrimeWarmRuntimeAsync: async cancellationToken => +var app = builder.Build(); + +app.Lifetime.ApplicationStarted.Register(() => +{ + if (appServerClient is null || !phantomOptions.UseWarmAppServer) { - if (appServerClient is null || !phantomOptions.UseWarmAppServer) - { - return; - } + return; + } + _ = Task.Run(async () => + { var correlationId = $"startup_{Guid.NewGuid():N}"; try { - await appServerClient.PrimeAsync(defaultProfile, correlationId, cancellationToken); + await appServerClient.PrimeAsync(defaultProfile, correlationId, CancellationToken.None); } catch (Exception ex) { @@ -117,8 +122,12 @@ await traceLogger.WriteEventAsync( ["serviceTier"] = defaultProfile.ServiceTier ?? "default" }); } - }, - WarmConfiguredEndpointsAsync: cancellationToken => WarmConfiguredEndpointsAsync( + }); +}); + +app.Lifetime.ApplicationStarted.Register(() => +{ + _ = Task.Run(() => WarmConfiguredEndpointsAsync( repoRoot, defaultProfile, phantomOptions, @@ -127,10 +136,13 @@ await traceLogger.WriteEventAsync( endpointContractCache, execSessionPool, cliArgumentsTemplateWasProvided, - cancellationToken))); -builder.Services.AddHostedService(); + CancellationToken.None)); +}); -var app = builder.Build(); +app.Lifetime.ApplicationStopping.Register(() => +{ + _ = appServerClient?.DisposeAsync().AsTask(); +}); app.MapPost("/dynamic-api", async (HttpRequest request, CancellationToken cancellationToken) => { @@ -225,7 +237,6 @@ IResult BuildProblem(string title, string? detail, int statusCode) ["routeKey"] = $"{appId}:{endpoint}" }); - var responseContractJson = resolvedContract.ResponseContractJson; await traceLogger.WriteEventAsync( correlationId, appId, @@ -238,8 +249,7 @@ await traceLogger.WriteEventAsync( { ["routeKey"] = resolvedContract.RouteKey, ["cacheHit"] = resolvedContract.CacheHit, - ["sourcePath"] = resolvedContract.SourcePath.Replace('\\', '/'), - ["outputSchemaDerived"] = resolvedContract.OutputSchemaWasDerived + ["sourcePath"] = resolvedContract.SourcePath.Replace('\\', '/') }); var fastMode = await TraceStepAsyncResult( @@ -307,6 +317,37 @@ await traceLogger.WriteEventAsync( ? BuildExecSessionKey(appId, endpoint, runtimeProfile, instructionBundle) : null; + Task ExecuteColdAsync() + { + return TraceStepAsyncResult( + traceLogger, + correlationId, + appId, + endpoint, + "codex.exec.cold", + () => InvokeCliAsync( + phantomOptions, + runtimeProfile, + repoRoot, + rawRequest, + cancellationToken, + traceLogger, + correlationId, + appId, + endpoint, + instructionBundle, + execSessionKey, + execSessionPool, + cliArgumentsTemplateWasProvided), + null, + new() + { + ["mode"] = "cold", + ["model"] = runtimeProfile.Model, + ["reasoningEffort"] = runtimeProfile.ReasoningEffort + }); + } + string cliRawResponse; try { @@ -341,64 +382,12 @@ await traceLogger.WriteEventAsync( detail: "warm execution failed, falling back to cold execution", metadata: new() { ["error"] = Truncate(ex.Message, 240) }); - cliRawResponse = await TraceStepAsyncResult( - traceLogger, - correlationId, - appId, - endpoint, - "codex.exec.cold", - () => InvokeCliAsync( - phantomOptions, - runtimeProfile, - repoRoot, - rawRequest, - cancellationToken, - traceLogger, - correlationId, - appId, - endpoint, - instructionBundle, - execSessionKey, - execSessionPool, - cliArgumentsTemplateWasProvided), - null, - new() - { - ["mode"] = "cold", - ["model"] = runtimeProfile.Model, - ["reasoningEffort"] = runtimeProfile.ReasoningEffort - }); + cliRawResponse = await ExecuteColdAsync(); } } else { - cliRawResponse = await TraceStepAsyncResult( - traceLogger, - correlationId, - appId, - endpoint, - "codex.exec.cold", - () => InvokeCliAsync( - phantomOptions, - runtimeProfile, - repoRoot, - rawRequest, - cancellationToken, - traceLogger, - correlationId, - appId, - endpoint, - instructionBundle, - execSessionKey, - execSessionPool, - cliArgumentsTemplateWasProvided), - null, - new() - { - ["mode"] = "cold", - ["model"] = runtimeProfile.Model, - ["reasoningEffort"] = runtimeProfile.ReasoningEffort - }); + cliRawResponse = await ExecuteColdAsync(); } } catch (Exception ex) @@ -418,23 +407,28 @@ await traceLogger.WriteEventAsync( using var cliResponseCleanup = cliResponse; - var contractMatch = await TraceStepAsyncResult( + var schemaValidation = await TraceStepAsyncResult( traceLogger, correlationId, appId, endpoint, - "response.contract-match", + "response.schema-validate", () => { - var match = MatchesContract(resolvedContract.ResponseContract, cliResponse.RootElement, "$", out var contractError); - return Task.FromResult((match, contractError)); + var evaluation = resolvedContract.OutputSchema.Evaluate( + cliResponse.RootElement, + new EvaluationOptions + { + OutputFormat = OutputFormat.List + }); + return Task.FromResult((evaluation.IsValid, evaluation)); }, null, new() { ["path"] = "/dynamic-api" }); - if (!contractMatch.match) + if (!schemaValidation.IsValid) { - return BuildProblem("CLI response failed the hard guard.", contractMatch.contractError, 502); + return BuildProblem("CLI response failed the hard guard.", BuildSchemaValidationError(schemaValidation.evaluation), 502); } await traceLogger.WriteEventAsync( @@ -631,6 +625,21 @@ static async Task WarmConfiguredEndpointsAsync( foreach (var target in configuredWarmStarts) { var correlationId = $"warmstart_{Guid.NewGuid():N}"; + Task WriteWarmStartSkipAsync(string detail, Dictionary? metadata = null) + { + metadata ??= new Dictionary(); + metadata["mode"] = target.WarmStart.Mode; + return traceLogger.WriteEventAsync( + correlationId, + target.AppId, + target.Endpoint, + "warmstart.exec-session", + "skipped", + null, + detail: detail, + metadata: metadata); + } + try { var resolvedContract = await TraceStepAsyncResult( @@ -679,43 +688,19 @@ await traceLogger.WriteEventAsync( if (execSessionPool is null || !options.UseExecSessionPool) { - await traceLogger.WriteEventAsync( - correlationId, - target.AppId, - target.Endpoint, - "warmstart.exec-session", - "skipped", - null, - detail: "Exec-session warm start skipped because the exec session pool is disabled.", - metadata: new() { ["mode"] = target.WarmStart.Mode }); + await WriteWarmStartSkipAsync("Exec-session warm start skipped because the exec session pool is disabled."); continue; } if (cliArgumentsTemplateWasProvided) { - await traceLogger.WriteEventAsync( - correlationId, - target.AppId, - target.Endpoint, - "warmstart.exec-session", - "skipped", - null, - detail: "Exec-session warm start skipped because a custom CliArgumentsTemplate is configured.", - metadata: new() { ["mode"] = target.WarmStart.Mode }); + await WriteWarmStartSkipAsync("Exec-session warm start skipped because a custom CliArgumentsTemplate is configured."); continue; } if (!target.WarmStart.ReadOnlyWarmup) { - await traceLogger.WriteEventAsync( - correlationId, - target.AppId, - target.Endpoint, - "warmstart.exec-session", - "skipped", - null, - detail: "Exec-session warm start skipped because the endpoint is not marked as readonly-safe for startup warmup.", - metadata: new() { ["mode"] = target.WarmStart.Mode }); + await WriteWarmStartSkipAsync("Exec-session warm start skipped because the endpoint is not marked as readonly-safe for startup warmup."); continue; } @@ -723,15 +708,9 @@ await traceLogger.WriteEventAsync( var existingSession = await execSessionPool.GetAsync(execSessionKey, cancellationToken); if (existingSession is not null) { - await traceLogger.WriteEventAsync( - correlationId, - target.AppId, - target.Endpoint, - "warmstart.exec-session", - "skipped", - null, - detail: "Exec-session warm start skipped because a compatible stored session already exists.", - metadata: new() + await WriteWarmStartSkipAsync( + "Exec-session warm start skipped because a compatible stored session already exists.", + new() { ["sessionKey"] = execSessionKey, ["sessionId"] = existingSession.SessionId @@ -741,15 +720,7 @@ await traceLogger.WriteEventAsync( if (string.IsNullOrWhiteSpace(target.WarmStart.WarmupRequest)) { - await traceLogger.WriteEventAsync( - correlationId, - target.AppId, - target.Endpoint, - "warmstart.exec-session", - "skipped", - null, - detail: "Exec-session warm start skipped because no warmupRequest is configured.", - metadata: new() { ["mode"] = target.WarmStart.Mode }); + await WriteWarmStartSkipAsync("Exec-session warm start skipped because no warmupRequest is configured."); continue; } @@ -880,7 +851,7 @@ await traceLogger.WriteEventAsync( detail: "Session pool skipped because a custom CliArgumentsTemplate is configured."); } - var directResult = await InvokeCliProcessAsync( + var directResult = await CodexCliExecutor.ExecuteAsync( options, profile, workingDirectory, @@ -888,8 +859,7 @@ await traceLogger.WriteEventAsync( cancellationToken, traceLogger, correlationId, - resumeSessionId: null, - allowEarlyTermination: true); + resumeSessionId: null); return directResult.RawResponse; } @@ -906,7 +876,7 @@ await traceLogger.WriteEventAsync( detail: "Session pool key is already in use; using a one-off cold exec instead.", metadata: new() { ["sessionKey"] = execSessionKey }); - var busyFallbackResult = await InvokeCliProcessAsync( + var busyFallbackResult = await CodexCliExecutor.ExecuteAsync( options, profile, workingDirectory, @@ -914,8 +884,7 @@ await traceLogger.WriteEventAsync( cancellationToken, traceLogger, correlationId, - resumeSessionId: null, - allowEarlyTermination: true); + resumeSessionId: null); return busyFallbackResult.RawResponse; } @@ -939,7 +908,7 @@ await traceLogger.WriteEventAsync( try { - var resumedResult = await InvokeCliProcessAsync( + var resumedResult = await CodexCliExecutor.ExecuteAsync( options, profile, workingDirectory, @@ -947,8 +916,7 @@ await traceLogger.WriteEventAsync( cancellationToken, traceLogger, correlationId, - resumeSessionId: storedSession.SessionId, - allowEarlyTermination: true); + resumeSessionId: storedSession.SessionId); return resumedResult.RawResponse; } catch (Exception ex) when (LooksLikeInvalidResumeSession(ex.Message)) @@ -984,7 +952,7 @@ await traceLogger.WriteEventAsync( ["bundleHash"] = instructionBundle!.BundleHash }); - var freshResult = await InvokeCliProcessAsync( + var freshResult = await CodexCliExecutor.ExecuteAsync( options, profile, workingDirectory, @@ -992,8 +960,7 @@ await traceLogger.WriteEventAsync( cancellationToken, traceLogger, correlationId, - resumeSessionId: null, - allowEarlyTermination: false); + resumeSessionId: null); if (!string.IsNullOrWhiteSpace(freshResult.SessionId)) { @@ -1027,223 +994,6 @@ await traceLogger.WriteEventAsync( return freshResult.RawResponse; } -static async Task InvokeCliProcessAsync( - PhantomOptions options, - RuntimeProfile profile, - string workingDirectory, - string rawRequest, - CancellationToken cancellationToken, - TraceLogger traceLogger, - string correlationId, - string? resumeSessionId, - bool allowEarlyTermination) -{ - var stopwatch = Stopwatch.StartNew(); - var outputPath = Path.Combine(Path.GetTempPath(), $"phantomapi-{Guid.NewGuid():N}.json"); - var arguments = resumeSessionId is null - ? TokenizeArguments(BuildCliArgumentsTemplate(options, profile).Replace("{output}", QuoteArgument(outputPath))) - : BuildResumeCliArguments(profile, outputPath, resumeSessionId); - - var startInfo = new ProcessStartInfo - { - FileName = options.CliCommand, - WorkingDirectory = workingDirectory, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - foreach (var argument in arguments) - { - startInfo.ArgumentList.Add(argument); - } - - using var process = new Process { StartInfo = startInfo }; - var processStarted = false; - try - { - if (!process.Start()) - { - throw new InvalidOperationException("Failed to start the CLI process."); - } - - processStarted = true; - - await traceLogger.WriteEventAsync( - correlationId, - null, - null, - "codex.exec.cold.start", - "success", - null, - metadata: new() - { - ["model"] = profile.Model, - ["reasoningEffort"] = profile.ReasoningEffort, - ["serviceTier"] = profile.ServiceTier ?? "default", - ["resumeSessionId"] = resumeSessionId, - ["allowEarlyTermination"] = allowEarlyTermination - }); - - await process.StandardInput.WriteAsync(rawRequest); - await process.StandardInput.FlushAsync(cancellationToken); - process.StandardInput.Close(); - - var stdoutTask = process.StandardOutput.ReadToEndAsync(cancellationToken); - var stderrTask = process.StandardError.ReadToEndAsync(cancellationToken); - - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(TimeSpan.FromSeconds(options.CliTimeoutSeconds)); - var waitForExitTask = process.WaitForExitAsync(timeoutCts.Token); - var waitForOutputTask = WaitForOutputFileAsync(outputPath, timeoutCts.Token); - string? earlyOutput = null; - - try - { - var completedTask = await Task.WhenAny(waitForExitTask, waitForOutputTask); - if (completedTask == waitForOutputTask) - { - earlyOutput = await waitForOutputTask; - await traceLogger.WriteEventAsync( - correlationId, - null, - null, - "codex.exec.cold.output-ready", - "success", - stopwatch.ElapsedMilliseconds, - metadata: new() - { - ["rawResponseLength"] = earlyOutput.Length, - ["resumeSessionId"] = resumeSessionId - }); - - if (allowEarlyTermination && !process.HasExited) - { - try - { - process.Kill(entireProcessTree: true); - } - catch - { - } - } - } - - await waitForExitTask; - } - catch (OperationCanceledException) - { - if (!process.HasExited) - { - try - { - process.Kill(entireProcessTree: true); - } - catch - { - } - } - - var partialStdout = (await stdoutTask).Trim(); - var partialStderr = (await stderrTask).Trim(); - var timeoutDetail = BuildCliTimeoutDetail(partialStdout, partialStderr); - throw new TimeoutException(timeoutDetail is null - ? $"CLI call timed out after {options.CliTimeoutSeconds} seconds." - : $"CLI call timed out after {options.CliTimeoutSeconds} seconds. {timeoutDetail}"); - } - - var stdout = (await stdoutTask).Trim(); - var stderr = (await stderrTask).Trim(); - - string rawResponse; - if (!string.IsNullOrWhiteSpace(earlyOutput)) - { - rawResponse = earlyOutput; - TryDeleteFile(outputPath); - } - else if (File.Exists(outputPath)) - { - rawResponse = File.ReadAllText(outputPath).Trim().Trim('\uFEFF'); - TryDeleteFile(outputPath); - } - else - { - rawResponse = stdout; - } - - if (string.IsNullOrWhiteSpace(earlyOutput) && process.ExitCode != 0) - { - var detail = string.IsNullOrWhiteSpace(stderr) ? stdout : stderr; - throw new InvalidOperationException(string.IsNullOrWhiteSpace(detail) ? $"CLI exited with code {process.ExitCode}." : detail); - } - - if (string.IsNullOrWhiteSpace(rawResponse)) - { - throw new InvalidOperationException("CLI returned an empty response."); - } - - var parsedSessionId = TryParseSessionId(stdout, stderr); - await traceLogger.WriteEventAsync( - correlationId, - null, - null, - "codex.exec.cold.complete", - "success", - stopwatch.ElapsedMilliseconds, - metadata: new() - { - ["exitCode"] = process.ExitCode, - ["rawResponseLength"] = rawResponse.Length, - ["resumeSessionId"] = resumeSessionId, - ["parsedSessionId"] = parsedSessionId - }); - - return new CodexCliExecutionResult(rawResponse, parsedSessionId); - } - catch (Exception ex) - { - var result = processStarted ? "failed" : "failed-to-start"; - await traceLogger.WriteEventAsync( - correlationId, - null, - null, - "codex.exec.cold.complete", - result, - stopwatch.ElapsedMilliseconds, - detail: ex.Message, - error: ex.Message, - exception: ex, - metadata: new() - { - ["processStarted"] = processStarted, - ["resumeSessionId"] = resumeSessionId - }); - - throw; - } -} - -static List BuildResumeCliArguments(RuntimeProfile profile, string outputPath, string sessionId) -{ - return - [ - "--dangerously-bypass-approvals-and-sandbox", - "exec", - "resume", - "-m", - profile.Model, - "-c", - $"model_reasoning_effort={profile.ReasoningEffort}", - "--skip-git-repo-check", - "--output-last-message", - outputPath, - sessionId, - "-" - ]; -} - static string BuildExecSessionKey(string appId, string endpoint, RuntimeProfile profile, CompiledInstructionBundle instructionBundle) { return string.Join( @@ -1256,17 +1006,6 @@ static string BuildExecSessionKey(string appId, string endpoint, RuntimeProfile instructionBundle.BundleHash); } -static string? TryParseSessionId(string stdout, string stderr) -{ - var combined = string.IsNullOrWhiteSpace(stderr) - ? stdout - : $"{stdout}\n{stderr}"; - var match = Regex.Match(combined, @"session id:\s*([0-9a-fA-F-]{36})", RegexOptions.IgnoreCase); - return match.Success - ? match.Groups[1].Value - : null; -} - static bool LooksLikeInvalidResumeSession(string message) { if (string.IsNullOrWhiteSpace(message)) @@ -1282,148 +1021,6 @@ static bool LooksLikeInvalidResumeSession(string message) || message.Contains("missing", StringComparison.OrdinalIgnoreCase)); } -static List TokenizeArguments(string commandLine) -{ - var tokens = new List(); - var current = new StringBuilder(); - var inQuotes = false; - - foreach (var character in commandLine) - { - if (character == '"') - { - inQuotes = !inQuotes; - continue; - } - - if (char.IsWhiteSpace(character) && !inQuotes) - { - if (current.Length > 0) - { - tokens.Add(current.ToString()); - current.Clear(); - } - - continue; - } - - current.Append(character); - } - - if (current.Length > 0) - { - tokens.Add(current.ToString()); - } - - return tokens; -} - -static string QuoteArgument(string value) -{ - return value.Contains(' ') ? $"\"{value}\"" : value; -} - -static string BuildCliArgumentsTemplate(PhantomOptions options, RuntimeProfile profile) -{ - var arguments = new List - { - "--dangerously-bypass-approvals-and-sandbox", - "exec", - "-m", - profile.Model, - "-c", - $"model_reasoning_effort={profile.ReasoningEffort}", - "--skip-git-repo-check", - "--output-last-message", - "{output}", - "-" - }; - - return string.Join(' ', arguments.Select(QuoteArgument)); -} - -static string? BuildCliTimeoutDetail(string stdout, string stderr) -{ - var candidate = string.IsNullOrWhiteSpace(stderr) ? stdout : stderr; - if (string.IsNullOrWhiteSpace(candidate)) - { - return null; - } - - var lastLine = candidate - .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .LastOrDefault(); - - if (string.IsNullOrWhiteSpace(lastLine)) - { - return null; - } - - return $"Last CLI activity: {Truncate(lastLine, 240)}"; -} - -static async Task WaitForOutputFileAsync(string outputPath, CancellationToken cancellationToken) -{ - while (!cancellationToken.IsCancellationRequested) - { - var candidate = TryReadCompletedJsonFile(outputPath); - if (!string.IsNullOrWhiteSpace(candidate)) - { - return candidate; - } - - await Task.Delay(150, cancellationToken); - } - - throw new OperationCanceledException(cancellationToken); -} - -static string? TryReadCompletedJsonFile(string outputPath) -{ - if (!File.Exists(outputPath)) - { - return null; - } - - try - { - var raw = File.ReadAllText(outputPath).Trim().Trim('\uFEFF'); - if (string.IsNullOrWhiteSpace(raw)) - { - return null; - } - - using var _ = JsonDocument.Parse(raw); - return raw; - } - catch (IOException) - { - return null; - } - catch (UnauthorizedAccessException) - { - return null; - } - catch (JsonException) - { - return null; - } -} - -static void TryDeleteFile(string path) -{ - try - { - if (File.Exists(path)) - { - File.Delete(path); - } - } - catch - { - } -} - static string Truncate(string value, int maxLength) { if (value.Length <= maxLength) @@ -1434,93 +1031,36 @@ static string Truncate(string value, int maxLength) return value[..(maxLength - 3)] + "..."; } -static bool MatchesContract(JsonElement contract, JsonElement response, string path, out string? error) +static string BuildSchemaValidationError(EvaluationResults evaluation) { - if (contract.ValueKind == JsonValueKind.Null) - { - if (response.ValueKind == JsonValueKind.Null) - { - error = null; - return true; - } - - error = $"{path} must be null, got {response.ValueKind}."; - return false; - } - - if (contract.ValueKind == JsonValueKind.Object) - { - if (response.ValueKind != JsonValueKind.Object) - { - error = $"{path} must be an object, got {response.ValueKind}."; - return false; - } - - foreach (var contractProperty in contract.EnumerateObject()) - { - if (!response.TryGetProperty(contractProperty.Name, out var responseProperty)) - { - error = $"{path} is missing property '{contractProperty.Name}'."; - return false; - } + var errors = new List(); + CollectSchemaErrors(evaluation, errors); - if (!MatchesContract(contractProperty.Value, responseProperty, $"{path}.{contractProperty.Name}", out error)) - { - return false; - } - } - - error = null; - return true; - } + return errors.Count == 0 + ? "The response did not match the output schema." + : string.Join(" | ", errors.Take(5)); +} - if (contract.ValueKind == JsonValueKind.Array) +static void CollectSchemaErrors(EvaluationResults evaluation, List errors) +{ + if (evaluation.Errors is not null) { - if (response.ValueKind != JsonValueKind.Array) + foreach (var entry in evaluation.Errors) { - error = $"{path} must be an array, got {response.ValueKind}."; - return false; + var location = string.IsNullOrWhiteSpace(evaluation.InstanceLocation.ToString()) + ? "$" + : evaluation.InstanceLocation.ToString(); + errors.Add($"{location}: {entry.Value}"); } - - var itemTemplate = contract.EnumerateArray().FirstOrDefault(); - foreach (var responseItem in response.EnumerateArray()) - { - if (itemTemplate.ValueKind == JsonValueKind.Undefined) - { - break; - } - - if (!MatchesContract(itemTemplate, responseItem, $"{path}[]", out error)) - { - return false; - } - } - - error = null; - return true; } - if (!KindsMatch(contract.ValueKind, response.ValueKind)) + if (evaluation.Details is null) { - error = $"{path} type mismatch, expected {contract.ValueKind}, got {response.ValueKind}."; - return false; - } - - error = null; - return true; -} - -static bool KindsMatch(JsonValueKind expected, JsonValueKind actual) -{ - if (expected is JsonValueKind.True or JsonValueKind.False) - { - return actual is JsonValueKind.True or JsonValueKind.False; + return; } - if (expected == actual) + foreach (var detail in evaluation.Details) { - return true; + CollectSchemaErrors(detail, errors); } - - return false; } diff --git a/src/PhantomApi/Runtime/CodexCliExecutor.cs b/src/PhantomApi/Runtime/CodexCliExecutor.cs new file mode 100644 index 0000000..9bb20c8 --- /dev/null +++ b/src/PhantomApi/Runtime/CodexCliExecutor.cs @@ -0,0 +1,279 @@ +using System.Diagnostics; +using System.Text; +using System.Text.RegularExpressions; +using RuntimeProfile = (string Model, string ReasoningEffort, string? ServiceTier); +using CodexCliExecutionResult = (string RawResponse, string? SessionId); + +static class CodexCliExecutor +{ + public static async Task ExecuteAsync( + PhantomOptions options, + RuntimeProfile profile, + string workingDirectory, + string rawRequest, + CancellationToken cancellationToken, + TraceLogger traceLogger, + string correlationId, + string? resumeSessionId) + { + var stopwatch = Stopwatch.StartNew(); + var arguments = resumeSessionId is null + ? TokenizeArguments(BuildCliArgumentsTemplate(profile)) + : BuildResumeCliArguments(profile, resumeSessionId); + + var startInfo = new ProcessStartInfo + { + FileName = options.CliCommand, + WorkingDirectory = workingDirectory, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + using var process = new Process { StartInfo = startInfo }; + var processStarted = false; + try + { + if (!process.Start()) + { + throw new InvalidOperationException("Failed to start the CLI process."); + } + + processStarted = true; + + await traceLogger.WriteEventAsync( + correlationId, + null, + null, + "codex.exec.cold.start", + "success", + null, + metadata: new() + { + ["model"] = profile.Model, + ["reasoningEffort"] = profile.ReasoningEffort, + ["serviceTier"] = profile.ServiceTier ?? "default", + ["resumeSessionId"] = resumeSessionId + }); + + await process.StandardInput.WriteAsync(rawRequest); + await process.StandardInput.FlushAsync(cancellationToken); + process.StandardInput.Close(); + + var stdoutTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + var stderrTask = process.StandardError.ReadToEndAsync(cancellationToken); + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(options.CliTimeoutSeconds)); + var waitForExitTask = process.WaitForExitAsync(timeoutCts.Token); + + try + { + await waitForExitTask; + } + catch (OperationCanceledException) + { + if (!process.HasExited) + { + TryKillProcess(process); + } + + var partialStdout = (await stdoutTask).Trim(); + var partialStderr = (await stderrTask).Trim(); + var timeoutDetail = BuildCliTimeoutDetail(partialStdout, partialStderr); + throw new TimeoutException(timeoutDetail is null + ? $"CLI call timed out after {options.CliTimeoutSeconds} seconds." + : $"CLI call timed out after {options.CliTimeoutSeconds} seconds. {timeoutDetail}"); + } + + var stdout = (await stdoutTask).Trim(); + var stderr = (await stderrTask).Trim(); + var responsePayload = stdout; + + if (string.IsNullOrWhiteSpace(responsePayload) && process.ExitCode != 0) + { + var detail = string.IsNullOrWhiteSpace(stderr) ? stdout : stderr; + throw new InvalidOperationException(string.IsNullOrWhiteSpace(detail) ? $"CLI exited with code {process.ExitCode}." : detail); + } + + if (string.IsNullOrWhiteSpace(responsePayload)) + { + throw new InvalidOperationException("CLI returned an empty response."); + } + + var parsedSessionId = TryParseSessionId(stdout, stderr); + await traceLogger.WriteEventAsync( + correlationId, + null, + null, + "codex.exec.cold.complete", + "success", + stopwatch.ElapsedMilliseconds, + metadata: new() + { + ["exitCode"] = process.ExitCode, + ["rawResponseLength"] = responsePayload.Length, + ["resumeSessionId"] = resumeSessionId, + ["responseSource"] = "stdout", + ["parsedSessionId"] = parsedSessionId + }); + + return (RawResponse: responsePayload, SessionId: parsedSessionId); + } + catch (Exception ex) + { + var result = processStarted ? "failed" : "failed-to-start"; + await traceLogger.WriteEventAsync( + correlationId, + null, + null, + "codex.exec.cold.complete", + result, + stopwatch.ElapsedMilliseconds, + detail: ex.Message, + error: ex.Message, + exception: ex, + metadata: new() + { + ["processStarted"] = processStarted, + ["resumeSessionId"] = resumeSessionId + }); + + throw; + } + } + + public static string BuildCliArgumentsTemplate(RuntimeProfile profile) + { + var arguments = new List + { + "--dangerously-bypass-approvals-and-sandbox", + "exec", + "-m", + profile.Model, + "-c", + $"model_reasoning_effort={profile.ReasoningEffort}", + "--skip-git-repo-check", + "-" + }; + + return string.Join(' ', arguments.Select(QuoteArgument)); + } + + private static List BuildResumeCliArguments(RuntimeProfile profile, string sessionId) + { + return + [ + "--dangerously-bypass-approvals-and-sandbox", + "exec", + "resume", + "-m", + profile.Model, + "-c", + $"model_reasoning_effort={profile.ReasoningEffort}", + "--skip-git-repo-check", + sessionId, + "-" + ]; + } + + private static string? TryParseSessionId(string stdout, string stderr) + { + var combined = string.IsNullOrWhiteSpace(stderr) + ? stdout + : $"{stdout}\n{stderr}"; + var match = Regex.Match(combined, @"session id:\s*([0-9a-fA-F-]{36})", RegexOptions.IgnoreCase); + return match.Success + ? match.Groups[1].Value + : null; + } + + private static List TokenizeArguments(string commandLine) + { + var tokens = new List(); + var current = new StringBuilder(); + var inQuotes = false; + + foreach (var character in commandLine) + { + if (character == '"') + { + inQuotes = !inQuotes; + continue; + } + + if (char.IsWhiteSpace(character) && !inQuotes) + { + if (current.Length > 0) + { + tokens.Add(current.ToString()); + current.Clear(); + } + + continue; + } + + current.Append(character); + } + + if (current.Length > 0) + { + tokens.Add(current.ToString()); + } + + return tokens; + } + + private static string QuoteArgument(string value) + { + return value.Contains(' ') ? $"\"{value}\"" : value; + } + + private static string? BuildCliTimeoutDetail(string stdout, string stderr) + { + var candidate = string.IsNullOrWhiteSpace(stderr) ? stdout : stderr; + if (string.IsNullOrWhiteSpace(candidate)) + { + return null; + } + + var lastLine = candidate + .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .LastOrDefault(); + + if (string.IsNullOrWhiteSpace(lastLine)) + { + return null; + } + + return $"Last CLI activity: {Truncate(lastLine, 240)}"; + } + + private static void TryKillProcess(Process process) + { + try + { + process.Kill(entireProcessTree: true); + } + catch + { + } + } + + private static string Truncate(string value, int maxLength) + { + if (value.Length <= maxLength) + { + return value; + } + + return value[..(maxLength - 3)] + "..."; + } +} diff --git a/src/PhantomApi/RuntimeProfile.cs b/src/PhantomApi/RuntimeProfile.cs deleted file mode 100644 index 9814bca..0000000 --- a/src/PhantomApi/RuntimeProfile.cs +++ /dev/null @@ -1 +0,0 @@ -sealed record RuntimeProfile(string Model, string ReasoningEffort, string? ServiceTier);