diff --git a/runtime/csharp/.env.example b/runtime/csharp/.env.example index b46957f2..3d5e8cbe 100644 --- a/runtime/csharp/.env.example +++ b/runtime/csharp/.env.example @@ -15,5 +15,10 @@ AZURE_OPENAI_API_KEY= AZURE_OPENAI_CHAT_DEPLOYMENT= AZURE_OPENAI_EMBEDDING_DEPLOYMENT= +# Azure Entra ID (for keyless auth tests) +# Set AZURE_TENANT_ID so DefaultAzureCredential picks the correct tenant. +# Requires 'Cognitive Services OpenAI User' role on the resource. +AZURE_TENANT_ID= + # Anthropic ANTHROPIC_API_KEY= diff --git a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs index 5ac342ff..3b90abd4 100644 --- a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs @@ -406,15 +406,19 @@ public async Task TurnAsync_WithToolCalls_ExecutesTools() } [Fact] - public async Task TurnAsync_MissingTool_Throws() + public async Task TurnAsync_MissingTool_FeedsErrorBackToLlm() { InvokerRegistry.RegisterExecutor("openai", new ToolCallingExecutor()); InvokerRegistry.RegisterProcessor("openai", new ToolCallingProcessor()); var agent = CreateAgent(); agent.Tools = [new FunctionTool { Name = "get_weather", Kind = "function" }]; - await Assert.ThrowsAsync( - () => Pipeline.TurnAsync(agent)); + // With §9.9, tool errors are caught and fed back to the LLM. + // The loop continues and returns the final processed response. + var result = await Pipeline.TurnAsync(agent); + Assert.NotNull(result); + // The error is caught, the loop continues to the next LLM call which succeeds + Assert.IsType(result); } // --- Thread Expansion --- diff --git a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs new file mode 100644 index 00000000..eb498e89 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs @@ -0,0 +1,605 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +// ────────────────────────────────────────── +// Mock executor that can be configured to throw +// ────────────────────────────────────────── + +/// +/// A mock executor that tracks calls and can throw on demand. +/// Each call either throws the next queued exception or returns the next queued response. +/// +file class ThrowingMockExecutor : IExecutor +{ + private readonly Queue _responses = new(); + private readonly Queue _exceptions = new(); + + public List> Calls { get; } = []; + + public void EnqueueResponse(object response) => _responses.Enqueue(response); + public void EnqueueException(Exception ex) => _exceptions.Enqueue(ex); + + public Task ExecuteAsync(Prompty agent, List messages) + { + Calls.Add(new List(messages)); + + // Check if we should throw + if (_exceptions.Count > 0) + { + var ex = _exceptions.Dequeue(); + if (ex is not null) + return Task.FromException(ex); + } + + return Task.FromResult(_responses.Dequeue()!); + } + + public List FormatToolMessages( + object rawResponse, + List toolCalls, + List toolResults, + string? textContent = null) + { + var msgs = new List(); + msgs.Add(new Message + { + Role = Roles.Assistant, + Parts = [new TextPart { Value = textContent ?? "" }], + Metadata = new Dictionary { ["tool_calls"] = toolCalls } + }); + for (int i = 0; i < toolCalls.Count; i++) + { + msgs.Add(new Message + { + Role = Roles.Tool, + Parts = [new TextPart { Value = toolResults[i] }], + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } + }); + } + return msgs; + } +} + +/// A passthrough renderer for resilience tests. +file class PassthroughRenderer : IRenderer +{ + public Task RenderAsync(Prompty agent, string template, Dictionary inputs) + => Task.FromResult(template); +} + +/// A passthrough parser for resilience tests. +file class PassthroughParser : IParser +{ + public Task> ParseAsync(Prompty agent, string rendered) + => Task.FromResult(new List + { + new() { Role = Roles.User, Parts = [new TextPart { Value = rendered }] } + }); +} + +/// A passthrough processor for resilience tests. +file class PassthroughProcessor : IProcessor +{ + public Task ProcessAsync(Prompty agent, object response) => + Task.FromResult(response); +} + +/// Shared test helpers for resilience tests. +file static class ResilienceHelper +{ + public const string TestProvider = "test-resilience"; + + public static Prompty CreateAgent(string instructions = "Hello") + { + return new Prompty + { + Name = "test-resilience-agent", + Instructions = instructions, + Model = new Model { Id = "test-model", Provider = TestProvider }, + Template = new Template + { + Format = new FormatConfig { Kind = "test-resilience-passthrough" }, + Parser = new ParserConfig { Kind = "test-resilience-passthrough" }, + } + }; + } + + public static ThrowingMockExecutor Register(ThrowingMockExecutor? executor = null) + { + executor ??= new ThrowingMockExecutor(); + InvokerRegistry.RegisterRenderer("test-resilience-passthrough", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("test-resilience-passthrough", new PassthroughParser()); + InvokerRegistry.RegisterExecutor(TestProvider, executor); + InvokerRegistry.RegisterProcessor(TestProvider, new PassthroughProcessor()); + return executor; + } + + public static ToolCallResult MakeToolCallResult(string name, string argsJson, string id = "call_1") + { + return new ToolCallResult + { + ToolCalls = + [ + new ToolCall { Id = id, Name = name, Arguments = argsJson } + ] + }; + } +} + +// ────────────────────────────────────────── +// §9.8: Resilient JSON Parsing Tests +// ────────────────────────────────────────── + +[Collection("InvokerRegistry")] +public class ResilientJsonParsingTests +{ + [Fact] + public void ParseArguments_ValidJson_ParsesDirectly() + { + var result = ToolDispatch.ParseArguments("""{"city": "NY"}"""); + Assert.Equal("NY", result["city"]?.ToString()); + } + + [Fact] + public void ParseArguments_MarkdownFences_Stripped() + { + var result = ToolDispatch.ParseArguments("```json\n{\"city\": \"NY\"}\n```"); + Assert.Equal("NY", result["city"]?.ToString()); + } + + [Fact] + public void ParseArguments_MarkdownFencesNoLanguage_Stripped() + { + var result = ToolDispatch.ParseArguments("```\n{\"city\": \"NY\"}\n```"); + Assert.Equal("NY", result["city"]?.ToString()); + } + + [Fact] + public void ParseArguments_JsonInProse_Extracted() + { + var result = ToolDispatch.ParseArguments("Here is: {\"city\": \"NY\"} done"); + Assert.Equal("NY", result["city"]?.ToString()); + } + + [Fact] + public void ParseArguments_TrailingCommas_Cleaned() + { + var result = ToolDispatch.ParseArguments("{\"city\": \"NY\",}"); + Assert.Equal("NY", result["city"]?.ToString()); + } + + [Fact] + public void ParseArguments_TrailingCommaInArray_Cleaned() + { + var result = ToolDispatch.ParseArguments("{\"items\": [1, 2,]}"); + Assert.False(result.ContainsKey("_error")); + } + + [Fact] + public void ParseArguments_Garbage_ReturnsError() + { + var result = ToolDispatch.ParseArguments("not json at all"); + Assert.True(result.ContainsKey("_error")); + } + + [Fact] + public void ParseArguments_EmptyString_ReturnsEmptyDict() + { + var result = ToolDispatch.ParseArguments(""); + Assert.Empty(result); + } + + [Fact] + public void ParseArguments_WhitespaceOnly_ReturnsEmptyDict() + { + var result = ToolDispatch.ParseArguments(" "); + Assert.Empty(result); + } + + [Fact] + public void ExtractFirstJsonBlock_RespectsStrings() + { + var block = ToolDispatch.ExtractFirstJsonBlock("prefix {\"k\": \"v{x}\"} suffix"); + Assert.NotNull(block); + var parsed = System.Text.Json.JsonSerializer.Deserialize>(block!); + Assert.Equal("v{x}", parsed!["k"]); + } + + [Fact] + public void ExtractFirstJsonBlock_NestedObjects() + { + var block = ToolDispatch.ExtractFirstJsonBlock("text {\"a\": {\"b\": 1}} end"); + Assert.NotNull(block); + Assert.Equal("{\"a\": {\"b\": 1}}", block); + } + + [Fact] + public void ExtractFirstJsonBlock_NoBraces_ReturnsNull() + { + var block = ToolDispatch.ExtractFirstJsonBlock("no json here"); + Assert.Null(block); + } + + [Fact] + public void ExtractFirstJsonBlock_EscapedQuotes() + { + var block = ToolDispatch.ExtractFirstJsonBlock("""stuff {"key": "val\"ue"} end"""); + Assert.NotNull(block); + Assert.Contains("key", block!); + } +} + +// ────────────────────────────────────────── +// §9.9: Tool Execution Error Safety Tests +// ────────────────────────────────────────── + +[Collection("InvokerRegistry")] +public class ToolExecutionErrorSafetyTests : IDisposable +{ + public ToolExecutionErrorSafetyTests() + { + InvokerRegistry.Clear(); + ToolDispatch.ClearTools(); + ToolDispatch.ClearToolHandlers(); + } + + public void Dispose() + { + InvokerRegistry.Clear(); + ToolDispatch.ClearTools(); + ToolDispatch.ClearToolHandlers(); + } + + [Fact] + public async Task ToolDispatch_HandlerThrows_LoopContinuesWithErrorFeedback() + { + var executor = ResilienceHelper.Register(); + + // First call: executor returns tool call + executor.EnqueueResponse(ResilienceHelper.MakeToolCallResult( + "bad_tool", """{"x":"1"}""")); + // Second call: executor returns final answer (after receiving error feedback) + executor.EnqueueResponse("recovered from error"); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = + [ + new FunctionTool { Name = "bad_tool", Kind = "function" } + ]; + + var tools = new Dictionary>> + { + ["bad_tool"] = _ => throw new InvalidOperationException("boom") + }; + + var result = await Pipeline.TurnAsync(agent, tools: tools); + Assert.Equal("recovered from error", result); + // Executor was called twice: first returned tool call, second got error feedback + final answer + Assert.Equal(2, executor.Calls.Count); + } + + [Fact] + public async Task ToolDispatch_HandlerThrows_ErrorMessageSentToLlm() + { + var executor = ResilienceHelper.Register(); + + executor.EnqueueResponse(ResilienceHelper.MakeToolCallResult( + "failing_tool", """{"a":"b"}""")); + executor.EnqueueResponse("final"); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = + [ + new FunctionTool { Name = "failing_tool", Kind = "function" } + ]; + + var tools = new Dictionary>> + { + ["failing_tool"] = _ => throw new InvalidOperationException("tool went wrong") + }; + + var result = await Pipeline.TurnAsync(agent, tools: tools); + Assert.Equal("final", result); + + // The second call should have the error result in the messages + var secondCallMessages = executor.Calls[1]; + var errorMsg = secondCallMessages.FirstOrDefault(m => + m.Role == Roles.Tool && m.Text.Contains("Error:") && m.Text.Contains("tool went wrong")); + Assert.NotNull(errorMsg); + } + + [Fact] + public async Task ToolDispatch_HandlerThrows_EmitsErrorEvent() + { + var executor = ResilienceHelper.Register(); + + executor.EnqueueResponse(ResilienceHelper.MakeToolCallResult( + "err_tool", """{"x":"1"}""")); + executor.EnqueueResponse("done"); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = + [ + new FunctionTool { Name = "err_tool", Kind = "function" } + ]; + + var tools = new Dictionary>> + { + ["err_tool"] = _ => throw new InvalidOperationException("kaboom") + }; + + var events = new List<(AgentEventType Type, Dictionary Data)>(); + EventCallback onEvent = (type, data) => events.Add((type, data)); + + await Pipeline.TurnAsync(agent, tools: tools, onEvent: onEvent); + + var errorEvent = events.FirstOrDefault(e => e.Type == AgentEventType.Error); + Assert.NotEqual(default, errorEvent); + Assert.Equal("err_tool", errorEvent.Data["tool"]); + Assert.Contains("kaboom", errorEvent.Data["error"]?.ToString()); + } + + [Fact] + public async Task ToolDispatch_ParseError_ReturnsErrorStringToLlm() + { + var executor = ResilienceHelper.Register(); + + // Tool call with completely invalid JSON + executor.EnqueueResponse(new ToolCallResult + { + ToolCalls = + [ + new ToolCall { Id = "call_bad", Name = "some_tool", Arguments = "totally not json" } + ] + }); + executor.EnqueueResponse("handled gracefully"); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = + [ + new FunctionTool { Name = "some_tool", Kind = "function" } + ]; + + var tools = new Dictionary>> + { + ["some_tool"] = args => Task.FromResult("should not reach") + }; + + var result = await Pipeline.TurnAsync(agent, tools: tools); + Assert.Equal("handled gracefully", result); + + // The second call messages should contain an error about JSON parsing + var secondCallMessages = executor.Calls[1]; + var errorToolMsg = secondCallMessages.FirstOrDefault(m => + m.Role == Roles.Tool && m.Text.Contains("Error:") && m.Text.Contains("Invalid JSON")); + Assert.NotNull(errorToolMsg); + } + + [Fact] + public async Task ToolDispatch_ParallelPath_HandlerThrows_LoopContinues() + { + var executor = ResilienceHelper.Register(); + + // Return two tool calls to trigger parallel path + executor.EnqueueResponse(new ToolCallResult + { + ToolCalls = + [ + new ToolCall { Id = "call_1", Name = "good_tool", Arguments = """{"x":"1"}""" }, + new ToolCall { Id = "call_2", Name = "bad_tool", Arguments = """{"x":"2"}""" } + ] + }); + executor.EnqueueResponse("parallel recovered"); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = + [ + new FunctionTool { Name = "good_tool", Kind = "function" }, + new FunctionTool { Name = "bad_tool", Kind = "function" } + ]; + + var tools = new Dictionary>> + { + ["good_tool"] = _ => Task.FromResult("ok"), + ["bad_tool"] = _ => throw new InvalidOperationException("parallel boom") + }; + + var result = await Pipeline.TurnAsync(agent, tools: tools, parallelToolCalls: true); + Assert.Equal("parallel recovered", result); + Assert.Equal(2, executor.Calls.Count); + } +} + +// ────────────────────────────────────────── +// §9.10: LLM Call Retry Tests +// ────────────────────────────────────────── + +[Collection("InvokerRegistry")] +public class LlmCallRetryTests : IDisposable +{ + public LlmCallRetryTests() + { + InvokerRegistry.Clear(); + ToolDispatch.ClearTools(); + ToolDispatch.ClearToolHandlers(); + } + + public void Dispose() + { + InvokerRegistry.Clear(); + ToolDispatch.ClearTools(); + ToolDispatch.ClearToolHandlers(); + } + + [Fact] + public async Task LlmRetry_SucceedsOnSecondAttempt() + { + var executor = new ThrowingMockExecutor(); + InvokerRegistry.RegisterRenderer("test-resilience-passthrough", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("test-resilience-passthrough", new PassthroughParser()); + InvokerRegistry.RegisterExecutor(ResilienceHelper.TestProvider, executor); + InvokerRegistry.RegisterProcessor(ResilienceHelper.TestProvider, new PassthroughProcessor()); + + // First call throws, second succeeds + executor.EnqueueException(new HttpRequestException("Service unavailable")); + executor.EnqueueResponse("success after retry"); + + var agent = ResilienceHelper.CreateAgent(); + // Need tools or agent features to enter agent loop path + agent.Tools = [new FunctionTool { Name = "dummy", Kind = "function" }]; + + var tools = new Dictionary>> + { + ["dummy"] = s => Task.FromResult("ok") + }; + + var result = await Pipeline.TurnAsync(agent, tools: tools, maxLlmRetries: 3); + Assert.Equal("success after retry", result); + // Two executor calls: first threw, second succeeded + Assert.Equal(2, executor.Calls.Count); + } + + [Fact] + public async Task LlmRetry_EmitsStatusEvent() + { + var executor = new ThrowingMockExecutor(); + InvokerRegistry.RegisterRenderer("test-resilience-passthrough", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("test-resilience-passthrough", new PassthroughParser()); + InvokerRegistry.RegisterExecutor(ResilienceHelper.TestProvider, executor); + InvokerRegistry.RegisterProcessor(ResilienceHelper.TestProvider, new PassthroughProcessor()); + + executor.EnqueueException(new HttpRequestException("timeout")); + executor.EnqueueResponse("ok"); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = [new FunctionTool { Name = "dummy", Kind = "function" }]; + + var tools = new Dictionary>> + { + ["dummy"] = s => Task.FromResult("ok") + }; + + var events = new List<(AgentEventType Type, Dictionary Data)>(); + EventCallback onEvent = (type, data) => events.Add((type, data)); + + await Pipeline.TurnAsync(agent, tools: tools, maxLlmRetries: 3, onEvent: onEvent); + + var retryEvent = events.FirstOrDefault(e => + e.Type == AgentEventType.Status && + e.Data.ContainsKey("message") && + e.Data["message"]?.ToString()?.Contains("retrying") == true); + Assert.NotEqual(default, retryEvent); + } + + [Fact] + public async Task LlmRetry_Exhausted_ThrowsExecuteError() + { + var executor = new ThrowingMockExecutor(); + InvokerRegistry.RegisterRenderer("test-resilience-passthrough", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("test-resilience-passthrough", new PassthroughParser()); + InvokerRegistry.RegisterExecutor(ResilienceHelper.TestProvider, executor); + InvokerRegistry.RegisterProcessor(ResilienceHelper.TestProvider, new PassthroughProcessor()); + + // All attempts fail + executor.EnqueueException(new HttpRequestException("fail 1")); + executor.EnqueueException(new HttpRequestException("fail 2")); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = [new FunctionTool { Name = "dummy", Kind = "function" }]; + + var tools = new Dictionary>> + { + ["dummy"] = s => Task.FromResult("ok") + }; + + var ex = await Assert.ThrowsAsync(() => + Pipeline.TurnAsync(agent, tools: tools, maxLlmRetries: 2)); + + Assert.Contains("2 retries", ex.Message); + Assert.NotNull(ex.Messages); + Assert.NotEmpty(ex.Messages); + } + + [Fact] + public async Task LlmRetry_ExecuteError_ContainsConversationState() + { + var executor = new ThrowingMockExecutor(); + InvokerRegistry.RegisterRenderer("test-resilience-passthrough", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("test-resilience-passthrough", new PassthroughParser()); + InvokerRegistry.RegisterExecutor(ResilienceHelper.TestProvider, executor); + InvokerRegistry.RegisterProcessor(ResilienceHelper.TestProvider, new PassthroughProcessor()); + + executor.EnqueueException(new HttpRequestException("always fails")); + + var agent = ResilienceHelper.CreateAgent("My prompt"); + agent.Tools = [new FunctionTool { Name = "dummy", Kind = "function" }]; + + var tools = new Dictionary>> + { + ["dummy"] = s => Task.FromResult("ok") + }; + + var ex = await Assert.ThrowsAsync(() => + Pipeline.TurnAsync(agent, tools: tools, maxLlmRetries: 1)); + + // Messages should contain the prepared messages + Assert.NotEmpty(ex.Messages); + var hasUserMsg = ex.Messages.Any(m => m.Role == Roles.User && m.Text.Contains("My prompt")); + Assert.True(hasUserMsg); + } + + [Fact] + public async Task LlmRetry_CancellationRespected_DuringBackoff() + { + var executor = new ThrowingMockExecutor(); + InvokerRegistry.RegisterRenderer("test-resilience-passthrough", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("test-resilience-passthrough", new PassthroughParser()); + InvokerRegistry.RegisterExecutor(ResilienceHelper.TestProvider, executor); + InvokerRegistry.RegisterProcessor(ResilienceHelper.TestProvider, new PassthroughProcessor()); + + // Always fail to trigger backoff + executor.EnqueueException(new HttpRequestException("fail")); + executor.EnqueueException(new HttpRequestException("fail")); + executor.EnqueueException(new HttpRequestException("fail")); + + var agent = ResilienceHelper.CreateAgent(); + agent.Tools = [new FunctionTool { Name = "dummy", Kind = "function" }]; + + var tools = new Dictionary>> + { + ["dummy"] = s => Task.FromResult("ok") + }; + + var cts = new CancellationTokenSource(); + // Cancel quickly during backoff + cts.CancelAfter(TimeSpan.FromMilliseconds(100)); + + // Should throw OperationCanceledException or TaskCanceledException during backoff + await Assert.ThrowsAnyAsync(() => + Pipeline.TurnAsync(agent, tools: tools, maxLlmRetries: 5, + cancellationToken: cts.Token)); + } + + [Fact] + public async Task LlmRetry_SimplePath_NoRetry() + { + // Simple path (no tools, no agent features) should NOT use retry + var executor = new ThrowingMockExecutor(); + InvokerRegistry.RegisterRenderer("test-resilience-passthrough", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("test-resilience-passthrough", new PassthroughParser()); + InvokerRegistry.RegisterExecutor(ResilienceHelper.TestProvider, executor); + InvokerRegistry.RegisterProcessor(ResilienceHelper.TestProvider, new PassthroughProcessor()); + + executor.EnqueueException(new HttpRequestException("simple path fail")); + + var agent = ResilienceHelper.CreateAgent(); + // No tools — simple path + + // Should throw directly without retry + await Assert.ThrowsAsync(() => + Pipeline.TurnAsync(agent)); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs new file mode 100644 index 00000000..fab3f8e4 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs @@ -0,0 +1,292 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// +/// Tests that structured output flows correctly through the top-level pipeline +/// entry points (InvokeAsync, RunAsync, InvokeAsync<T>) with mocked providers. +/// Guards against the class of bug where invoke() returns the raw wrapper instead +/// of unwrapped clean data. +/// +[Collection("InvokerRegistry")] +public class StructuredOutputPipelineTests : IDisposable +{ + private const string StructuredJson = """{"temperature":72.5,"unit":"F","city":"Seattle"}"""; + + public StructuredOutputPipelineTests() + { + InvokerRegistry.Clear(); + InvokerRegistry.RegisterRenderer("jinja2", new PassthroughRenderer()); + InvokerRegistry.RegisterParser("prompty", new SingleMessageParser()); + } + + public void Dispose() + { + InvokerRegistry.Clear(); + } + + // ----------------------------------------------------------------------- + // InvokeAsync — returns StructuredResult (clean data, not raw LLM wrapper) + // ----------------------------------------------------------------------- + + [Fact] + public async Task InvokeAsync_WithStructuredOutput_ReturnsStructuredResult() + { + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(StructuredJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var result = await Pipeline.InvokeAsync(agent); + + Assert.IsType(result); + var sr = (StructuredResult)result; + Assert.Equal(72.5, sr["temperature"]); + Assert.Equal("F", sr["unit"]); + Assert.Equal("Seattle", sr["city"]); + } + + [Fact] + public async Task InvokeAsync_WithStructuredOutput_PreservesRawJson() + { + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(StructuredJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var result = await Pipeline.InvokeAsync(agent); + + var sr = Assert.IsType(result); + Assert.Equal(StructuredJson, sr.RawJson); + } + + [Fact] + public async Task InvokeAsync_WithStructuredOutput_ResultIsDictionary() + { + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(StructuredJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var result = await Pipeline.InvokeAsync(agent); + + // StructuredResult inherits from Dictionary — callers can use it as a dict + Assert.IsAssignableFrom>(result); + } + + // ----------------------------------------------------------------------- + // RunAsync — execute + process path + // ----------------------------------------------------------------------- + + [Fact] + public async Task RunAsync_WithStructuredOutput_ReturnsCleanData() + { + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(StructuredJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var messages = new List + { + new() { Role = Roles.User, Parts = [new TextPart { Value = "What is the weather?" }] } + }; + + var result = await Pipeline.RunAsync(agent, messages); + + var sr = Assert.IsType(result); + Assert.Equal("Seattle", sr["city"]); + Assert.Equal(72.5, sr["temperature"]); + } + + [Fact] + public async Task RunAsync_Raw_ReturnsUnprocessedResponse() + { + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(StructuredJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var messages = new List + { + new() { Role = Roles.User, Parts = [new TextPart { Value = "What is the weather?" }] } + }; + + var result = await Pipeline.RunAsync(agent, messages, raw: true); + + // raw: true skips ProcessAsync — we get the executor's raw return value + Assert.IsType(result); + Assert.Equal(StructuredJson, result); + } + + // ----------------------------------------------------------------------- + // InvokeAsync — generic typed deserialization + // ----------------------------------------------------------------------- + + public record WeatherData(double Temperature, string Unit, string City); + + [Fact] + public async Task InvokeAsyncGeneric_WithStructuredOutput_DeserializesCorrectly() + { + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(StructuredJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var weather = await Pipeline.InvokeAsync(agent); + + Assert.Equal(72.5, weather.Temperature); + Assert.Equal("F", weather.Unit); + Assert.Equal("Seattle", weather.City); + } + + [Fact] + public async Task InvokeAsyncGeneric_WithNestedStructuredOutput_DeserializesCorrectly() + { + var nestedJson = """{"location":{"city":"Portland","state":"OR"},"temp":65.0}"""; + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(nestedJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var result = await Pipeline.InvokeAsync(agent); + + Assert.NotNull(result.Location); + Assert.Equal("Portland", result.Location.City); + Assert.Equal("OR", result.Location.State); + Assert.Equal(65.0, result.Temp); + } + + public record LocationInfo(string City, string State); + public record NestedWeather(LocationInfo Location, double Temp); + + // ----------------------------------------------------------------------- + // Array structured output + // ----------------------------------------------------------------------- + + [Fact] + public async Task InvokeAsync_WithArrayStructuredOutput_ReturnsStructuredResult() + { + var arrayJson = """{"items":["rain","snow","sun"],"count":3}"""; + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor(arrayJson)); + InvokerRegistry.RegisterProcessor("openai", new StructuredOutputProcessor()); + + var agent = CreateAgentWithOutputs(); + var result = await Pipeline.InvokeAsync(agent); + + var sr = Assert.IsType(result); + var items = sr["items"] as List; + Assert.NotNull(items); + Assert.Equal(3, items!.Count); + Assert.Equal(3L, sr["count"]); + } + + // ----------------------------------------------------------------------- + // Without output schema — processor returns plain string (no wrapping) + // ----------------------------------------------------------------------- + + [Fact] + public async Task InvokeAsync_WithoutOutputSchema_ReturnsPlainString() + { + InvokerRegistry.RegisterExecutor("openai", new RawJsonExecutor("plain text response")); + InvokerRegistry.RegisterProcessor("openai", new PlainProcessor()); + + var agent = CreateAgent(); // No outputs defined + var result = await Pipeline.InvokeAsync(agent); + + Assert.IsType(result); + Assert.Equal("processed:plain text response", result); + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + private static Prompty CreateAgent(string provider = "openai") + { + var data = new Dictionary + { + ["name"] = "structured-test", + ["instructions"] = "You are a weather bot.", + ["model"] = new Dictionary + { + ["id"] = "gpt-4", + ["provider"] = provider, + }, + ["template"] = new Dictionary + { + ["format"] = new Dictionary { ["kind"] = "jinja2" }, + ["parser"] = new Dictionary { ["kind"] = "prompty" }, + }, + }; + return Prompty.Load(data, new LoadContext()); + } + + private static Prompty CreateAgentWithOutputs() + { + var agent = CreateAgent(); + agent.Outputs = + [ + new Property { Name = "temperature", Kind = "float", Description = "Temperature value" }, + new Property { Name = "unit", Kind = "string", Description = "Temperature unit" }, + new Property { Name = "city", Kind = "string", Description = "City name" }, + ]; + return agent; + } +} + +// ----------------------------------------------------------------------- +// Mock implementations for structured output tests +// ----------------------------------------------------------------------- + +/// +/// Renderer that passes through the template unchanged. +/// +internal class PassthroughRenderer : IRenderer +{ + public Task RenderAsync(Prompty agent, string template, Dictionary inputs) + => Task.FromResult(template); +} + +/// +/// Parser that wraps any rendered text into a single user message. +/// +internal class SingleMessageParser : IParser +{ + public Task> ParseAsync(Prompty agent, string rendered) + => Task.FromResult>( + [new Message { Role = Roles.User, Parts = [new TextPart { Value = rendered }] }]); +} + +/// +/// Executor that returns a raw JSON string (simulating an LLM structured output response). +/// +internal class RawJsonExecutor : IExecutor +{ + private readonly string _rawJson; + + public RawJsonExecutor(string rawJson) => _rawJson = rawJson; + + public Task ExecuteAsync(Prompty agent, List messages) + => Task.FromResult(_rawJson); + + public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) + => []; +} + +/// +/// Processor that converts raw JSON into a StructuredResult (the real OpenAI processor +/// does this when outputSchema is present). This is the critical path we're testing: +/// the pipeline must return the StructuredResult, not wrap or stringify it. +/// +internal class StructuredOutputProcessor : IProcessor +{ + public Task ProcessAsync(Prompty agent, object response) + { + var json = response as string ?? throw new InvalidOperationException("Expected string response"); + return Task.FromResult(StructuredResult.FromJson(json)); + } +} + +/// +/// Processor that returns a plain processed string (no structured output). +/// +internal class PlainProcessor : IProcessor +{ + public Task ProcessAsync(Prompty agent, object response) + => Task.FromResult($"processed:{response}"); +} diff --git a/runtime/csharp/Prompty.Core/Pipeline.cs b/runtime/csharp/Prompty.Core/Pipeline.cs index 84963b2f..6db585c0 100644 --- a/runtime/csharp/Prompty.Core/Pipeline.cs +++ b/runtime/csharp/Prompty.Core/Pipeline.cs @@ -231,7 +231,8 @@ public static async Task TurnAsync( int? contextBudget = null, Guardrails? guardrails = null, Steering? steering = null, - bool parallelToolCalls = false) + bool parallelToolCalls = false, + int maxLlmRetries = 3) { var label = turnNumber.HasValue ? $"turn {turnNumber.Value}" : "turn"; return await Trace.TraceAsync($"prompty.turn", async (emit) => @@ -330,7 +331,7 @@ public static async Task TurnAsync( AgentEvents.EmitEvent(onEvent, AgentEventType.Status, new Dictionary { ["iteration"] = iteration, ["phase"] = "executing" }); - response2 = await ExecuteAsync(agent, messages); + response2 = await InvokeWithRetryAsync(agent, messages, maxLlmRetries, onEvent, cancellationToken); // If response is a stream, consume it fully before processing. if (response2 is PromptyStream stream) @@ -386,11 +387,21 @@ public static async Task TurnAsync( tasks.Add(Task.Run(async () => { - var toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => + string toolResponse; + try { - toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - return await ToolDispatch.DispatchAsync(agent, call, tools); - }); + toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => + { + toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); + return await ToolDispatch.DispatchAsync(agent, call, tools); + }); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + toolResponse = $"Error: Tool '{call.Name}' failed: {ex.Message}"; + AgentEvents.EmitEvent(onEvent, AgentEventType.Error, + new Dictionary { ["tool"] = call.Name, ["error"] = ex.Message }); + } return (capturedIndex, toolResponse); })); } @@ -437,11 +448,21 @@ public static async Task TurnAsync( AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallStart, new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - var toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => + string toolResponse; + try + { + toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => + { + toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); + return await ToolDispatch.DispatchAsync(agent, call, tools); + }); + } + catch (Exception ex) when (ex is not OperationCanceledException) { - toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - return await ToolDispatch.DispatchAsync(agent, call, tools); - }); + toolResponse = $"Error: Tool '{call.Name}' failed: {ex.Message}"; + AgentEvents.EmitEvent(onEvent, AgentEventType.Error, + new Dictionary { ["tool"] = call.Name, ["error"] = ex.Message }); + } toolResults.Add(toolResponse); AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, @@ -514,11 +535,12 @@ public static async Task TurnAsync( int? contextBudget = null, Guardrails? guardrails = null, Steering? steering = null, - bool parallelToolCalls = false) + bool parallelToolCalls = false, + int maxLlmRetries = 3) { var agent = PromptyLoader.Load(path); return await TurnAsync(agent, inputs, tools, maxIterations, raw, turnNumber, onEvent, - cancellationToken, contextBudget, guardrails, steering, parallelToolCalls); + cancellationToken, contextBudget, guardrails, steering, parallelToolCalls, maxLlmRetries); } // ----------------------------------------------------------------------- @@ -558,10 +580,11 @@ public static async Task TurnAsync( int? contextBudget = null, Guardrails? guardrails = null, Steering? steering = null, - bool parallelToolCalls = false) + bool parallelToolCalls = false, + int maxLlmRetries = 3) { var result = await TurnAsync(agent, inputs, tools, maxIterations, raw, turnNumber, onEvent, - cancellationToken, contextBudget, guardrails, steering, parallelToolCalls); + cancellationToken, contextBudget, guardrails, steering, parallelToolCalls, maxLlmRetries); return PromptyCast.Cast(result); } @@ -580,13 +603,58 @@ public static async Task TurnAsync( int? contextBudget = null, Guardrails? guardrails = null, Steering? steering = null, - bool parallelToolCalls = false) + bool parallelToolCalls = false, + int maxLlmRetries = 3) { var result = await TurnAsync(path, inputs, tools, maxIterations, raw, turnNumber, onEvent, - cancellationToken, contextBudget, guardrails, steering, parallelToolCalls); + cancellationToken, contextBudget, guardrails, steering, parallelToolCalls, maxLlmRetries); return PromptyCast.Cast(result); } + // ----------------------------------------------------------------------- + // LLM Retry Helper (§9.10) + // ----------------------------------------------------------------------- + + /// + /// Invoke ExecuteAsync with exponential backoff retry on transient failures. + /// + private static async Task InvokeWithRetryAsync( + Prompty agent, + List messages, + int maxRetries, + EventCallback? onEvent, + CancellationToken cancellationToken) + { + var attempts = 0; + while (true) + { + try + { + return await ExecuteAsync(agent, messages); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + attempts++; + if (attempts >= maxRetries) + { + throw new ExecuteError( + $"LLM call failed after {maxRetries} retries: {ex.Message}", + new List(messages)); + } + + AgentEvents.EmitEvent(onEvent, AgentEventType.Status, + new Dictionary + { + ["message"] = $"LLM call failed, retrying (attempt {attempts + 1}/{maxRetries})..." + }); + + // Exponential backoff with jitter, capped at 60s + var backoff = Math.Min(Math.Pow(2, attempts) + Random.Shared.NextDouble(), 60); + await Task.Delay(TimeSpan.FromSeconds(backoff), cancellationToken); + } + } + } + // ----------------------------------------------------------------------- // Thread Expansion // ----------------------------------------------------------------------- @@ -677,3 +745,17 @@ public class ToolCallResult public string? Content { get; set; } public List ToolCalls { get; set; } = []; } + +/// +/// Error from agent loop that includes accumulated conversation state (§9.10). +/// +public class ExecuteError : Exception +{ + public List Messages { get; } + + public ExecuteError(string message, List messages) + : base(message) + { + Messages = messages; + } +} diff --git a/runtime/csharp/Prompty.Core/ToolDispatch.cs b/runtime/csharp/Prompty.Core/ToolDispatch.cs index f02bba0d..b2fb9a71 100644 --- a/runtime/csharp/Prompty.Core/ToolDispatch.cs +++ b/runtime/csharp/Prompty.Core/ToolDispatch.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text.RegularExpressions; + namespace Prompty.Core; /// @@ -174,6 +176,12 @@ public static async Task DispatchAsync( { var arguments = ParseArguments(call.Arguments); + // If all parse strategies failed, return error to LLM (§9.8) + if (arguments.ContainsKey("_error")) + { + return $"Error: Invalid JSON in tool arguments for '{call.Name}': {arguments["_error"]}"; + } + // Apply bindings from the tool definition var toolDef = FindToolDefinition(agent, call.Name); if (toolDef?.Bindings is not null) @@ -239,16 +247,106 @@ public static async Task DispatchAsync( } /// - /// Parse JSON arguments string into a dictionary. + /// Parse tool arguments JSON with fallback strategies per spec §9.8. + /// Strategy order: direct parse → strip markdown fences → extract JSON block → strip trailing commas. /// public static Dictionary ParseArguments(string arguments) { if (string.IsNullOrWhiteSpace(arguments)) return []; + // Strategy 1: Direct parse + var direct = TryParseJson(arguments); + if (direct is not null) return direct; + + // Strategy 2: Strip markdown code fences + var fenceMatch = Regex.Match(arguments, @"^\s*```(?:json)?\s*\n?([\s\S]*?)\n?\s*```\s*$"); + if (fenceMatch.Success) + { + var stripped = fenceMatch.Groups[1].Value; + var fenced = TryParseJson(stripped); + if (fenced is not null) + { + Console.Error.WriteLine("[prompty] Parsed tool arguments after stripping markdown fences"); + return fenced; + } + } + + // Strategy 3: Extract first balanced JSON block + var block = ExtractFirstJsonBlock(arguments); + if (block is not null) + { + var extracted = TryParseJson(block); + if (extracted is not null) + { + Console.Error.WriteLine("[prompty] Parsed tool arguments after extracting JSON block"); + return extracted; + } + } + + // Strategy 4: Strip trailing commas before } or ] + var cleaned = Regex.Replace(arguments, @",\s*([}\]])", "$1"); + if (cleaned != arguments) + { + var trailingFixed = TryParseJson(cleaned); + if (trailingFixed is not null) + { + Console.Error.WriteLine("[prompty] Parsed tool arguments after stripping trailing commas"); + return trailingFixed; + } + } + + // All strategies failed — return error marker (NOT silent empty dict) + return new Dictionary { ["_error"] = "All JSON parse strategies failed for tool arguments" }; + } + + /// + /// Extract the first balanced {...} block from text, respecting string escapes. + /// + internal static string? ExtractFirstJsonBlock(string text) + { + var start = text.IndexOf('{'); + if (start == -1) return null; + + var depth = 0; + var inString = false; + var escapeNext = false; + + for (var i = start; i < text.Length; i++) + { + var ch = text[i]; + if (escapeNext) + { + escapeNext = false; + continue; + } + if (inString) + { + if (ch == '\\') escapeNext = true; + else if (ch == '"') inString = false; + continue; + } + switch (ch) + { + case '"': inString = true; break; + case '{': depth++; break; + case '}': + depth--; + if (depth == 0) return text[start..(i + 1)]; + break; + } + } + return null; + } + + /// + /// Try to parse a JSON string into a dictionary. Returns null on failure. + /// + private static Dictionary? TryParseJson(string json) + { try { - var doc = System.Text.Json.JsonDocument.Parse(arguments); + var doc = System.Text.Json.JsonDocument.Parse(json); var result = new Dictionary(); foreach (var prop in doc.RootElement.EnumerateObject()) { @@ -266,7 +364,7 @@ public static async Task DispatchAsync( } catch (System.Text.Json.JsonException) { - return new Dictionary { ["_raw"] = arguments }; + return null; } } } diff --git a/runtime/csharp/Prompty.Providers.Tests/AgentLoopTests.cs b/runtime/csharp/Prompty.Providers.Tests/AgentLoopTests.cs index cbfc967b..4d3fbb75 100644 --- a/runtime/csharp/Prompty.Providers.Tests/AgentLoopTests.cs +++ b/runtime/csharp/Prompty.Providers.Tests/AgentLoopTests.cs @@ -177,7 +177,7 @@ public async Task TurnAsync_MaxIterations_Throws() } [Fact] - public async Task TurnAsync_MissingTool_ThrowsToolHandlerError() + public async Task TurnAsync_MissingTool_ErrorFedBackToLlm() { var executor = new SequenceExecutor( [ @@ -185,6 +185,8 @@ public async Task TurnAsync_MissingTool_ThrowsToolHandlerError() { ToolCalls = [new ToolCall { Id = "c1", Name = "missing_tool", Arguments = "{}" }], }, + // Second response after error feedback — loop continues + "recovered after missing tool", ]); InvokerRegistry.RegisterExecutor("openai", executor); @@ -193,9 +195,10 @@ public async Task TurnAsync_MissingTool_ThrowsToolHandlerError() var agent = CreateTestAgent(tools: [new FunctionTool { Name = "missing_tool", Kind = "function" }]); - // No user tools provided — function handler will throw - await Assert.ThrowsAsync( - () => Pipeline.TurnAsync(agent)); + // With §9.9, tool errors are caught and fed back to the LLM. + // The loop continues and returns the final response. + var result = await Pipeline.TurnAsync(agent); + Assert.Equal("recovered after missing tool", result); } [Fact] diff --git a/runtime/csharp/Prompty.Providers.Tests/SpecVectorAgentTests.cs b/runtime/csharp/Prompty.Providers.Tests/SpecVectorAgentTests.cs index 878d04d8..92df5b0b 100644 --- a/runtime/csharp/Prompty.Providers.Tests/SpecVectorAgentTests.cs +++ b/runtime/csharp/Prompty.Providers.Tests/SpecVectorAgentTests.cs @@ -169,15 +169,19 @@ await Assert.ThrowsAsync( } else if (errorMsg.Contains("not registered")) { - // The agent loop may throw ToolHandlerError (which extends InvalidOperationException) - // or run out of mock responses + // With §9.9 tool error safety, the tool error is caught and fed back to the LLM. + // The mock executor may then run out of responses, triggering LLM retry exhaustion. try { await Pipeline.TurnAsync(agent, tools: toolFunctions); } catch (InvalidOperationException) { - // Expected — either ToolHandlerError or mock ran out of responses + // Expected — ToolHandlerError or mock ran out of responses + } + catch (ExecuteError) + { + // Expected — LLM retries exhausted after tool error was fed back } } } diff --git a/runtime/csharp/Prompty.Providers.Tests/ToolDispatchTests.cs b/runtime/csharp/Prompty.Providers.Tests/ToolDispatchTests.cs index 2fd3b161..310d274d 100644 --- a/runtime/csharp/Prompty.Providers.Tests/ToolDispatchTests.cs +++ b/runtime/csharp/Prompty.Providers.Tests/ToolDispatchTests.cs @@ -210,11 +210,10 @@ public void ParseArguments_EmptyString_ReturnsEmptyDict() } [Fact] - public void ParseArguments_InvalidJson_ReturnsRawFallback() + public void ParseArguments_InvalidJson_ReturnsErrorFallback() { var result = ToolDispatch.ParseArguments("not json"); - Assert.True(result.ContainsKey("_raw")); - Assert.Equal("not json", result["_raw"]); + Assert.True(result.ContainsKey("_error")); } [Fact] diff --git a/runtime/python/prompty/.env.example b/runtime/python/prompty/.env.example index afc7451f..78229359 100644 --- a/runtime/python/prompty/.env.example +++ b/runtime/python/prompty/.env.example @@ -22,5 +22,10 @@ AZURE_OPENAI_CHAT_DEPLOYMENT= AZURE_OPENAI_EMBEDDING_DEPLOYMENT= AZURE_OPENAI_IMAGE_DEPLOYMENT= +# Azure Entra ID (for keyless auth tests) +# Set AZURE_TENANT_ID so DefaultAzureCredential picks the correct tenant. +# Requires 'Cognitive Services OpenAI User' role on the resource. +AZURE_TENANT_ID= + # Anthropic ANTHROPIC_API_KEY= diff --git a/runtime/python/prompty/prompty/__init__.py b/runtime/python/prompty/prompty/__init__.py index c1099e63..b1eb6fd8 100644 --- a/runtime/python/prompty/prompty/__init__.py +++ b/runtime/python/prompty/prompty/__init__.py @@ -102,6 +102,7 @@ "emit_event", "CancellationToken", "CancelledError", + "ExecuteError", "estimate_chars", "summarize_dropped", "trim_to_context_window", @@ -131,6 +132,7 @@ # Loader from .core.loader import load, load_async +from .core.pipeline import ExecuteError from .core.steering import Steering from .core.structured import StructuredResult, cast from .core.tool_decorator import bind_tools, tool diff --git a/runtime/python/prompty/prompty/core/__init__.py b/runtime/python/prompty/prompty/core/__init__.py index fc084696..8f03fa8e 100644 --- a/runtime/python/prompty/prompty/core/__init__.py +++ b/runtime/python/prompty/prompty/core/__init__.py @@ -17,6 +17,7 @@ from .guardrails import GuardrailError, GuardrailResult, Guardrails from .loader import default_save_context, load, load_async from .pipeline import ( + ExecuteError, invoke, invoke_async, prepare, diff --git a/runtime/python/prompty/prompty/core/pipeline.py b/runtime/python/prompty/prompty/core/pipeline.py index 226a6485..7fd99541 100644 --- a/runtime/python/prompty/prompty/core/pipeline.py +++ b/runtime/python/prompty/prompty/core/pipeline.py @@ -25,6 +25,8 @@ import asyncio import json +import random +import time from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from typing import Any @@ -67,6 +69,8 @@ "invoke_async", "turn", "turn_async", + # Errors + "ExecuteError", # Helpers (used by tests) "_get_rich_input_names", "_inject_thread_markers", @@ -76,6 +80,19 @@ ] +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + + +class ExecuteError(Exception): + """Error from agent loop that includes accumulated conversation state.""" + + def __init__(self, message: str, messages: list[Message] | None = None) -> None: + super().__init__(message) + self.messages = messages or [] + + # --------------------------------------------------------------------------- # Input validation # --------------------------------------------------------------------------- @@ -601,6 +618,74 @@ async def _invoke_executor_async( return await executor.execute_async(agent, messages) +# --------------------------------------------------------------------------- +# LLM call retry wrappers (spec §9.10) +# --------------------------------------------------------------------------- + + +def _invoke_with_retry( + agent: Prompty, + messages: list[Message], + max_retries: int, + on_event: EventCallback | None = None, + cancel: CancellationToken | None = None, +) -> Any: + """Call executor with retry and exponential backoff per §9.10.""" + attempts = 0 + while True: + try: + return _invoke_executor(agent, messages) + except Exception as e: + attempts += 1 + if attempts >= max_retries: + raise ExecuteError( + f"LLM call failed after {max_retries} retries: {e}", + messages=list(messages), + ) from e + # Emit status event + emit_event( + on_event, + "status", + {"message": f"LLM call failed, retrying (attempt {attempts + 1}/{max_retries})..."}, + ) + # Exponential backoff with jitter, capped at 60s + backoff = min(2**attempts + random.random(), 60) + # Check cancellation during backoff + if cancel is not None and cancel.is_cancelled: + raise + time.sleep(backoff) + + +async def _invoke_with_retry_async( + agent: Prompty, + messages: list[Message], + max_retries: int, + on_event: EventCallback | None = None, + cancel: CancellationToken | None = None, +) -> Any: + """Async variant of :func:`_invoke_with_retry`.""" + attempts = 0 + while True: + try: + return await _invoke_executor_async(agent, messages) + except Exception as e: + attempts += 1 + if attempts >= max_retries: + raise ExecuteError( + f"LLM call failed after {max_retries} retries: {e}", + messages=list(messages), + ) from e + emit_event( + on_event, + "status", + {"message": f"LLM call failed, retrying (attempt {attempts + 1}/{max_retries})..."}, + ) + backoff = min(2**attempts + random.random(), 60) + if cancel is not None and cancel.is_cancelled: + raise + await asyncio.sleep(backoff) + + # --------------------------------------------------------------------------- # Pipeline: process() # --------------------------------------------------------------------------- @@ -774,6 +859,7 @@ async def invoke_async( # --------------------------------------------------------------------------- _DEFAULT_MAX_ITERATIONS = 10 +_DEFAULT_MAX_LLM_RETRIES = 3 @trace @@ -791,6 +877,7 @@ def turn( steering: Steering | None = None, parallel_tool_calls: bool = False, target_type: type | None = None, + max_llm_retries: int = _DEFAULT_MAX_LLM_RETRIES, ) -> Any: """Conversational round-trip: prepare → [executor + tool loop] → process. @@ -830,6 +917,8 @@ def turn( If ``True``, execute multiple tool calls concurrently. target_type: If provided, cast the final result to this type via :func:`cast`. + max_llm_retries: + Maximum number of LLM call retries in the agent loop (default 3). Returns ------- @@ -842,6 +931,8 @@ def turn( If the cancellation token is triggered. GuardrailError If an input or output guardrail denies the operation. + ExecuteError + If LLM call retries are exhausted in the agent loop. ValueError If *max_iterations* is exceeded. """ @@ -909,8 +1000,8 @@ def turn( emit_event(on_event, "cancelled", {}) raise CancelledError() - # Call LLM (directly via executor, not via run) - response = _invoke_executor(agent, messages) + # Call LLM (with retry per §9.10 in agent loop) + response = _invoke_with_retry(agent, messages, max_llm_retries, on_event, cancel) # Streaming: consume through processor, extract tool calls if _is_stream(response): @@ -1033,6 +1124,7 @@ async def turn_async( steering: Steering | None = None, parallel_tool_calls: bool = False, target_type: type | None = None, + max_llm_retries: int = _DEFAULT_MAX_LLM_RETRIES, ) -> Any: """Async variant of :func:`turn`.""" from ..tracing.tracer import Tracer @@ -1102,8 +1194,8 @@ async def turn_async( emit_event(on_event, "cancelled", {}) raise CancelledError() - # Call LLM (directly via executor, not via run) - response = await _invoke_executor_async(agent, messages) + # Call LLM (with retry per §9.10 in agent loop) + response = await _invoke_with_retry_async(agent, messages, max_llm_retries, on_event, cancel) # Streaming: consume through processor, extract tool calls if _is_stream(response): @@ -1693,8 +1785,12 @@ def _dispatch_one(tc: Any) -> str: if gr.rewrite is not None: arguments = json.dumps(gr.rewrite) if isinstance(gr.rewrite, dict) else gr.rewrite - # Execute tool - result = dispatch_tool(name, arguments, tools, agent, parent_inputs) + # Execute tool (with safety net per §9.9) + try: + result = dispatch_tool(name, arguments, tools, agent, parent_inputs) + except Exception as e: + result = f"Error: Tool '{name}' failed: {type(e).__name__}: {e}" + emit_event(on_event, "error", {"tool": name, "error": str(e)}) # §13.1 — Emit tool_result emit_event(on_event, "tool_result", {"name": name, "result": result}) @@ -1745,8 +1841,12 @@ async def _dispatch_one(tc: Any) -> str: if gr.rewrite is not None: arguments = json.dumps(gr.rewrite) if isinstance(gr.rewrite, dict) else gr.rewrite - # Execute tool - result = await dispatch_tool_async(name, arguments, tools, agent, parent_inputs) + # Execute tool (with safety net per §9.9) + try: + result = await dispatch_tool_async(name, arguments, tools, agent, parent_inputs) + except Exception as e: + result = f"Error: Tool '{name}' failed: {type(e).__name__}: {e}" + emit_event(on_event, "error", {"tool": name, "error": str(e)}) # §13.1 — Emit tool_result emit_event(on_event, "tool_result", {"name": name, "result": result}) diff --git a/runtime/python/prompty/prompty/core/tool_dispatch.py b/runtime/python/prompty/prompty/core/tool_dispatch.py index 94ce38b8..9991dd6d 100644 --- a/runtime/python/prompty/prompty/core/tool_dispatch.py +++ b/runtime/python/prompty/prompty/core/tool_dispatch.py @@ -23,6 +23,8 @@ import inspect import json +import re +import warnings from collections.abc import Callable from pathlib import Path from typing import Any, Protocol, runtime_checkable @@ -43,6 +45,8 @@ "CustomToolHandler", "dispatch_tool", "dispatch_tool_async", + "_resilient_json_parse", + "_extract_first_json_block", ] @@ -368,6 +372,89 @@ async def execute_tool_async( raise NotImplementedError("Custom tool dispatch is not yet implemented") +# --------------------------------------------------------------------------- +# Resilient JSON parsing (spec §9.8) +# --------------------------------------------------------------------------- + + +def _resilient_json_parse(raw: str) -> dict | list | None: + """Parse JSON with fallback strategies per spec §9.8. + + Returns parsed value on success, None if all strategies fail. + """ + # Strategy 1: Direct parse + try: + return json.loads(raw) + except (json.JSONDecodeError, ValueError): + pass + + # Strategy 2: Strip markdown code fences + fence_match = re.match(r"^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$", raw, re.DOTALL) + if fence_match: + stripped = fence_match.group(1) + try: + result = json.loads(stripped) + warnings.warn("Parsed tool arguments after stripping markdown fences", stacklevel=2) + return result + except (json.JSONDecodeError, ValueError): + pass + + # Strategy 3: Extract first balanced JSON block + block = _extract_first_json_block(raw) + if block is not None: + try: + result = json.loads(block) + warnings.warn("Parsed tool arguments after extracting JSON block", stacklevel=2) + return result + except (json.JSONDecodeError, ValueError): + pass + + # Strategy 4: Strip trailing commas before } or ] + cleaned = re.sub(r",\s*([}\]])", r"\1", raw) + if cleaned != raw: + try: + result = json.loads(cleaned) + warnings.warn("Parsed tool arguments after stripping trailing commas", stacklevel=2) + return result + except (json.JSONDecodeError, ValueError): + pass + + return None # All strategies failed + + +def _extract_first_json_block(text: str) -> str | None: + """Extract the first balanced ``{...}`` block, respecting string escapes.""" + start = text.find("{") + if start == -1: + return None + + depth = 0 + in_string = False + escape_next = False + + for i in range(start, len(text)): + ch = text[i] + if escape_next: + escape_next = False + continue + if in_string: + if ch == "\\": + escape_next = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return text[start : i + 1] + + return None + + # --------------------------------------------------------------------------- # Main dispatch entry points # --------------------------------------------------------------------------- @@ -424,11 +511,11 @@ def dispatch_tool( str The tool result, or an error message string on failure. """ - # 1. Parse arguments - try: - args = json.loads(arguments_json) if arguments_json else {} - except json.JSONDecodeError as e: - return f"Error: invalid JSON arguments for '{tool_name}': {e}" + # 1. Parse arguments (resilient per §9.8) + parsed = _resilient_json_parse(arguments_json) if arguments_json else {} + if parsed is None: + return f"Error: Invalid JSON in tool arguments for '{tool_name}': all parse strategies failed" + args = parsed if isinstance(parsed, dict) else {"_raw": parsed} # 2. Resolve bindings if agent is not None and parent_inputs: @@ -491,11 +578,11 @@ async def dispatch_tool_async( functions and calls ``handler.execute_tool_async()`` for registered handlers. """ - # 1. Parse arguments - try: - args = json.loads(arguments_json) if arguments_json else {} - except json.JSONDecodeError as e: - return f"Error: invalid JSON arguments for '{tool_name}': {e}" + # 1. Parse arguments (resilient per §9.8) + parsed = _resilient_json_parse(arguments_json) if arguments_json else {} + if parsed is None: + return f"Error: Invalid JSON in tool arguments for '{tool_name}': all parse strategies failed" + args = parsed if isinstance(parsed, dict) else {"_raw": parsed} # 2. Resolve bindings if agent is not None and parent_inputs: diff --git a/runtime/python/prompty/prompty/providers/foundry/executor.py b/runtime/python/prompty/prompty/providers/foundry/executor.py index 2b393ce3..24886754 100644 --- a/runtime/python/prompty/prompty/providers/foundry/executor.py +++ b/runtime/python/prompty/prompty/providers/foundry/executor.py @@ -1,13 +1,13 @@ """Foundry executor — calls Azure OpenAI APIs (chat, embedding, image). -Uses ``AzureOpenAI`` from the ``openai`` package. Supports two connection modes: +Uses ``AzureOpenAI`` from the ``openai`` package. Supports three connection modes: - ``kind: key`` with ``apiKey`` — direct API key authentication. - ``kind: reference`` with ``name`` — looks up a pre-registered client via :func:`prompty.register_connection`. - -The old implicit ``DefaultAzureCredential`` fallback has been removed. -Use ``kind: reference`` with an explicitly registered client instead. +- ``kind: foundry`` — Entra ID (DefaultAzureCredential) authentication via + ``azure-identity``. No API key required; uses token-based auth with scope + ``https://cognitiveservices.azure.com/.default``. Registered as ``foundry`` in ``prompty.executors``. """ @@ -20,6 +20,7 @@ from ...core.connections import get_connection from ...model import ( ApiKeyConnection, + FoundryConnection, Prompty, ReferenceConnection, ) @@ -28,6 +29,8 @@ __all__ = ["FoundryExecutor"] +_COGNITIVE_SERVICES_SCOPE = "https://cognitiveservices.azure.com/.default" + class FoundryExecutor(_BaseExecutor): """Executor for Azure OpenAI via the Foundry provider. @@ -39,6 +42,7 @@ class FoundryExecutor(_BaseExecutor): - ``kind: key`` + ``apiKey`` → builds ``AzureOpenAI`` with the API key. - ``kind: key`` without ``apiKey`` → raises ``ValueError``. - ``kind: reference`` → looks up a pre-registered client by name. + - ``kind: foundry`` → Entra ID auth via ``DefaultAzureCredential``. """ _trace_prefix = "AzureOpenAI" @@ -82,10 +86,15 @@ def _resolve_client(self, agent: Prompty) -> Any: if isinstance(conn, ReferenceConnection): return get_connection(conn.name) + if isinstance(conn, FoundryConnection): + return self._build_client_from_entra(conn, agent) + if isinstance(conn, ApiKeyConnection): return self._build_client_from_key(conn) - raise ValueError(f"Foundry executor requires connection kind 'key' or 'reference', got: {type(conn).__name__}") + raise ValueError( + f"Foundry executor requires connection kind 'key', 'reference', or 'foundry', got: {type(conn).__name__}" + ) def _resolve_client_async(self, agent: Prompty) -> Any: """Resolve the async Azure OpenAI client from connection config.""" @@ -94,10 +103,15 @@ def _resolve_client_async(self, agent: Prompty) -> Any: if isinstance(conn, ReferenceConnection): return get_connection(conn.name) + if isinstance(conn, FoundryConnection): + return self._build_async_client_from_entra(conn, agent) + if isinstance(conn, ApiKeyConnection): return self._build_async_client_from_key(conn) - raise ValueError(f"Foundry executor requires connection kind 'key' or 'reference', got: {type(conn).__name__}") + raise ValueError( + f"Foundry executor requires connection kind 'key', 'reference', or 'foundry', got: {type(conn).__name__}" + ) def _build_client_from_key(self, conn: ApiKeyConnection) -> Any: """Build a sync AzureOpenAI client from an ApiKeyConnection.""" @@ -160,3 +174,61 @@ def _build_async_client_from_key(self, conn: ApiKeyConnection) -> Any: **kwargs, ) return client + + def _build_client_from_entra(self, conn: FoundryConnection, agent: Prompty) -> Any: + """Build a sync AzureOpenAI client using Entra ID (DefaultAzureCredential).""" + from azure.identity import DefaultAzureCredential, get_bearer_token_provider + from openai import AzureOpenAI + + credential = DefaultAzureCredential() + token_provider = get_bearer_token_provider(credential, _COGNITIVE_SERVICES_SCOPE) + + kwargs: dict[str, Any] = { + "azure_ad_token_provider": token_provider, + "api_version": "2024-12-01-preview", + } + if conn.endpoint: + kwargs["azure_endpoint"] = conn.endpoint + if agent.model.id: + kwargs["azure_deployment"] = agent.model.id + + with Tracer.start("AzureOpenAI(EntraID)") as t: + t("type", "LLM") + t("signature", "AzureOpenAI.ctor(EntraID)") + client = AzureOpenAI( + default_headers={ + "User-Agent": f"prompty/{VERSION}", + "x-ms-useragent": f"prompty/{VERSION}", + }, + **kwargs, + ) + return client + + def _build_async_client_from_entra(self, conn: FoundryConnection, agent: Prompty) -> Any: + """Build an async AsyncAzureOpenAI client using Entra ID (DefaultAzureCredential).""" + from azure.identity import DefaultAzureCredential, get_bearer_token_provider + from openai import AsyncAzureOpenAI + + credential = DefaultAzureCredential() + token_provider = get_bearer_token_provider(credential, _COGNITIVE_SERVICES_SCOPE) + + kwargs: dict[str, Any] = { + "azure_ad_token_provider": token_provider, + "api_version": "2024-12-01-preview", + } + if conn.endpoint: + kwargs["azure_endpoint"] = conn.endpoint + if agent.model.id: + kwargs["azure_deployment"] = agent.model.id + + with Tracer.start("AsyncAzureOpenAI(EntraID)") as t: + t("type", "LLM") + t("signature", "AsyncAzureOpenAI.ctor(EntraID)") + client = AsyncAzureOpenAI( + default_headers={ + "User-Agent": f"prompty/{VERSION}", + "x-ms-useragent": f"prompty/{VERSION}", + }, + **kwargs, + ) + return client diff --git a/runtime/python/prompty/tests/integration/conftest.py b/runtime/python/prompty/tests/integration/conftest.py index 35bf3a88..3ce73d33 100644 --- a/runtime/python/prompty/tests/integration/conftest.py +++ b/runtime/python/prompty/tests/integration/conftest.py @@ -59,6 +59,7 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: has_openai = bool(_OPENAI_KEY) has_azure = bool(_AZURE_KEY and _AZURE_ENDPOINT and _AZURE_CHAT_DEPLOYMENT) has_foundry = has_azure # Foundry uses Azure OpenAI credentials +has_entra = bool(_AZURE_ENDPOINT and _AZURE_CHAT_DEPLOYMENT) # Entra ID: endpoint + deployment, no API key needed has_anthropic = bool(_ANTHROPIC_KEY) has_direct_openai = bool(_DIRECT_OPENAI_KEY) @@ -81,6 +82,10 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: skip_azure_image = skip_foundry_image # backward-compat alias skip_anthropic = pytest.mark.skipif(not has_anthropic, reason="ANTHROPIC_API_KEY not set") skip_direct_openai = pytest.mark.skipif(not has_direct_openai, reason="DIRECT_OPENAI_API_KEY not set") +skip_entra = pytest.mark.skipif( + not has_entra, + reason="AZURE_OPENAI_ENDPOINT or AZURE_OPENAI_CHAT_DEPLOYMENT not set (Entra ID tests)", +) # --------------------------------------------------------------------------- @@ -262,3 +267,47 @@ def make_anthropic_agent( if metadata is not None: data["metadata"] = metadata return Prompty.load(data) + + +def make_entra_agent( + *, + api_type: str = "chat", + deployment: str | None = None, + options: dict[str, Any] | None = None, + tools: list[dict[str, Any]] | None = None, + output_schema: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, +) -> Any: + """Build a Prompty for Azure OpenAI via Entra ID (DefaultAzureCredential). + + Uses ``FoundryConnection`` (``kind: foundry``) with no API key — authenticates + via ``DefaultAzureCredential`` from ``azure-identity``. + """ + from prompty.model import Prompty + + if deployment is None: + deployment = _AZURE_CHAT_DEPLOYMENT + + data: dict[str, Any] = { + "name": "integration-test-entra", + "model": { + "id": deployment, + "provider": "foundry", + "apiType": api_type, + "connection": { + "kind": "foundry", + "endpoint": _AZURE_ENDPOINT, + }, + }, + } + if options: + data["model"]["options"] = options + if tools: + data["tools"] = tools + if output_schema: + data["outputs"] = ( + output_schema.get("properties", output_schema) if isinstance(output_schema, dict) else output_schema + ) + if metadata is not None: + data["metadata"] = metadata + return Prompty.load(data) diff --git a/runtime/python/prompty/tests/integration/test_entra_id.py b/runtime/python/prompty/tests/integration/test_entra_id.py new file mode 100644 index 00000000..b8ed2cfd --- /dev/null +++ b/runtime/python/prompty/tests/integration/test_entra_id.py @@ -0,0 +1,112 @@ +"""Integration tests — Entra ID (Azure AD) authentication against Azure OpenAI via Foundry provider. + +Uses ``AzureOpenAI`` with ``DefaultAzureCredential`` (no API key). +Requires ``AZURE_OPENAI_ENDPOINT`` and ``AZURE_OPENAI_CHAT_DEPLOYMENT`` env vars. +Skips gracefully when Azure credentials are not available (run ``az login`` first). +""" + +from __future__ import annotations + +import pytest + +from prompty.core.types import Message, TextPart +from prompty.providers.foundry.executor import FoundryExecutor +from prompty.providers.foundry.processor import FoundryProcessor + +from .conftest import make_entra_agent, skip_entra + + +def _hello_messages() -> list[Message]: + return [ + Message( + role="system", + parts=[TextPart(value="You are a helpful assistant. Reply in one short sentence.")], + ), + Message(role="user", parts=[TextPart(value="Say hello.")]), + ] + + +# --------------------------------------------------------------------------- +# Entra ID (DefaultAzureCredential) — no API key +# --------------------------------------------------------------------------- + + +@skip_entra +class TestEntraId: + executor = FoundryExecutor() + processor = FoundryProcessor() + + def test_entra_id_token_acquisition(self): + """Verify DefaultAzureCredential can acquire a token for Cognitive Services.""" + try: + from azure.identity import DefaultAzureCredential + except ImportError: + pytest.skip("azure-identity not installed") + + credential = DefaultAzureCredential() + try: + token = credential.get_token("https://cognitiveservices.azure.com/.default") + except Exception as exc: + pytest.skip(f"DefaultAzureCredential failed — run `az login` first. ({exc})") + + assert token.token, "Token should not be empty" + assert token.expires_on > 0, "Token should have a valid expiration" + + def test_entra_id_chat_completion(self): + """Chat completion using Entra ID auth (FoundryConnection, no apiKey).""" + agent = make_entra_agent(options={"maxOutputTokens": 50, "temperature": 0}) + messages = _hello_messages() + + try: + response = self.executor.execute(agent, messages) + except ImportError: + pytest.skip("azure-identity not installed") + except Exception as exc: + _skip_on_auth_error(exc) + raise + + result = self.processor.process(agent, response) + assert isinstance(result, str) + assert len(result) > 0 + + @pytest.mark.asyncio + async def test_entra_id_chat_completion_async(self): + """Async chat completion using Entra ID auth.""" + agent = make_entra_agent(options={"maxOutputTokens": 50, "temperature": 0}) + messages = _hello_messages() + + try: + response = await self.executor.execute_async(agent, messages) + except ImportError: + pytest.skip("azure-identity not installed") + except Exception as exc: + _skip_on_auth_error(exc) + raise + + result = await self.processor.process_async(agent, response) + assert isinstance(result, str) + assert len(result) > 0 + + +def _skip_on_auth_error(exc: Exception) -> None: + """Skip the test with a helpful message if the error is auth-related.""" + exc_type = type(exc).__name__ + + # azure.identity.CredentialUnavailableError or AuthenticationError + if "Credential" in exc_type or "Authentication" in exc_type: + pytest.skip(f"Azure Entra ID credentials not available — run `az login` first. ({exc})") + + # openai.AuthenticationError (HTTP 401/403) + if "AuthenticationError" in exc_type or "PermissionDenied" in exc_type: + pytest.skip( + f"Azure Entra ID authorization failed — ensure your identity has " + f"'Cognitive Services OpenAI User' role on the resource. ({exc})" + ) + + # HTTP status code check for generic API errors + status = getattr(exc, "status_code", None) or getattr(exc, "status", None) + if status in (401, 403): + pytest.skip( + f"Azure Entra ID authorization failed (HTTP {status}) — ensure your identity has " + f"'Cognitive Services OpenAI User' role on the resource. ({exc})" + ) diff --git a/runtime/python/prompty/tests/test_invoker.py b/runtime/python/prompty/tests/test_invoker.py index ebc816fc..880dc8c4 100644 --- a/runtime/python/prompty/tests/test_invoker.py +++ b/runtime/python/prompty/tests/test_invoker.py @@ -829,3 +829,139 @@ def test_ignores_string(self): agent = _make_agent(inputs=[p]) assert _get_rich_input_names(agent) == {} + + +# --------------------------------------------------------------------------- +# Tests: Structured output through the pipeline +# --------------------------------------------------------------------------- + + +class MockStructuredProcessor: + """Processor that returns a parsed dict, simulating structured output.""" + + def __init__(self, structured_data: dict[str, Any]) -> None: + self._data = structured_data + + def process(self, agent: Any, response: Any) -> dict[str, Any]: + return self._data + + async def process_async(self, agent: Any, response: Any) -> dict[str, Any]: + return self.process(agent, response) + + +class TestStructuredOutputPipeline: + """Ensure invoke() properly passes through structured output from the processor. + + This catches the class of bug found in Rust where invoke() unwrapped or + dropped the structured result before returning it to the caller. + """ + + STRUCTURED_DATA: dict[str, Any] = { + "name": "Ada Lovelace", + "age": 36, + "interests": ["mathematics", "computing"], + } + + def _make_structured_agent(self) -> Any: + """Create an agent with outputs defined (triggers structured output path).""" + from prompty.model import Property + + out_prop = Property() + out_prop.name = "name" + out_prop.kind = "string" + + agent = _make_agent( + instructions="system:\nExtract info.\n\nuser:\n{{text}}", + provider="openai", + ) + agent.outputs = [out_prop] + return agent + + def _patch_all_structured(self): + return _patch_entry_points( + renderers=[("jinja2", MockRenderer)], + parsers=[("prompty", MockParser)], + executors=[("openai", MockExecutor)], + processors=[("openai", MockStructuredProcessor(self.STRUCTURED_DATA))], + ) + + def test_invoke_structured_output_with_mocks(self): + """Full pipeline invoke() returns the structured dict from the processor.""" + agent = self._make_structured_agent() + + with self._patch_all_structured(): + result = invoke(agent, {"text": "Ada was born in 1815"}) + + assert isinstance(result, dict) + assert result == self.STRUCTURED_DATA + assert result["name"] == "Ada Lovelace" + assert result["age"] == 36 + assert result["interests"] == ["mathematics", "computing"] + + @pytest.mark.asyncio + async def test_invoke_structured_output_async_with_mocks(self): + """Async invoke_async() also returns the structured dict unchanged.""" + from prompty.invoker import invoke_async + + agent = self._make_structured_agent() + + with self._patch_all_structured(): + result = await invoke_async(agent, {"text": "Ada was born in 1815"}) + + assert isinstance(result, dict) + assert result == self.STRUCTURED_DATA + assert result["name"] == "Ada Lovelace" + + def test_invoke_structured_output_not_unwrapped(self): + """Verify the pipeline does not accidentally unwrap a nested dict result.""" + nested_data: dict[str, Any] = { + "person": {"name": "Ada", "age": 36}, + "meta": {"source": "test"}, + } + agent = self._make_structured_agent() + + patch = _patch_entry_points( + renderers=[("jinja2", MockRenderer)], + parsers=[("prompty", MockParser)], + executors=[("openai", MockExecutor)], + processors=[("openai", MockStructuredProcessor(nested_data))], + ) + with patch: + result = invoke(agent, {"text": "test"}) + + assert result is not None + assert isinstance(result, dict) + assert result["person"]["name"] == "Ada" + assert result["meta"]["source"] == "test" + + def test_invoke_structured_output_with_target_type(self): + """invoke() with target_type casts the structured result.""" + from dataclasses import dataclass + + @dataclass + class Person: + name: str + age: int + interests: list[str] + + agent = self._make_structured_agent() + + with self._patch_all_structured(): + result = invoke(agent, {"text": "test"}, target_type=Person) + + assert isinstance(result, Person) + assert result.name == "Ada Lovelace" + assert result.age == 36 + + def test_run_structured_output_with_mocks(self): + """run() (executor + process) also passes through structured output.""" + from prompty.invoker import run + + agent = self._make_structured_agent() + messages = [Message(role="user", parts=[TextPart(value="test")])] + + with self._patch_all_structured(): + result = run(agent, messages) + + assert isinstance(result, dict) + assert result == self.STRUCTURED_DATA diff --git a/runtime/python/prompty/tests/test_resilience.py b/runtime/python/prompty/tests/test_resilience.py new file mode 100644 index 00000000..a779b7b1 --- /dev/null +++ b/runtime/python/prompty/tests/test_resilience.py @@ -0,0 +1,302 @@ +"""Tests for agent loop resilience features (§9.8, §9.9, §9.10).""" + +from __future__ import annotations + +import json +import warnings +from unittest.mock import MagicMock, patch + +import pytest + +from prompty.core.tool_dispatch import ( + _extract_first_json_block, + _resilient_json_parse, + dispatch_tool, + dispatch_tool_async, +) + +# --------------------------------------------------------------------------- +# §9.8: Resilient JSON parsing +# --------------------------------------------------------------------------- + + +class TestResilientJsonParse: + """§9.8: Resilient argument parsing.""" + + def test_direct_parse(self): + result = _resilient_json_parse('{"city": "NY"}') + assert result == {"city": "NY"} + + def test_direct_parse_array(self): + result = _resilient_json_parse("[1, 2, 3]") + assert result == [1, 2, 3] + + def test_markdown_fences(self): + raw = '```json\n{"city": "NY"}\n```' + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _resilient_json_parse(raw) + assert result == {"city": "NY"} + assert len(w) == 1 + assert "markdown fences" in str(w[0].message) + + def test_markdown_fences_no_lang(self): + raw = '```\n{"city": "NY"}\n```' + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _resilient_json_parse(raw) + assert result == {"city": "NY"} + assert len(w) == 1 + + def test_extract_json_block(self): + raw = 'Here is the result: {"city": "NY"} enjoy!' + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _resilient_json_parse(raw) + assert result == {"city": "NY"} + assert len(w) == 1 + assert "JSON block" in str(w[0].message) + + def test_trailing_commas(self): + raw = '{"city": "NY", "temp": 72,}' + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _resilient_json_parse(raw) + assert result["city"] == "NY" + assert result["temp"] == 72 + assert len(w) == 1 + assert "trailing commas" in str(w[0].message) + + def test_all_fail_returns_none(self): + result = _resilient_json_parse("not json at all") + assert result is None + + def test_no_silent_empty_object(self): + """Spec: MUST NOT silently substitute empty object.""" + result = _resilient_json_parse("garbage text") + assert result is None # NOT {} + + def test_valid_json_no_warnings(self): + """Direct parse should not emit warnings.""" + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _resilient_json_parse('{"a": 1}') + assert result == {"a": 1} + assert len(w) == 0 + + def test_empty_string(self): + """Empty string is not valid JSON.""" + result = _resilient_json_parse("") + assert result is None + + +class TestExtractJsonBlock: + def test_respects_string_escapes(self): + raw = r'prefix {"key": "value with {braces}"} suffix' + block = _extract_first_json_block(raw) + parsed = json.loads(block) + assert parsed["key"] == "value with {braces}" + + def test_no_json(self): + assert _extract_first_json_block("no json here") is None + + def test_nested_objects(self): + raw = 'text {"a": {"b": 1}} more' + block = _extract_first_json_block(raw) + parsed = json.loads(block) + assert parsed["a"]["b"] == 1 + + def test_escaped_quotes_in_string(self): + raw = r'{"key": "value with \"escaped\" quotes"}' + block = _extract_first_json_block(raw) + assert block is not None + parsed = json.loads(block) + assert "escaped" in parsed["key"] + + +class TestDispatchToolResilience: + """Integration: dispatch_tool with resilient parsing.""" + + def test_dispatch_with_markdown_fences(self): + def my_tool(city: str = "") -> str: + return f"Weather in {city}" + + result = dispatch_tool( + "my_tool", + '```json\n{"city": "NY"}\n```', + {"my_tool": my_tool}, + MagicMock(), + {}, + ) + assert "Weather in NY" in result + + def test_dispatch_with_garbage_args(self): + def my_tool(**kwargs: object) -> str: + return "ok" + + result = dispatch_tool( + "my_tool", + "totally not json", + {"my_tool": my_tool}, + MagicMock(), + {}, + ) + assert "Error" in result + assert "all parse strategies failed" in result + + +# --------------------------------------------------------------------------- +# §9.9: Tool execution error safety +# --------------------------------------------------------------------------- + + +class TestToolErrorSafety: + """§9.9: Tool execution error safety.""" + + def test_tool_exception_returns_error_string(self): + """Tool that raises should return error string, not propagate.""" + + def bad_tool(**kwargs: object) -> str: + raise RuntimeError("tool exploded") + + result = dispatch_tool( + "bad_tool", + "{}", + {"bad_tool": bad_tool}, + MagicMock(), + {}, + ) + assert "Error" in result + assert "exploded" in result + + @pytest.mark.asyncio + async def test_async_tool_exception_returns_error_string(self): + """Async tool that raises should return error string, not propagate.""" + + async def bad_tool(**kwargs: object) -> str: + raise RuntimeError("async boom") + + result = await dispatch_tool_async( + "bad_tool", + "{}", + {"bad_tool": bad_tool}, + MagicMock(), + {}, + ) + assert "Error" in result + assert "async boom" in result + + def test_tool_error_does_not_propagate(self): + """dispatch_tool should never raise — always returns str.""" + + def exploding_tool(**kwargs: object) -> str: + raise ValueError("kaboom") + + # This should NOT raise + result = dispatch_tool( + "exploding_tool", + "{}", + {"exploding_tool": exploding_tool}, + MagicMock(), + {}, + ) + assert isinstance(result, str) + assert "kaboom" in result + + +# --------------------------------------------------------------------------- +# §9.10: LLM call retry +# --------------------------------------------------------------------------- + + +class TestLlmRetry: + """§9.10: LLM call retry.""" + + @patch("prompty.core.pipeline._invoke_executor") + @patch("prompty.core.pipeline.prepare") + @patch("prompty.core.pipeline.process") + @patch("prompty.core.pipeline.time.sleep") + def test_retry_success_on_second_attempt(self, mock_sleep, mock_process, mock_prepare, mock_execute): + from prompty.core.pipeline import turn + from prompty.core.types import Message, TextPart + + mock_prepare.return_value = [Message(role="user", parts=[TextPart(value="hi")])] + mock_process.return_value = "processed result" + + # Fail first, succeed second + final_response = MagicMock() + final_response.choices = [MagicMock()] + final_response.choices[0].finish_reason = "stop" + final_response.choices[0].message.tool_calls = None + final_response.choices[0].message.content = "success" + + mock_execute.side_effect = [Exception("transient"), final_response] + + # Need tools to enter agent loop + turn( + MagicMock(), + {}, + tools={"dummy": lambda: "ok"}, + max_llm_retries=3, + ) + assert mock_execute.call_count == 2 + mock_sleep.assert_called_once() + + @patch("prompty.core.pipeline._invoke_executor") + @patch("prompty.core.pipeline.prepare") + @patch("prompty.core.pipeline.time.sleep") + def test_retry_exhausted_raises_execute_error(self, mock_sleep, mock_prepare, mock_execute): + from prompty.core.pipeline import ExecuteError, turn + from prompty.core.types import Message, TextPart + + mock_prepare.return_value = [Message(role="user", parts=[TextPart(value="hi")])] + mock_execute.side_effect = Exception("persistent failure") + + with pytest.raises(ExecuteError) as exc_info: + turn( + MagicMock(), + {}, + tools={"dummy": lambda: "ok"}, + max_llm_retries=2, + ) + assert exc_info.value.messages is not None + assert len(exc_info.value.messages) > 0 + assert "persistent failure" in str(exc_info.value) + + @patch("prompty.core.pipeline._invoke_executor") + @patch("prompty.core.pipeline.prepare") + @patch("prompty.core.pipeline.time.sleep") + def test_no_retry_on_fast_path(self, mock_sleep, mock_prepare, mock_execute): + """Fast path (no tools) should NOT use retry.""" + from prompty.core.pipeline import turn + from prompty.core.types import Message, TextPart + + mock_prepare.return_value = [Message(role="user", parts=[TextPart(value="hi")])] + mock_execute.side_effect = Exception("should not retry") + + # No tools = fast path, no retry + with pytest.raises(Exception, match="should not retry"): + turn(MagicMock(), {}, tools=None) + + assert mock_execute.call_count == 1 + mock_sleep.assert_not_called() + + def test_execute_error_has_messages(self): + from prompty.core.pipeline import ExecuteError + from prompty.core.types import Message, TextPart + + msgs = [Message(role="user", parts=[TextPart(value="test")])] + err = ExecuteError("test error", messages=msgs) + assert str(err) == "test error" + assert err.messages == msgs + + def test_execute_error_default_messages(self): + from prompty.core.pipeline import ExecuteError + + err = ExecuteError("test error") + assert err.messages == [] + + def test_execute_error_importable_from_prompty(self): + from prompty import ExecuteError + + assert issubclass(ExecuteError, Exception) diff --git a/runtime/python/prompty/tests/test_run_agent.py b/runtime/python/prompty/tests/test_run_agent.py index dadcaa5a..01bc1a46 100644 --- a/runtime/python/prompty/tests/test_run_agent.py +++ b/runtime/python/prompty/tests/test_run_agent.py @@ -118,7 +118,7 @@ def test_success(self): def test_bad_json(self): tools = {"fn": lambda: None} result = dispatch_tool("fn", "not json", tools, None, {}) - assert "invalid JSON" in result + assert "Invalid JSON" in result assert "fn" in result def test_tool_exception(self): @@ -158,7 +158,7 @@ async def async_fn(city: str) -> str: def test_bad_json(self): tools = {"fn": lambda: None} result = asyncio.get_event_loop().run_until_complete(dispatch_tool_async("fn", "{bad}", tools, None, {})) - assert "invalid JSON" in result + assert "Invalid JSON" in result # --------------------------------------------------------------------------- @@ -321,7 +321,7 @@ def test_bad_json_tool_args_recovers(self, mock_prepare, mock_execute, mock_proc assert result == "Fixed it." # Check that error message was sent back as tool result tool_msg = [m for m in messages if m.role == "tool"][0] - assert "invalid JSON" in tool_msg.parts[0].value + assert "Invalid JSON" in tool_msg.parts[0].value @patch("prompty.core.pipeline.process") @patch("prompty.core.pipeline._invoke_executor") diff --git a/runtime/python/prompty/tests/test_spec_vectors.py b/runtime/python/prompty/tests/test_spec_vectors.py index 38ffd3f8..a941acd0 100644 --- a/runtime/python/prompty/tests/test_spec_vectors.py +++ b/runtime/python/prompty/tests/test_spec_vectors.py @@ -1200,7 +1200,7 @@ def _test_agent_error_real( inputs=inp.get("parent_inputs"), tools=tool_functions, ) - except StopIteration: + except (StopIteration, Exception): pass # Mock ran out of responses — that's fine for error vectors else: pytest.fail(f"Agent '{name}': unknown error type: {error_msg}") diff --git a/runtime/python/prompty/tests/test_tool_dispatch.py b/runtime/python/prompty/tests/test_tool_dispatch.py index 73cbc6d3..c637a932 100644 --- a/runtime/python/prompty/tests/test_tool_dispatch.py +++ b/runtime/python/prompty/tests/test_tool_dispatch.py @@ -236,7 +236,7 @@ class TestDispatchTool: def test_invalid_json(self): result = dispatch_tool("fn", "not valid json", user_tools={}, agent=_make_agent(), parent_inputs={}) assert "Error" in result - assert "invalid JSON" in result + assert "Invalid JSON" in result def test_empty_arguments(self): result = dispatch_tool( diff --git a/runtime/rust/.env.example b/runtime/rust/.env.example index 9076b123..21766705 100644 --- a/runtime/rust/.env.example +++ b/runtime/rust/.env.example @@ -10,6 +10,11 @@ AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o-mini AZURE_OPENAI_EMBEDDING_DEPLOYMENT=text-embedding-3-small +# Azure Entra ID (for keyless auth tests) +# Set AZURE_TENANT_ID so DefaultAzureCredential picks the correct tenant. +# Requires 'Cognitive Services OpenAI User' role on the resource. +AZURE_TENANT_ID= + # Anthropic ANTHROPIC_API_KEY= ANTHROPIC_MODEL=claude-sonnet-4-20250514 diff --git a/runtime/rust/prompty-foundry/tests/entra_id.rs b/runtime/rust/prompty-foundry/tests/entra_id.rs new file mode 100644 index 00000000..34063aeb --- /dev/null +++ b/runtime/rust/prompty-foundry/tests/entra_id.rs @@ -0,0 +1,327 @@ +//! Integration tests for Entra ID (keyless / DefaultAzureCredential) auth. +//! +//! These tests verify that the Foundry executor can authenticate via +//! `DefaultAzureCredential` when no API key is provided. They require: +//! - `AZURE_OPENAI_ENDPOINT` +//! - `AZURE_OPENAI_CHAT_DEPLOYMENT` +//! - `AZURE_TENANT_ID` (so DefaultAzureCredential picks the right tenant) +//! - A valid Azure identity (e.g. Azure CLI login, managed identity, etc.) +//! +//! Run with: +//! ```sh +//! cargo test -p prompty-foundry --features entra_id --test entra_id -- --ignored --test-threads=1 +//! ``` + +#![cfg(feature = "entra_id")] + +use prompty::model::Prompty; +use prompty::model::context::LoadContext; +use prompty::{TurnOptions, register_defaults}; +use serde_json::{Value, json}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn load_dotenv() { + let env_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join(".env"); + if let Ok(contents) = std::fs::read_to_string(env_path) { + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + if let Some((key, value)) = line.split_once('=') { + let key = key.trim(); + let value = value.trim(); + if std::env::var(key).is_err() { + // SAFETY: integration tests run with --test-threads=1. + unsafe { std::env::set_var(key, value) }; + } + } + } + } +} + +macro_rules! skip_if_no_env { + ($var:expr) => { + if std::env::var($var).unwrap_or_default().is_empty() { + eprintln!("Skipping: {} not set", $var); + return; + } + }; +} + +fn setup() { + load_dotenv(); + register_defaults(); + prompty_foundry::register(); +} + +fn chat_deployment() -> String { + std::env::var("AZURE_OPENAI_CHAT_DEPLOYMENT").unwrap_or_else(|_| "gpt-4o-mini".into()) +} + +fn endpoint() -> String { + std::env::var("AZURE_OPENAI_ENDPOINT").unwrap_or_default() +} + +/// Temporarily remove `AZURE_OPENAI_API_KEY` so the executor takes the Entra ID +/// path. Returns the old value (if any) so the caller can restore it. +fn suppress_api_key() -> Option { + let old = std::env::var("AZURE_OPENAI_API_KEY").ok(); + // SAFETY: integration tests run with --test-threads=1. + unsafe { std::env::remove_var("AZURE_OPENAI_API_KEY") }; + old +} + +/// Restore `AZURE_OPENAI_API_KEY` after the test. +fn restore_api_key(old: Option) { + // SAFETY: integration tests run with --test-threads=1. + match old { + Some(v) => unsafe { std::env::set_var("AZURE_OPENAI_API_KEY", v) }, + None => unsafe { std::env::remove_var("AZURE_OPENAI_API_KEY") }, + } +} + +fn build_foundry_chat_agent(question: &str, options: Value) -> Prompty { + let data = json!({ + "name": "entra-id-chat-test", + "kind": "prompt", + "model": { + "id": chat_deployment(), + "provider": "foundry", + "apiType": "chat", + "connection": { + "kind": "foundry", + "endpoint": endpoint(), + }, + "options": options, + }, + "template": { + "format": { "kind": "nunjucks" }, + "parser": { "kind": "prompty" }, + }, + "instructions": format!( + "system:\nYou are a helpful assistant. Be very brief.\nuser:\n{question}" + ), + }); + Prompty::load_from_value(&data, &LoadContext::default()) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +/// Verify that `DefaultAzureCredential` can acquire a token for the +/// Azure Cognitive Services scope. +#[tokio::test] +#[ignore] +async fn test_entra_id_token_acquisition() { + setup(); + skip_if_no_env!("AZURE_OPENAI_ENDPOINT"); + + use azure_core::credentials::TokenCredential; + use azure_identity::DefaultAzureCredential; + + let credential = + DefaultAzureCredential::new().expect("DefaultAzureCredential should be created"); + + let token = credential + .get_token(&["https://cognitiveservices.azure.com/.default"]) + .await + .expect("Should acquire Entra ID token for Cognitive Services scope"); + + assert!( + !token.token.secret().is_empty(), + "Token should not be empty" + ); + eprintln!( + "Entra ID token acquired successfully (length: {})", + token.token.secret().len() + ); +} + +/// Chat completion using `kind: "foundry"` + Entra ID (no API key). +#[tokio::test] +#[ignore] +async fn test_entra_id_chat_completion() { + setup(); + skip_if_no_env!("AZURE_OPENAI_ENDPOINT"); + skip_if_no_env!("AZURE_OPENAI_CHAT_DEPLOYMENT"); + + let old_key = suppress_api_key(); + + let agent = build_foundry_chat_agent( + "Say hello in exactly 3 words.", + json!({ "temperature": 0, "maxOutputTokens": 100 }), + ); + + let result = prompty::invoke_agent(&agent, None).await; + + restore_api_key(old_key); + + let result = result.expect("Entra ID chat completion should succeed"); + assert!(result.is_string(), "result should be a string: {result:?}"); + let text = result.as_str().unwrap(); + assert!(!text.is_empty(), "result should not be empty"); + eprintln!("Entra ID chat result: {text}"); +} + +/// Streaming chat completion using `kind: "foundry"` + Entra ID (no API key). +#[tokio::test] +#[ignore] +async fn test_entra_id_chat_completion_streaming() { + setup(); + skip_if_no_env!("AZURE_OPENAI_ENDPOINT"); + skip_if_no_env!("AZURE_OPENAI_CHAT_DEPLOYMENT"); + + let old_key = suppress_api_key(); + + let agent = build_foundry_chat_agent( + "What is 2 + 2? Answer with just the number.", + json!({ "temperature": 0, "maxOutputTokens": 50 }), + ); + + let result = prompty::turn(&agent, None, Some(TurnOptions::default())).await; + + restore_api_key(old_key); + + let result = result.expect("Entra ID streaming chat should succeed"); + let text = result.as_str().unwrap_or(&result.to_string()).to_string(); + assert!(!text.is_empty(), "streaming result should not be empty"); + eprintln!("Entra ID streaming result: {text}"); +} + +/// Structured output via `outputSchema` using Entra ID auth. +#[tokio::test] +#[ignore] +async fn test_entra_id_structured_output() { + setup(); + skip_if_no_env!("AZURE_OPENAI_ENDPOINT"); + skip_if_no_env!("AZURE_OPENAI_CHAT_DEPLOYMENT"); + + let old_key = suppress_api_key(); + + let data = json!({ + "name": "entra-id-structured-test", + "kind": "prompt", + "model": { + "id": chat_deployment(), + "provider": "foundry", + "apiType": "chat", + "connection": { + "kind": "foundry", + "endpoint": endpoint(), + }, + "options": { "temperature": 0, "maxOutputTokens": 200 }, + }, + "template": { + "format": { "kind": "nunjucks" }, + "parser": { "kind": "prompty" }, + }, + "outputs": [ + { "name": "city", "kind": "string", "description": "The city name", "required": true }, + { "name": "country", "kind": "string", "description": "The country name", "required": true }, + ], + "instructions": "system:\nYou are a geography expert. Return structured data.\nuser:\nTell me about Paris.", + }); + let agent = Prompty::load_from_value(&data, &LoadContext::default()); + + let result = prompty::invoke_agent(&agent, None).await; + + restore_api_key(old_key); + + let result = result.expect("Entra ID structured output should succeed"); + let obj = match result { + Value::Object(ref o) => o.clone(), + Value::String(ref s) => serde_json::from_str::(s) + .expect("structured output string should be valid JSON") + .as_object() + .expect("parsed JSON should be an object") + .clone(), + other => panic!("Expected object or JSON string, got: {other:?}"), + }; + + assert!(obj.contains_key("city"), "missing 'city' field: {obj:?}"); + assert!( + obj.contains_key("country"), + "missing 'country' field: {obj:?}" + ); + eprintln!("Entra ID structured output: {obj:?}"); +} + +/// Agent with tool calling using Entra ID auth. +#[tokio::test] +#[ignore] +async fn test_entra_id_agent_tool_calling() { + setup(); + skip_if_no_env!("AZURE_OPENAI_ENDPOINT"); + skip_if_no_env!("AZURE_OPENAI_CHAT_DEPLOYMENT"); + + let old_key = suppress_api_key(); + + let data = json!({ + "name": "entra-id-agent-test", + "kind": "prompt", + "model": { + "id": chat_deployment(), + "provider": "foundry", + "apiType": "agent", + "connection": { + "kind": "foundry", + "endpoint": endpoint(), + }, + "options": { "temperature": 0, "maxOutputTokens": 300 }, + }, + "template": { + "format": { "kind": "nunjucks" }, + "parser": { "kind": "prompty" }, + }, + "tools": [ + { + "name": "get_weather", + "kind": "function", + "description": "Get the current weather for a city", + "parameters": { + "properties": [ + { "name": "city", "kind": "string", "description": "The city name", "required": true } + ] + }, + } + ], + "instructions": "system:\nYou are a helpful assistant with weather tools. Use the get_weather tool when asked about weather. Be brief.\nuser:\nWhat is the weather in Seattle?", + }); + let agent = Prompty::load_from_value(&data, &LoadContext::default()); + + let mut tools = std::collections::HashMap::new(); + tools.insert( + "get_weather".to_string(), + prompty::ToolHandler::Sync(Box::new(|args: Value| { + let city = args + .get("city") + .and_then(|c| c.as_str()) + .unwrap_or("unknown"); + Ok(format!("72°F and sunny in {city}")) + })), + ); + + let result = prompty::turn(&agent, None, Some(TurnOptions::with_tools(tools))).await; + + restore_api_key(old_key); + + let result = result.expect("Entra ID agent tool calling should succeed"); + let text = result.as_str().unwrap_or(&result.to_string()).to_string(); + let mentions_weather = text.contains("72") + || text.to_lowercase().contains("sunny") + || text.to_lowercase().contains("weather") + || text.to_lowercase().contains("seattle"); + assert!( + mentions_weather, + "Agent response should mention weather info: {text}" + ); + eprintln!("Entra ID agent result: {text}"); +} diff --git a/runtime/rust/prompty/Cargo.toml b/runtime/rust/prompty/Cargo.toml index 4f1208f9..35b81cca 100644 --- a/runtime/rust/prompty/Cargo.toml +++ b/runtime/rust/prompty/Cargo.toml @@ -17,7 +17,7 @@ serde_json = "1" serde_yaml = "0.9" thiserror = "2" async-trait = "0.1" -tokio = { version = "1", features = ["sync", "fs", "rt"] } +tokio = { version = "1", features = ["sync", "fs", "rt", "time", "macros"] } minijinja = "2" rand = "0.9" regex = "1" diff --git a/runtime/rust/prompty/src/interfaces.rs b/runtime/rust/prompty/src/interfaces.rs index 4f06ffbe..825348e3 100644 --- a/runtime/rust/prompty/src/interfaces.rs +++ b/runtime/rust/prompty/src/interfaces.rs @@ -47,8 +47,31 @@ pub enum InvokerError { /// The operation was cancelled via the cancellation token. #[error("cancelled: {0}")] Cancelled(String), + + /// LLM call failed after retries, carrying accumulated conversation state (§9.10). + #[error("{0}")] + ExecuteRetryExhausted(ExecuteError), + + /// A generic retryable/other error. + #[error("{0}")] + Other(String), +} + +/// Error from the agent loop that includes accumulated conversation state (§9.10). +#[derive(Debug)] +pub struct ExecuteError { + pub message: String, + pub messages: Vec, } +impl std::fmt::Display for ExecuteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for ExecuteError {} + // --------------------------------------------------------------------------- // Renderer // --------------------------------------------------------------------------- diff --git a/runtime/rust/prompty/src/lib.rs b/runtime/rust/prompty/src/lib.rs index 33407121..9456d7f2 100644 --- a/runtime/rust/prompty/src/lib.rs +++ b/runtime/rust/prompty/src/lib.rs @@ -58,7 +58,7 @@ pub use guardrails::{ GuardrailError, GuardrailPhase, GuardrailResult, Guardrails, InputGuardrail, OutputGuardrail, ToolGuardrail, }; -pub use interfaces::{Executor, InvokerError, Parser, Processor, Renderer}; +pub use interfaces::{ExecuteError, Executor, InvokerError, Parser, Processor, Renderer}; pub use loader::{LoadError, load, load_async, load_from_string}; pub use model::Prompty; pub use pipeline::{ diff --git a/runtime/rust/prompty/src/pipeline.rs b/runtime/rust/prompty/src/pipeline.rs index 80c179a4..8825f338 100644 --- a/runtime/rust/prompty/src/pipeline.rs +++ b/runtime/rust/prompty/src/pipeline.rs @@ -504,7 +504,8 @@ pub async fn invoke( let messages = prepare(agent, inputs).await?; let provider = resolve_provider(agent); let response = registry::invoke_executor(&provider, agent, &messages).await?; - process(agent, response).await + let processed = process(agent, response).await?; + Ok(unwrap_structured(&processed)) } .await; @@ -638,6 +639,8 @@ pub struct TurnOptions { /// Optional validator for structured output (called via cast after processing). #[allow(clippy::type_complexity)] pub validator: Option Result<(), String> + Send + Sync>>, + /// Maximum retries for LLM calls with exponential backoff (§9.10, default: 3). + pub max_llm_retries: usize, } impl Default for TurnOptions { @@ -653,6 +656,7 @@ impl Default for TurnOptions { steering: None, parallel_tool_calls: false, validator: None, + max_llm_retries: 3, } } } @@ -900,67 +904,83 @@ pub async fn turn( } } - // Execute LLM — streaming or non-streaming + // Execute LLM — streaming or non-streaming, with retry (§9.10) let streaming = is_streaming(agent); - let (tool_calls, processed, raw_response) = if streaming { - // Streaming path: execute_stream → PromptyStream → process_stream → consume - match registry::invoke_executor_stream(&provider, agent, &messages).await { - Ok(sse_stream) => { - let prompty_stream = PromptyStream::from_stream("PromptyStream", sse_stream); - let chunk_stream = - registry::invoke_processor_stream(&provider, Box::pin(prompty_stream))?; + let mut llm_attempts = 0u32; + let (tool_calls, processed, raw_response) = loop { + // Check cancellation before each attempt + if opts.is_cancelled() { + iter_span.end(); + opts.emit(AgentEvent::Cancelled); + span.emit("error", &json!("Operation cancelled")); + span.end(); + return Err(InvokerError::Cancelled("Operation cancelled".to_string())); + } - let (tool_calls_vec, text) = { - use futures::StreamExt; - let mut tool_calls_vec = Vec::new(); - let mut text_parts = Vec::new(); - let mut stream_error = None; - futures::pin_mut!(chunk_stream); - while let Some(chunk) = chunk_stream.next().await { - match chunk { - StreamChunk::Text(t) => { - opts.emit(AgentEvent::Token(t.clone())); - text_parts.push(t); - } - StreamChunk::Thinking(t) => { - opts.emit(AgentEvent::Thinking(t)); - } - StreamChunk::Tool(tc) => { - tool_calls_vec.push(tc); - } - StreamChunk::Error(e) => { - stream_error = Some(e); - break; + match execute_llm_attempt(&provider, agent, &messages, streaming, &opts).await { + Ok(result) => break result, + Err(e) => { + llm_attempts += 1; + if llm_attempts >= opts.max_llm_retries as u32 { + iter_span.end(); + span.emit( + "error", + &json!(format!( + "LLM call failed after {} retries: {}", + opts.max_llm_retries, e + )), + ); + span.end(); + return Err(InvokerError::ExecuteRetryExhausted( + crate::interfaces::ExecuteError { + message: format!( + "LLM call failed after {} retries: {}", + opts.max_llm_retries, e + ), + messages: messages.clone(), + }, + )); + } + opts.emit(AgentEvent::Status(format!( + "LLM call failed, retrying (attempt {}/{})...", + llm_attempts + 1, + opts.max_llm_retries + ))); + // Exponential backoff with jitter, capped at 60s + // Formula: min(2^attempts + jitter, 60) per spec §9.10 + let backoff_secs = { + use rand::Rng; + let jitter: f64 = rand::rng().random(); + (2.0_f64.powi(llm_attempts as i32) + jitter).min(60.0) + }; + let delay = std::time::Duration::from_secs_f64(backoff_secs); + + // Check cancellation during backoff sleep (spec §9.10) + if let Some(ref cancel_flag) = opts.cancelled { + let cancel_flag = cancel_flag.clone(); + tokio::select! { + _ = tokio::time::sleep(delay) => {} + _ = async { + loop { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + if cancel_flag.load(std::sync::atomic::Ordering::Relaxed) { + return; + } } + } => { + opts.emit(AgentEvent::Cancelled); + span.emit("error", &json!("Operation cancelled during retry backoff")); + span.end(); + return Err(InvokerError::Cancelled( + "Operation cancelled during retry backoff".to_string(), + )); } } - if let Some(err) = stream_error { - iter_span.end(); - span.emit("error", &json!(err)); - span.end(); - return Err(InvokerError::Execute(err.into())); - } - (tool_calls_vec, text_parts.join("")) - }; - - let processed = json!(text); - (tool_calls_vec, processed, json!(null)) - } - Err(_) => { - // Fallback to non-streaming if executor doesn't support it - let raw_response = - registry::invoke_executor(&provider, agent, &messages).await?; - let processed = process(agent, raw_response.clone()).await?; - let tool_calls = extract_tool_calls_from_processed(&processed); - (tool_calls, processed, raw_response) + } else { + tokio::time::sleep(delay).await; + } } } - } else { - // Non-streaming path - let raw_response = registry::invoke_executor(&provider, agent, &messages).await?; - let processed = process(agent, raw_response.clone()).await?; - let tool_calls = extract_tool_calls_from_processed(&processed); - (tool_calls, processed, raw_response) }; if tool_calls.is_empty() { @@ -1103,6 +1123,77 @@ fn has_any_tools(agent: &Prompty) -> bool { false } +/// Execute a single LLM attempt (streaming or non-streaming). +/// Returns (tool_calls, processed, raw_response) on success, or an error string on failure. +async fn execute_llm_attempt( + provider: &str, + agent: &Prompty, + messages: &[Message], + streaming: bool, + opts: &TurnOptions, +) -> Result<(Vec, Value, Value), String> { + if streaming { + match registry::invoke_executor_stream(provider, agent, messages).await { + Ok(sse_stream) => { + let prompty_stream = PromptyStream::from_stream("PromptyStream", sse_stream); + let chunk_stream = + registry::invoke_processor_stream(provider, Box::pin(prompty_stream)) + .map_err(|e| e.to_string())?; + + use futures::StreamExt; + let mut tool_calls_vec = Vec::new(); + let mut text_parts = Vec::new(); + let mut stream_error = None; + futures::pin_mut!(chunk_stream); + while let Some(chunk) = chunk_stream.next().await { + match chunk { + StreamChunk::Text(t) => { + opts.emit(AgentEvent::Token(t.clone())); + text_parts.push(t); + } + StreamChunk::Thinking(t) => { + opts.emit(AgentEvent::Thinking(t)); + } + StreamChunk::Tool(tc) => { + tool_calls_vec.push(tc); + } + StreamChunk::Error(e) => { + stream_error = Some(e); + break; + } + } + } + if let Some(err) = stream_error { + return Err(err); + } + let text = text_parts.join(""); + let processed = json!(text); + Ok((tool_calls_vec, processed, json!(null))) + } + Err(stream_err) => { + // Fallback to non-streaming if executor doesn't support it + let raw_response = registry::invoke_executor(provider, agent, messages) + .await + .map_err(|e| format!("{stream_err} (stream), then {e} (non-stream)"))?; + let processed = process(agent, raw_response.clone()) + .await + .map_err(|e| e.to_string())?; + let tool_calls = extract_tool_calls_from_processed(&processed); + Ok((tool_calls, processed, raw_response)) + } + } + } else { + let raw_response = registry::invoke_executor(provider, agent, messages) + .await + .map_err(|e| e.to_string())?; + let processed = process(agent, raw_response.clone()) + .await + .map_err(|e| e.to_string())?; + let tool_calls = extract_tool_calls_from_processed(&processed); + Ok((tool_calls, processed, raw_response)) + } +} + /// Dispatch tool calls sequentially, checking cancellation and tool guardrails. async fn dispatch_tools_sequential( tool_calls: &[ToolCall], @@ -1138,8 +1229,32 @@ async fn dispatch_tools_sequential( arguments: tc.arguments.clone(), }); - let result = - crate::tool_dispatch::dispatch_tool(tc, &opts.tools, agent, parent_inputs).await; + // §9.9: Wrap dispatch in catch_unwind for panic safety + let result = { + let fut = std::panic::AssertUnwindSafe(crate::tool_dispatch::dispatch_tool( + tc, + &opts.tools, + agent, + parent_inputs, + )); + match futures::FutureExt::catch_unwind(fut).await { + Ok(r) => r, + Err(panic_info) => { + let msg = if let Some(s) = panic_info.downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = panic_info.downcast_ref::() { + s.clone() + } else { + "unknown panic".to_string() + }; + opts.emit(AgentEvent::Error(format!( + "Tool '{}' panicked: {}", + tc.name, msg + ))); + format!("Error: Tool '{}' panicked: {}", tc.name, msg) + } + } + }; opts.emit(AgentEvent::ToolResult { name: tc.name.clone(), @@ -1165,7 +1280,7 @@ async fn dispatch_tools_parallel( }); } - // Dispatch all in parallel + // Dispatch all in parallel with panic safety (§9.9) let futures: Vec<_> = tool_calls .iter() .map(|tc| { @@ -1181,7 +1296,29 @@ async fn dispatch_tools_parallel( return format!("Error: Tool guardrail denied: {reason}"); } } - crate::tool_dispatch::dispatch_tool(tc, tools, agent, parent_inputs).await + let fut = std::panic::AssertUnwindSafe(crate::tool_dispatch::dispatch_tool( + tc, + tools, + agent, + parent_inputs, + )); + match futures::FutureExt::catch_unwind(fut).await { + Ok(r) => r, + Err(panic_info) => { + let msg = if let Some(s) = panic_info.downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = panic_info.downcast_ref::() { + s.clone() + } else { + "unknown panic".to_string() + }; + opts.emit(AgentEvent::Error(format!( + "Tool '{}' panicked: {}", + tc.name, msg + ))); + format!("Error: Tool '{}' panicked: {}", tc.name, msg) + } + } } }) .collect(); @@ -1581,6 +1718,7 @@ mod tests { fn test_turn_options_default() { let opts = TurnOptions::default(); assert_eq!(opts.max_iterations, 10); + assert_eq!(opts.max_llm_retries, 3); assert!(!opts.raw); assert!(opts.tools.is_empty()); assert!(!opts.is_cancelled()); @@ -2159,4 +2297,158 @@ mod tests { let result = invoke(&agent, None).await.unwrap(); assert_eq!(result, "The weather in Seattle is 72°F."); } + + // ----------------------------------------------------------------------- + // LLM retry tests (§9.10) + // ----------------------------------------------------------------------- + + /// Mock executor that fails the first N calls, then succeeds. + struct FailThenSucceedExecutor { + call_count: Arc, + fail_until: usize, + } + + #[async_trait::async_trait] + impl crate::interfaces::Executor for FailThenSucceedExecutor { + async fn execute( + &self, + _agent: &Prompty, + _messages: &[Message], + ) -> Result { + let n = self.call_count.fetch_add(1, Ordering::SeqCst); + if n < self.fail_until { + Err(InvokerError::Execute("transient failure".into())) + } else { + Ok(serde_json::json!({ + "choices": [{"message": {"content": "success after retry"}}] + })) + } + } + } + + /// Mock executor that always fails. + struct AlwaysFailExecutor; + + #[async_trait::async_trait] + impl crate::interfaces::Executor for AlwaysFailExecutor { + async fn execute( + &self, + _agent: &Prompty, + _messages: &[Message], + ) -> Result { + Err(InvokerError::Execute("persistent failure".into())) + } + } + + #[tokio::test] + #[serial] + async fn test_llm_retry_success_on_second_attempt() { + ensure_defaults(); + let key = "retry_test_success"; + let call_count = Arc::new(AtomicUsize::new(0)); + registry::register_executor( + key, + FailThenSucceedExecutor { + call_count: call_count.clone(), + fail_until: 1, + }, + ); + registry::register_processor(key, MockProcessor); + + let agent = make_simple_agent(key); + let mut tools = HashMap::new(); + tools.insert( + "dummy".into(), + ToolHandler::Sync(Box::new(|_| Ok("ok".into()))), + ); + + let opts = TurnOptions { + tools, + max_llm_retries: 3, + ..Default::default() + }; + + let result = turn(&agent, None, Some(opts)).await.unwrap(); + assert_eq!( + call_count.load(Ordering::SeqCst), + 2, + "Should have failed once and succeeded once" + ); + assert_eq!(result, "success after retry"); + } + + #[tokio::test] + #[serial] + async fn test_llm_retry_exhausted_carries_messages() { + ensure_defaults(); + let key = "retry_test_exhaust"; + registry::register_executor(key, AlwaysFailExecutor); + registry::register_processor(key, MockProcessor); + + let agent = make_simple_agent(key); + let mut tools = HashMap::new(); + tools.insert( + "dummy".into(), + ToolHandler::Sync(Box::new(|_| Ok("ok".into()))), + ); + + let opts = TurnOptions { + tools, + max_llm_retries: 2, + ..Default::default() + }; + + let err = turn(&agent, None, Some(opts)).await.unwrap_err(); + let err_str = format!("{}", err); + assert!( + err_str.contains("retries") || err_str.contains("failed"), + "Error should mention retry exhaustion: {}", + err_str + ); + } + + #[tokio::test] + #[serial] + async fn test_llm_retry_emits_status_events() { + ensure_defaults(); + let key = "retry_test_events"; + let call_count = Arc::new(AtomicUsize::new(0)); + registry::register_executor( + key, + FailThenSucceedExecutor { + call_count: call_count.clone(), + fail_until: 1, + }, + ); + registry::register_processor(key, MockProcessor); + + let agent = make_simple_agent(key); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let events_clone = events.clone(); + + let mut tools = HashMap::new(); + tools.insert( + "dummy".into(), + ToolHandler::Sync(Box::new(|_| Ok("ok".into()))), + ); + + let opts = TurnOptions { + tools, + max_llm_retries: 3, + on_event: Some(Box::new(move |event| { + events_clone.lock().unwrap().push(format!("{:?}", event)); + })), + ..Default::default() + }; + + let _ = turn(&agent, None, Some(opts)).await.unwrap(); + let captured = events.lock().unwrap(); + assert!( + captured + .iter() + .any(|e| e.contains("Status") && e.contains("retrying")), + "Expected retry status event, got: {:?}", + *captured + ); + } } diff --git a/runtime/rust/prompty/src/tool_dispatch.rs b/runtime/rust/prompty/src/tool_dispatch.rs index 264e9da6..01517cde 100644 --- a/runtime/rust/prompty/src/tool_dispatch.rs +++ b/runtime/rust/prompty/src/tool_dispatch.rs @@ -200,6 +200,91 @@ pub fn resolve_bindings( args } +// --------------------------------------------------------------------------- +// Resilient JSON parsing (§9.8) +// --------------------------------------------------------------------------- + +/// Attempt to parse JSON with fallback strategies per spec §9.8. +/// Returns Ok(value) on success, or Err(error_message) if all strategies fail. +fn resilient_json_parse(raw: &str) -> Result { + // Strategy 1: Direct parse + if let Ok(v) = serde_json::from_str(raw) { + return Ok(v); + } + + // Strategy 2: Strip markdown code fences + let fence_re = regex::Regex::new(r"(?s)^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$").unwrap(); + if let Some(caps) = fence_re.captures(raw) { + let stripped = caps.get(1).unwrap().as_str(); + if stripped != raw { + if let Ok(v) = serde_json::from_str(stripped) { + eprintln!("[prompty] Parsed tool arguments after stripping markdown fences"); + return Ok(v); + } + } + } + + // Strategy 3: Extract first balanced JSON block { ... } + if let Some(block) = extract_first_json_block(raw) { + if let Ok(v) = serde_json::from_str(&block) { + eprintln!("[prompty] Parsed tool arguments after extracting JSON block"); + return Ok(v); + } + } + + // Strategy 4: Strip trailing commas before } or ] + let comma_re = regex::Regex::new(r",\s*([}\]])").unwrap(); + let cleaned = comma_re.replace_all(raw, "$1"); + if cleaned != raw { + if let Ok(v) = serde_json::from_str(&cleaned) { + eprintln!("[prompty] Parsed tool arguments after stripping trailing commas"); + return Ok(v); + } + } + + Err(format!( + "All JSON parse strategies failed for: {}", + &raw[..raw.len().min(200)] + )) +} + +/// Extract the first balanced `{...}` block from text, respecting string escapes. +fn extract_first_json_block(text: &str) -> Option { + let start = text.find('{')?; + let mut depth = 0i32; + let mut in_string = false; + let mut escape_next = false; + let bytes = text.as_bytes(); + + for i in start..bytes.len() { + let ch = bytes[i] as char; + if escape_next { + escape_next = false; + continue; + } + if in_string { + if ch == '\\' { + escape_next = true; + } else if ch == '"' { + in_string = false; + } + continue; + } + match ch { + '"' => in_string = true, + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + return Some(text[start..=i].to_string()); + } + } + _ => {} + } + } + None +} + // --------------------------------------------------------------------------- // dispatch_tool — 3-layer resolution // --------------------------------------------------------------------------- @@ -221,8 +306,7 @@ pub async fn dispatch_tool( agent: &Prompty, parent_inputs: Option<&serde_json::Value>, ) -> String { - let args_result: Result = serde_json::from_str(&tool_call.arguments); - let mut args = match args_result { + let mut args = match resilient_json_parse(&tool_call.arguments) { Ok(a) => a, Err(e) => return format!("Error: Invalid tool arguments JSON: {e}"), }; @@ -303,13 +387,43 @@ fn find_tool_def(agent: &Prompty, name: &str) -> Option { } /// Execute a user-provided tool handler (from TurnOptions.tools). +/// Wraps sync handlers in catch_unwind for panic safety (§9.9). async fn execute_user_handler( handler: &crate::pipeline::ToolHandler, args: serde_json::Value, ) -> Result> { match handler { - crate::pipeline::ToolHandler::Sync(f) => f(args), - crate::pipeline::ToolHandler::Async(f) => f(args).await, + crate::pipeline::ToolHandler::Sync(f) => { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(args))) { + Ok(result) => result, + Err(panic_info) => { + let msg = if let Some(s) = panic_info.downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = panic_info.downcast_ref::() { + s.clone() + } else { + "unknown panic".to_string() + }; + Err(format!("Tool handler panicked: {msg}").into()) + } + } + } + crate::pipeline::ToolHandler::Async(f) => { + let fut = std::panic::AssertUnwindSafe(f(args)); + match futures::FutureExt::catch_unwind(fut).await { + Ok(result) => result, + Err(panic_info) => { + let msg = if let Some(s) = panic_info.downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = panic_info.downcast_ref::() { + s.clone() + } else { + "unknown panic".to_string() + }; + Err(format!("Tool handler panicked: {msg}").into()) + } + } + } } } @@ -973,4 +1087,94 @@ mod tests { assert!(result.contains("exotic_provider")); assert!(result.starts_with("Error:")); } + + // --- Resilient JSON parsing tests (§9.8) --- + + #[test] + fn test_resilient_parse_direct() { + let result = resilient_json_parse(r#"{"city": "NY"}"#).unwrap(); + assert_eq!(result["city"], "NY"); + } + + #[test] + fn test_resilient_parse_markdown_fences() { + let input = "```json\n{\"city\": \"NY\"}\n```"; + let result = resilient_json_parse(input).unwrap(); + assert_eq!(result["city"], "NY"); + } + + #[test] + fn test_resilient_parse_markdown_fences_no_lang() { + let input = "```\n{\"city\": \"NY\"}\n```"; + let result = resilient_json_parse(input).unwrap(); + assert_eq!(result["city"], "NY"); + } + + #[test] + fn test_resilient_parse_extract_block() { + let input = "Here is the JSON: {\"city\": \"NY\"} and some more text"; + let result = resilient_json_parse(input).unwrap(); + assert_eq!(result["city"], "NY"); + } + + #[test] + fn test_resilient_parse_trailing_commas() { + let input = r#"{"city": "NY", "temp": 72,}"#; + let result = resilient_json_parse(input).unwrap(); + assert_eq!(result["city"], "NY"); + } + + #[test] + fn test_resilient_parse_all_fail() { + let result = resilient_json_parse("this is not json at all"); + assert!(result.is_err()); + } + + #[test] + fn test_resilient_parse_no_silent_empty_object() { + // Spec: MUST NOT silently substitute empty object + let result = resilient_json_parse("garbage"); + assert!(result.is_err()); + } + + #[test] + fn test_extract_first_json_block_respects_strings() { + // Braces inside strings should NOT be matched + let input = r#"prefix {"key": "value with {braces}"} suffix"#; + let block = extract_first_json_block(input).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&block).unwrap(); + assert_eq!(parsed["key"], "value with {braces}"); + } + + // --- Tool panic safety test (§9.9) --- + + #[tokio::test] + #[serial] + async fn test_tool_panic_caught_in_dispatch() { + clear_tools(); + clear_tool_handlers(); + + let mut user_tools = HashMap::new(); + user_tools.insert( + "panicking_tool".into(), + PipelineToolHandler::Sync(Box::new(|_args| { + panic!("tool handler panicked!"); + })), + ); + + let tc = make_tool_call("panicking_tool", "{}"); + + // Should NOT panic — should return error string + let result = dispatch_tool(&tc, &user_tools, &default_agent(), None).await; + assert!( + result.contains("Error"), + "Expected error string, got: {}", + result + ); + assert!( + result.contains("panic"), + "Expected panic mention, got: {}", + result + ); + } } diff --git a/runtime/rust/prompty/tests/structured_pipeline.rs b/runtime/rust/prompty/tests/structured_pipeline.rs new file mode 100644 index 00000000..25cf7b2e --- /dev/null +++ b/runtime/rust/prompty/tests/structured_pipeline.rs @@ -0,0 +1,274 @@ +//! Tests that `invoke()` and `run()` correctly unwrap the `__prompty_structured` +//! envelope before returning results to the caller. +//! +//! These tests use mock executor/processor pairs registered under unique keys +//! (same pattern as `agent_vectors.rs`). The mock processor wraps its output +//! in the structured envelope (simulating what `pipeline::process()` does when +//! `agent.outputs` is non-empty). `invoke()` and `run()` must call +//! `unwrap_structured()` so the caller sees clean data, not the envelope. + +use async_trait::async_trait; +use serde_json::{Value, json}; + +use prompty::interfaces::{Executor, InvokerError, Processor}; +use prompty::model::Prompty; +use prompty::model::context::LoadContext; +use prompty::types::{Message, Role}; + +// --------------------------------------------------------------------------- +// MockExecutor — returns a canned chat-completion response +// --------------------------------------------------------------------------- + +struct MockStructuredExecutor { + response: Value, +} + +impl MockStructuredExecutor { + fn new(response: Value) -> Self { + Self { response } + } +} + +#[async_trait] +impl Executor for MockStructuredExecutor { + async fn execute( + &self, + _agent: &Prompty, + _messages: &[Message], + ) -> Result { + Ok(self.response.clone()) + } +} + +// --------------------------------------------------------------------------- +// MockStructuredProcessor — extracts content as parsed JSON (like a real +// processor). The pipeline's `process()` function then calls +// `wrap_structured_if_needed()` which wraps the result in the +// `__prompty_structured` envelope when the agent has `outputs`. +// `invoke()` and `run()` must then call `unwrap_structured()` to strip it. +// --------------------------------------------------------------------------- + +struct MockStructuredProcessor; + +#[async_trait] +impl Processor for MockStructuredProcessor { + async fn process(&self, _agent: &Prompty, response: Value) -> Result { + let content_str = response["choices"][0]["message"]["content"] + .as_str() + .unwrap_or("{}"); + + // Return parsed JSON object — the pipeline wraps this in the envelope + let data: Value = serde_json::from_str(content_str).unwrap_or(json!(content_str)); + Ok(data) + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Register mock executor + processor under a unique key. +fn register_structured_mocks(key: &str, response: Value) { + prompty::register_executor(key, MockStructuredExecutor::new(response)); + prompty::register_processor(key, MockStructuredProcessor); +} + +/// Build a Prompty agent with outputs defined (triggers structured wrapping). +fn build_structured_agent(provider_key: &str) -> Prompty { + let data = json!({ + "name": "structured_test", + "kind": "prompt", + "model": { + "id": "gpt-4", + "provider": provider_key, + }, + "instructions": "system:\nReturn JSON.\n\nuser:\n{{ question }}", + "outputs": { + "city": { "kind": "string" }, + "country": { "kind": "string" }, + }, + "template": { + "format": { "kind": "nunjucks" }, + "parser": { "kind": "prompty" }, + }, + }); + Prompty::load_from_value(&data, &LoadContext::default()) +} + +/// Canned OpenAI-style chat completion with JSON content. +fn canned_response() -> Value { + json!({ + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": r#"{"city":"Paris","country":"France"}"#, + }, + "finish_reason": "stop", + }], + }) +} + +/// The expected unwrapped data. +fn expected_data() -> Value { + json!({"city": "Paris", "country": "France"}) +} + +// =================================================================== +// invoke() — one-shot pipeline must unwrap __prompty_structured +// =================================================================== + +#[tokio::test] +async fn test_invoke_unwraps_structured_output() { + let key = "structured_invoke_test"; + register_structured_mocks(key, canned_response()); + prompty::pipeline::register_defaults(); + + let agent = build_structured_agent(key); + let inputs = json!({"question": "What is the capital of France?"}); + + let result = prompty::invoke_agent(&agent, Some(&inputs)).await.unwrap(); + + // Must be the unwrapped data, NOT the __prompty_structured envelope + assert_eq!( + result, + expected_data(), + "invoke() should unwrap structured output" + ); + assert!( + result.get("__prompty_structured").is_none(), + "invoke() result must not contain __prompty_structured marker" + ); +} + +// =================================================================== +// run() — executor+process must unwrap __prompty_structured +// =================================================================== + +#[tokio::test] +async fn test_run_unwraps_structured_output() { + let key = "structured_run_test"; + register_structured_mocks(key, canned_response()); + prompty::pipeline::register_defaults(); + + let agent = build_structured_agent(key); + + // Pre-prepare messages (what `prepare()` would produce) + let messages = vec![ + Message::text(Role::System, "Return JSON."), + Message::text(Role::User, "What is the capital of France?"), + ]; + + let result = prompty::run(&agent, &messages).await.unwrap(); + + // Must be the unwrapped data, NOT the __prompty_structured envelope + assert_eq!( + result, + expected_data(), + "run() should unwrap structured output" + ); + assert!( + result.get("__prompty_structured").is_none(), + "run() result must not contain __prompty_structured marker" + ); +} + +// =================================================================== +// Sanity: non-structured output passes through unchanged +// =================================================================== + +/// A processor that returns plain text (no structured wrapping). +struct MockPlainProcessor; + +#[async_trait] +impl Processor for MockPlainProcessor { + async fn process(&self, _agent: &Prompty, response: Value) -> Result { + let content = response["choices"][0]["message"]["content"] + .as_str() + .unwrap_or(""); + Ok(Value::String(content.to_string())) + } +} + +#[tokio::test] +async fn test_invoke_plain_output_unchanged() { + let key = "plain_invoke_test"; + let response = json!({ + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + }, + "finish_reason": "stop", + }], + }); + prompty::register_executor(key, MockStructuredExecutor::new(response)); + prompty::register_processor(key, MockPlainProcessor); + prompty::pipeline::register_defaults(); + + // Agent WITHOUT outputs — no structured wrapping expected + let data = json!({ + "name": "plain_test", + "kind": "prompt", + "model": { + "id": "gpt-4", + "provider": key, + }, + "instructions": "system:\nYou are helpful.\n\nuser:\n{{ question }}", + "template": { + "format": { "kind": "nunjucks" }, + "parser": { "kind": "prompty" }, + }, + }); + let agent = Prompty::load_from_value(&data, &LoadContext::default()); + let inputs = json!({"question": "Capital of France?"}); + + let result = prompty::invoke_agent(&agent, Some(&inputs)).await.unwrap(); + + assert_eq!( + result.as_str().unwrap(), + "The capital of France is Paris.", + "plain text should pass through unchanged" + ); +} + +#[tokio::test] +async fn test_run_plain_output_unchanged() { + let key = "plain_run_test"; + let response = json!({ + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "42", + }, + "finish_reason": "stop", + }], + }); + prompty::register_executor(key, MockStructuredExecutor::new(response)); + prompty::register_processor(key, MockPlainProcessor); + prompty::pipeline::register_defaults(); + + let data = json!({ + "name": "plain_run", + "kind": "prompt", + "model": { + "id": "gpt-4", + "provider": key, + }, + "instructions": "system:\nAnswer.\n\nuser:\nWhat is 6*7?", + "template": { + "format": { "kind": "nunjucks" }, + "parser": { "kind": "prompty" }, + }, + }); + let agent = Prompty::load_from_value(&data, &LoadContext::default()); + let messages = vec![ + Message::text(Role::System, "Answer."), + Message::text(Role::User, "What is 6*7?"), + ]; + + let result = prompty::run(&agent, &messages).await.unwrap(); + assert_eq!(result.as_str().unwrap(), "42"); +} diff --git a/runtime/typescript/.env.example b/runtime/typescript/.env.example index 10a822e6..1397eafa 100644 --- a/runtime/typescript/.env.example +++ b/runtime/typescript/.env.example @@ -21,5 +21,10 @@ AZURE_OPENAI_API_KEY= AZURE_OPENAI_CHAT_DEPLOYMENT= AZURE_OPENAI_EMBEDDING_DEPLOYMENT= +# Azure Entra ID (for keyless auth tests) +# Set AZURE_TENANT_ID so DefaultAzureCredential picks the correct tenant. +# Requires 'Cognitive Services OpenAI User' role on the resource. +AZURE_TENANT_ID= + # Anthropic ANTHROPIC_API_KEY= diff --git a/runtime/typescript/package-lock.json b/runtime/typescript/package-lock.json index 74826727..fd9672d4 100644 --- a/runtime/typescript/package-lock.json +++ b/runtime/typescript/package-lock.json @@ -3498,7 +3498,7 @@ }, "packages/anthropic": { "name": "@prompty/anthropic", - "version": "2.0.0-alpha.5", + "version": "2.0.0-alpha.8", "license": "MIT", "devDependencies": { "@anthropic-ai/sdk": "^0.39.0", @@ -3514,7 +3514,7 @@ }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.39.0", - "@prompty/core": "^2.0.0-alpha.5" + "@prompty/core": "^2.0.0-alpha.8" }, "peerDependenciesMeta": { "@anthropic-ai/sdk": { @@ -3524,7 +3524,7 @@ }, "packages/core": { "name": "@prompty/core", - "version": "2.0.0-alpha.5", + "version": "2.0.0-alpha.8", "license": "MIT", "dependencies": { "gray-matter": "^4.0.3", @@ -3558,7 +3558,7 @@ }, "packages/foundry": { "name": "@prompty/foundry", - "version": "2.0.0-alpha.5", + "version": "2.0.0-alpha.8", "license": "MIT", "dependencies": { "@azure/ai-projects": "^2.0.1", @@ -3566,8 +3566,8 @@ "openai": "^4.80.0" }, "devDependencies": { - "@prompty/core": "^2.0.0-alpha.5", - "@prompty/openai": "^2.0.0-alpha.5", + "@prompty/core": "^2.0.0-alpha.8", + "@prompty/openai": "^2.0.0-alpha.8", "@types/node": "^20.11.0", "dotenv": "^16.4.0", "tsup": "^8.4.0", @@ -3578,19 +3578,19 @@ "node": ">=18.0.0" }, "peerDependencies": { - "@prompty/core": "^2.0.0-alpha.5", - "@prompty/openai": "^2.0.0-alpha.5" + "@prompty/core": "^2.0.0-alpha.8", + "@prompty/openai": "^2.0.0-alpha.8" } }, "packages/openai": { "name": "@prompty/openai", - "version": "2.0.0-alpha.5", + "version": "2.0.0-alpha.8", "license": "MIT", "dependencies": { "openai": "^4.80.0" }, "devDependencies": { - "@prompty/core": "^2.0.0-alpha.5", + "@prompty/core": "^2.0.0-alpha.8", "@types/node": "^20.11.0", "dotenv": "^16.4.0", "tsup": "^8.4.0", @@ -3601,7 +3601,7 @@ "node": ">=18.0.0" }, "peerDependencies": { - "@prompty/core": "^2.0.0-alpha.5" + "@prompty/core": "^2.0.0-alpha.8" } } } diff --git a/runtime/typescript/packages/core/src/core/index.ts b/runtime/typescript/packages/core/src/core/index.ts index 4ed65f20..3230d31e 100644 --- a/runtime/typescript/packages/core/src/core/index.ts +++ b/runtime/typescript/packages/core/src/core/index.ts @@ -13,6 +13,7 @@ export { turn, invoke, resolveBindings, + ExecuteError, type TurnOptions, type InvokeOptions, } from "./pipeline.js"; @@ -26,6 +27,8 @@ export { getToolHandler, clearToolHandlers, dispatchTool, + resilientJsonParse, + extractFirstJsonBlock, } from "./tool-dispatch.js"; export { type AgentEventType, type EventCallback, emitEvent } from "./agent-events.js"; export { CancelledError, checkCancellation } from "./cancellation.js"; diff --git a/runtime/typescript/packages/core/src/core/pipeline.ts b/runtime/typescript/packages/core/src/core/pipeline.ts index 8fc557e6..05d9f2ba 100644 --- a/runtime/typescript/packages/core/src/core/pipeline.ts +++ b/runtime/typescript/packages/core/src/core/pipeline.ts @@ -43,7 +43,7 @@ import { getRenderer, getParser, getExecutor, getProcessor } from "./registry.js import { getLastNonces, clearLastNonces } from "../renderers/common.js"; import { traceSpan, sanitizeValue } from "../tracing/tracer.js"; import { load } from "./loader.js"; -import { dispatchTool } from "./tool-dispatch.js"; +import { dispatchTool, resilientJsonParse } from "./tool-dispatch.js"; import { type EventCallback, emitEvent } from "./agent-events.js"; import { CancelledError, checkCancellation } from "./cancellation.js"; import { trimToContextWindow } from "./context.js"; @@ -59,6 +59,25 @@ const DEFAULT_FORMAT = "nunjucks"; const DEFAULT_PARSER = "prompty"; const DEFAULT_PROVIDER = "openai"; const DEFAULT_MAX_ITERATIONS = 10; +const DEFAULT_MAX_LLM_RETRIES = 3; + +// --------------------------------------------------------------------------- +// ExecuteError (§9.10) +// --------------------------------------------------------------------------- + +/** + * Error from agent loop that includes accumulated conversation state. + * Allows callers to resume by passing messages back as thread input. + */ +export class ExecuteError extends Error { + public readonly messages: Message[]; + + constructor(message: string, messages: Message[]) { + super(message); + this.name = "ExecuteError"; + this.messages = messages; + } +} /** Replace raw nonce strings with readable `{{thread:name}}` in trace output. */ function sanitizeNonces(value: unknown): unknown { @@ -446,6 +465,61 @@ export async function invoke( }); } +// --------------------------------------------------------------------------- +// LLM call retry (§9.10) +// --------------------------------------------------------------------------- + +/** + * Invoke executor.execute with exponential-backoff retry on transient failures. + * Respects AbortSignal for cancellation during backoff. + */ +async function invokeWithRetry( + executor: { execute(agent: Prompty, messages: Message[]): Promise }, + agent: Prompty, + messages: Message[], + maxRetries: number, + onEvent?: EventCallback, + signal?: AbortSignal, +): Promise { + let attempts = 0; + while (true) { + try { + return await executor.execute(agent, messages); + } catch (err) { + // Never retry cancellation + if (err instanceof CancelledError) throw err; + attempts++; + if (attempts >= maxRetries) { + throw new ExecuteError( + `LLM call failed after ${maxRetries} retries: ${err instanceof Error ? err.message : String(err)}`, + [...messages], + ); + } + // Emit status event + emitEvent(onEvent, "status", { + message: `LLM call failed, retrying (attempt ${attempts + 1}/${maxRetries})...`, + }); + // Exponential backoff with jitter, capped at 60s (§9.10) + // backoff = min(2^attempts + jitter(), 60) — values in seconds + const backoffSecs = Math.min(Math.pow(2, attempts) + Math.random(), 60); + // Check cancellation before sleeping + if (signal?.aborted) { + throw new CancelledError("Operation cancelled during retry backoff"); + } + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, backoffSecs * 1000); + if (signal) { + const onAbort = () => { + clearTimeout(timer); + reject(new CancelledError("Operation cancelled during retry backoff")); + }; + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + } + } +} + // --------------------------------------------------------------------------- // Turn: one conversational round-trip (§14) // --------------------------------------------------------------------------- @@ -472,6 +546,8 @@ export interface TurnOptions { steering?: Steering; /** Allow parallel tool execution within a single round (§13.6). */ parallelToolCalls?: boolean; + /** Maximum LLM call retries on transient failure in agent loop (§9.10, default: 3). */ + maxLlmRetries?: number; } /** @@ -602,6 +678,7 @@ export async function turn( // Agent mode: prepare → [executor → toolCalls]* → executor → process const maxIterations = options?.maxIterations ?? DEFAULT_MAX_ITERATIONS; + const maxLlmRetries = options?.maxLlmRetries ?? DEFAULT_MAX_LLM_RETRIES; const onEvent = options?.onEvent; const signal = options?.signal; const contextBudget = options?.contextBudget; @@ -663,8 +740,8 @@ export async function turn( throw err; } - // Call LLM - response = await executor.execute(agent, messages); + // Call LLM — §9.10: retry on transient failure + response = await invokeWithRetry(executor, agent, messages, maxLlmRetries, onEvent, signal); // Streaming: consume the stream, extract tool calls from buffered chunks if (isAsyncIterable(response)) { @@ -1071,15 +1148,7 @@ async function dispatchOneToolWithExtensions( // §13.4 — Tool guardrail if (guardrails) { - let parsedArgs: Record = {}; - try { - const parsed = JSON.parse(tc.arguments); - if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) { - parsedArgs = parsed as Record; - } - } catch { - // ignore parse errors for guardrail check - } + const parsedArgs = resilientJsonParse(tc.arguments) ?? {}; const gr = guardrails.checkTool(tc.name, parsedArgs); if (!gr.allowed) { const deniedMsg = `Tool denied by guardrail: ${gr.reason}`; @@ -1095,7 +1164,14 @@ async function dispatchOneToolWithExtensions( let result: string; let parsedArgs: unknown; try { - parsedArgs = JSON.parse(tc.arguments); + // §9.8 — Resilient JSON parsing for tool arguments + parsedArgs = resilientJsonParse(tc.arguments); + if (parsedArgs === null) { + result = `Error: Tool '${tc.name}' received unparseable arguments`; + emitEvent(onEvent, "error", { tool: tc.name, error: "Unparseable tool arguments" }); + emitEvent(onEvent, "tool_result", { name: tc.name, result }); + return result; + } if (agent && parentInputs && typeof parsedArgs === "object" && parsedArgs !== null && !Array.isArray(parsedArgs)) { parsedArgs = resolveBindings(agent, tc.name, parsedArgs as Record, parentInputs); } @@ -1110,11 +1186,19 @@ async function dispatchOneToolWithExtensions( } catch (err) { // Re-throw cancellation errors if (err instanceof CancelledError) throw err; - result = `Error: ${err instanceof Error ? err.message : String(err)}`; + const errorMsg = err instanceof Error ? err.message : String(err); + // §9.9 — Emit error event on tool execution failure + emitEvent(onEvent, "error", { tool: tc.name, error: errorMsg }); + result = `Error: Tool '${tc.name}' failed: ${errorMsg}`; } // §13.1 — Emit tool_result emitEvent(onEvent, "tool_result", { name: tc.name, result }); + + // §9.9 — Emit error event when tool result indicates failure + if (result.startsWith("Error:")) { + emitEvent(onEvent, "error", { tool: tc.name, error: result }); + } return result; } diff --git a/runtime/typescript/packages/core/src/core/tool-dispatch.ts b/runtime/typescript/packages/core/src/core/tool-dispatch.ts index e53f4c84..6eeef728 100644 --- a/runtime/typescript/packages/core/src/core/tool-dispatch.ts +++ b/runtime/typescript/packages/core/src/core/tool-dispatch.ts @@ -259,6 +259,98 @@ class CustomToolHandler implements ToolHandler { } } +// --------------------------------------------------------------------------- +// Resilient JSON parsing (§9.8) +// --------------------------------------------------------------------------- + +/** + * Extract the first balanced `{...}` block from text, respecting string escapes. + */ +export function extractFirstJsonBlock(text: string): string | null { + const start = text.indexOf("{"); + if (start === -1) return null; + + let depth = 0; + let inString = false; + let escapeNext = false; + + for (let i = start; i < text.length; i++) { + const ch = text[i]; + if (escapeNext) { + escapeNext = false; + continue; + } + if (inString) { + if (ch === "\\") escapeNext = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') inString = true; + else if (ch === "{") depth++; + else if (ch === "}") { + depth--; + if (depth === 0) return text.slice(start, i + 1); + } + } + return null; +} + +/** + * Parse JSON with fallback strategies per spec §9.8. + * Returns parsed value on success, or null if all strategies fail. + */ +export function resilientJsonParse(raw: string): Record | null { + // Strategy 1: Direct parse + try { + const result = JSON.parse(raw); + if (typeof result === "object" && result !== null) return result as Record; + return { _raw: result }; + } catch { + // continue to fallbacks + } + + // Strategy 2: Strip markdown code fences + const fenceMatch = raw.match(/^\s*```(?:json)?\s*\n?([\s\S]*?)\n?\s*```\s*$/); + if (fenceMatch) { + try { + const result = JSON.parse(fenceMatch[1]); + console.warn("[prompty] Parsed tool arguments after stripping markdown fences"); + if (typeof result === "object" && result !== null) return result as Record; + return { _raw: result }; + } catch { + // continue + } + } + + // Strategy 3: Extract first balanced JSON block + const block = extractFirstJsonBlock(raw); + if (block !== null) { + try { + const result = JSON.parse(block); + console.warn("[prompty] Parsed tool arguments after extracting JSON block"); + if (typeof result === "object" && result !== null) return result as Record; + return { _raw: result }; + } catch { + // continue + } + } + + // Strategy 4: Strip trailing commas before } or ] + const cleaned = raw.replace(/,\s*([}\]])/g, "$1"); + if (cleaned !== raw) { + try { + const result = JSON.parse(cleaned); + console.warn("[prompty] Parsed tool arguments after stripping trailing commas"); + if (typeof result === "object" && result !== null) return result as Record; + return { _raw: result }; + } catch { + // continue + } + } + + return null; // All strategies failed +} + // --------------------------------------------------------------------------- // Main dispatch function // --------------------------------------------------------------------------- diff --git a/runtime/typescript/packages/core/tests/resilience.test.ts b/runtime/typescript/packages/core/tests/resilience.test.ts new file mode 100644 index 00000000..dd0d4af2 --- /dev/null +++ b/runtime/typescript/packages/core/tests/resilience.test.ts @@ -0,0 +1,524 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { + resilientJsonParse, + extractFirstJsonBlock, +} from "../src/core/tool-dispatch.js"; +import { + turn, + ExecuteError, +} from "../src/core/pipeline.js"; +import { + registerRenderer, + registerParser, + registerExecutor, + registerProcessor, +} from "../src/core/registry.js"; +import { Message, text } from "../src/core/types.js"; +import { Prompty } from "@prompty/core"; +import type { Renderer, Parser, Executor, Processor } from "../src/core/interfaces.js"; + +// --------------------------------------------------------------------------- +// Resilient JSON Parsing (§9.8) +// --------------------------------------------------------------------------- + +describe("Resilient JSON Parsing (§9.8)", () => { + it("parses valid JSON directly", () => { + expect(resilientJsonParse('{"city": "NY"}')).toEqual({ city: "NY" }); + }); + + it("parses JSON array and returns object", () => { + const result = resilientJsonParse("[1, 2, 3]"); + expect(result).toEqual([1, 2, 3]); + }); + + it("wraps primitives in _raw", () => { + expect(resilientJsonParse("42")).toEqual({ _raw: 42 }); + expect(resilientJsonParse('"hello"')).toEqual({ _raw: "hello" }); + }); + + it("strips markdown fences", () => { + const raw = '```json\n{"city": "NY"}\n```'; + expect(resilientJsonParse(raw)).toEqual({ city: "NY" }); + }); + + it("strips markdown fences without json hint", () => { + const raw = '```\n{"temp": 72}\n```'; + expect(resilientJsonParse(raw)).toEqual({ temp: 72 }); + }); + + it("extracts JSON block from prose", () => { + const raw = 'Here is the result: {"city": "NY"} enjoy!'; + expect(resilientJsonParse(raw)).toEqual({ city: "NY" }); + }); + + it("strips trailing commas", () => { + const raw = '{"city": "NY", "temp": 72,}'; + const result = resilientJsonParse(raw); + expect(result).toEqual({ city: "NY", temp: 72 }); + }); + + it("strips trailing comma in array", () => { + const raw = '{"items": [1, 2, 3,]}'; + const result = resilientJsonParse(raw); + expect(result).toEqual({ items: [1, 2, 3] }); + }); + + it("returns null when all strategies fail", () => { + expect(resilientJsonParse("not json at all")).toBeNull(); + }); + + it("does NOT silently substitute empty object", () => { + expect(resilientJsonParse("garbage")).toBeNull(); + }); + + it("handles empty string", () => { + expect(resilientJsonParse("")).toBeNull(); + }); + + it("handles nested valid JSON", () => { + const raw = '{"a": {"b": {"c": 1}}}'; + expect(resilientJsonParse(raw)).toEqual({ a: { b: { c: 1 } } }); + }); +}); + +// --------------------------------------------------------------------------- +// extractFirstJsonBlock +// --------------------------------------------------------------------------- + +describe("extractFirstJsonBlock", () => { + it("respects string escapes", () => { + const raw = 'prefix {"key": "value with {braces}"} suffix'; + const block = extractFirstJsonBlock(raw); + expect(JSON.parse(block!)).toEqual({ key: "value with {braces}" }); + }); + + it("returns null with no JSON", () => { + expect(extractFirstJsonBlock("no json here")).toBeNull(); + }); + + it("handles nested objects", () => { + const raw = 'text {"a": {"b": 1}} more'; + const block = extractFirstJsonBlock(raw); + expect(JSON.parse(block!)).toEqual({ a: { b: 1 } }); + }); + + it("handles escaped quotes in strings", () => { + const raw = 'x {"key": "val\\"ue"} y'; + const block = extractFirstJsonBlock(raw); + expect(JSON.parse(block!)).toEqual({ key: 'val"ue' }); + }); + + it("returns null for unbalanced braces", () => { + expect(extractFirstJsonBlock("text { no close")).toBeNull(); + }); + + it("extracts first block when multiple exist", () => { + const raw = '{"a": 1} and {"b": 2}'; + const block = extractFirstJsonBlock(raw); + expect(JSON.parse(block!)).toEqual({ a: 1 }); + }); +}); + +// --------------------------------------------------------------------------- +// LLM Call Retry (§9.10) +// --------------------------------------------------------------------------- + +class MockRenderer implements Renderer { + async render(_agent: Prompty, template: string, inputs: Record): Promise { + let result = template; + for (const [key, val] of Object.entries(inputs)) { + result = result.replace(`{{${key}}}`, String(val)); + } + return result; + } +} + +class MockParser implements Parser { + async parse(_agent: Prompty, rendered: string): Promise { + return [new Message("user", [text(rendered)])]; + } +} + +class MockProcessor implements Processor { + async process(_agent: Prompty, response: unknown): Promise { + const r = response as Record; + const choices = r.choices as Record[]; + const msg = choices[0].message as Record; + return msg.content; + } +} + +function makeAgent(): Prompty { + const agent = new Prompty({ + name: "test", + model: "gpt-4o", + instructions: "Hello, {{name}}!", + }); + agent.template = { format: { kind: "mock" }, parser: { kind: "mock" } } as any; + (agent as any).model = { provider: "retrymock" }; + return agent; +} + +describe("LLM Call Retry (§9.10)", () => { + beforeEach(() => { + vi.useFakeTimers(); + registerRenderer("mock", new MockRenderer()); + registerParser("mock", new MockParser()); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("retries on transient failure and succeeds", async () => { + let callCount = 0; + const retryExecutor: Executor = { + async execute(_agent: Prompty, _messages: Message[]): Promise { + callCount++; + if (callCount === 1) { + throw new Error("Transient network error"); + } + // Second call: return tool calls first time + if (callCount === 2) { + return { + choices: [{ + message: { + role: "assistant", + content: null, + tool_calls: [{ + id: "call_1", + type: "function", + function: { name: "greet", arguments: '{"who":"World"}' }, + }], + }, + }], + }; + } + // Third call: return final response + return { + choices: [{ message: { role: "assistant", content: "Retry success!" } }], + }; + }, + formatToolMessages( + _rawResponse: unknown, + toolCalls: { id: string; name: string; arguments: string }[], + toolResults: string[], + textContent = "", + ): Message[] { + const messages: Message[] = []; + const rawToolCalls = toolCalls.map((tc) => ({ + id: tc.id, type: "function", function: { name: tc.name, arguments: tc.arguments }, + })); + messages.push(new Message("assistant", textContent ? [text(textContent)] : [], { tool_calls: rawToolCalls })); + for (let i = 0; i < toolCalls.length; i++) { + messages.push(new Message("tool", [text(toolResults[i])], { tool_call_id: toolCalls[i].id, name: toolCalls[i].name })); + } + return messages; + }, + }; + + registerExecutor("retrymock", retryExecutor); + registerProcessor("retrymock", new MockProcessor()); + + const agent = makeAgent(); + const tools = { greet: (args: Record) => `Hello ${args.who}!` }; + + const promise = turn(agent, { name: "Test" }, { tools: tools as any, maxLlmRetries: 3 }); + await vi.runAllTimersAsync(); + const result = await promise; + expect(result).toBe("Retry success!"); + expect(callCount).toBe(3); // 1 fail + 1 tool call + 1 final + }); + + it("throws ExecuteError with messages on exhaustion", async () => { + const alwaysFailExecutor: Executor = { + async execute(): Promise { + throw new Error("Service unavailable"); + }, + formatToolMessages( + _rawResponse: unknown, + toolCalls: { id: string; name: string; arguments: string }[], + toolResults: string[], + textContent = "", + ): Message[] { + return []; + }, + }; + + registerExecutor("retrymock", alwaysFailExecutor); + registerProcessor("retrymock", new MockProcessor()); + + const agent = makeAgent(); + const tools = { greet: () => "hello" }; + + const promise = turn(agent, { name: "Test" }, { tools: tools as any, maxLlmRetries: 2 }); + // Attach catch handler immediately to prevent unhandled rejection during timer flush + const settled = promise.catch((e: unknown) => e); + await vi.runAllTimersAsync(); + const err = await settled; + expect(err).toBeInstanceOf(ExecuteError); + const execErr = err as ExecuteError; + expect(execErr.message).toContain("2 retries"); + expect(execErr.message).toContain("Service unavailable"); + expect(execErr.messages).toBeInstanceOf(Array); + expect(execErr.messages.length).toBeGreaterThan(0); + }); + + it("emits status event before retry", async () => { + let callCount = 0; + const flakeyExecutor: Executor = { + async execute(_agent: Prompty, _messages: Message[]): Promise { + callCount++; + if (callCount === 1) throw new Error("Temporary failure"); + // Return tool call then final + if (callCount === 2) { + return { + choices: [{ + message: { + role: "assistant", + content: null, + tool_calls: [{ + id: "call_1", + type: "function", + function: { name: "greet", arguments: '{}' }, + }], + }, + }], + }; + } + return { choices: [{ message: { role: "assistant", content: "Done" } }] }; + }, + formatToolMessages( + _rawResponse: unknown, + toolCalls: { id: string; name: string; arguments: string }[], + toolResults: string[], + textContent = "", + ): Message[] { + const messages: Message[] = []; + const rawToolCalls = toolCalls.map((tc) => ({ + id: tc.id, type: "function", function: { name: tc.name, arguments: tc.arguments }, + })); + messages.push(new Message("assistant", textContent ? [text(textContent)] : [], { tool_calls: rawToolCalls })); + for (let i = 0; i < toolCalls.length; i++) { + messages.push(new Message("tool", [text(toolResults[i])], { tool_call_id: toolCalls[i].id, name: toolCalls[i].name })); + } + return messages; + }, + }; + + registerExecutor("retrymock", flakeyExecutor); + registerProcessor("retrymock", new MockProcessor()); + + const events: Array<{ type: string; data: Record }> = []; + const onEvent = (type: string, data: Record) => { + events.push({ type, data }); + }; + + const agent = makeAgent(); + const tools = { greet: () => "hi" }; + + const promise = turn(agent, { name: "Test" }, { + tools: tools as any, + maxLlmRetries: 3, + onEvent: onEvent as any, + }); + await vi.runAllTimersAsync(); + await promise; + + const statusEvents = events.filter(e => e.type === "status"); + const retryEvent = statusEvents.find(e => + typeof e.data.message === "string" && e.data.message.includes("retrying"), + ); + expect(retryEvent).toBeDefined(); + expect(retryEvent!.data.message).toContain("attempt 2/3"); + }); + + it("does not retry in simple mode (no tools)", async () => { + let callCount = 0; + const failOnceExecutor: Executor = { + async execute(): Promise { + callCount++; + throw new Error("API error"); + }, + formatToolMessages() { return []; }, + }; + + registerExecutor("retrymock", failOnceExecutor); + registerProcessor("retrymock", new MockProcessor()); + + const agent = makeAgent(); + + // No tools = simple mode, should NOT retry + await expect( + turn(agent, { name: "Test" }), + ).rejects.toThrow("API error"); + expect(callCount).toBe(1); // Only called once, no retry + }); + + it("respects maxLlmRetries: 1 (no retries)", async () => { + let callCount = 0; + const alwaysFail: Executor = { + async execute(): Promise { + callCount++; + throw new Error("fail"); + }, + formatToolMessages() { return []; }, + }; + + registerExecutor("retrymock", alwaysFail); + registerProcessor("retrymock", new MockProcessor()); + + const agent = makeAgent(); + const tools = { greet: () => "hi" }; + + const promise = turn(agent, { name: "Test" }, { tools: tools as any, maxLlmRetries: 1 }); + const settled = promise.catch((e: unknown) => e); + await vi.runAllTimersAsync(); + const err = await settled; + expect(err).toBeInstanceOf(ExecuteError); + expect(callCount).toBe(1); // Only one attempt + }); +}); + +// --------------------------------------------------------------------------- +// Tool Execution Error Safety (§9.9) +// --------------------------------------------------------------------------- + +describe("Tool Execution Error Safety (§9.9)", () => { + beforeEach(() => { + registerRenderer("mock", new MockRenderer()); + registerParser("mock", new MockParser()); + }); + + it("emits error event when tool execution fails", async () => { + let callCount = 0; + const toolCallExecutor: Executor = { + async execute(_agent: Prompty, _messages: Message[]): Promise { + callCount++; + if (callCount === 1) { + return { + choices: [{ + message: { + role: "assistant", + content: null, + tool_calls: [{ + id: "call_1", + type: "function", + function: { name: "bad_tool", arguments: '{}' }, + }], + }, + }], + }; + } + return { choices: [{ message: { role: "assistant", content: "Recovered" } }] }; + }, + formatToolMessages( + _rawResponse: unknown, + toolCalls: { id: string; name: string; arguments: string }[], + toolResults: string[], + textContent = "", + ): Message[] { + const messages: Message[] = []; + const rawToolCalls = toolCalls.map((tc) => ({ + id: tc.id, type: "function", function: { name: tc.name, arguments: tc.arguments }, + })); + messages.push(new Message("assistant", textContent ? [text(textContent)] : [], { tool_calls: rawToolCalls })); + for (let i = 0; i < toolCalls.length; i++) { + messages.push(new Message("tool", [text(toolResults[i])], { tool_call_id: toolCalls[i].id, name: toolCalls[i].name })); + } + return messages; + }, + }; + + registerExecutor("retrymock", toolCallExecutor); + registerProcessor("retrymock", new MockProcessor()); + + const events: Array<{ type: string; data: Record }> = []; + const onEvent = (type: string, data: Record) => { + events.push({ type, data }); + }; + + const agent = makeAgent(); + // Tool that throws + const tools = { + bad_tool: () => { throw new Error("tool explosion"); }, + }; + + const result = await turn(agent, { name: "Test" }, { + tools: tools as any, + onEvent: onEvent as any, + maxLlmRetries: 1, + }); + + // The loop should recover and produce a final result + expect(result).toBe("Recovered"); + + // An error event should have been emitted for the tool failure + const errorEvents = events.filter(e => e.type === "error"); + expect(errorEvents.length).toBeGreaterThan(0); + const toolError = errorEvents.find(e => e.data.tool === "bad_tool"); + expect(toolError).toBeDefined(); + expect(toolError!.data.error).toContain("tool explosion"); + }); + + it("tool errors are returned as strings, not thrown", async () => { + let callCount = 0; + const toolCallExecutor: Executor = { + async execute(_agent: Prompty, messages: Message[]): Promise { + callCount++; + if (callCount === 1) { + return { + choices: [{ + message: { + role: "assistant", + content: null, + tool_calls: [{ + id: "call_1", + type: "function", + function: { name: "crashing_tool", arguments: '{"x": 1}' }, + }], + }, + }], + }; + } + // The tool result with the error message gets sent back to LLM + const lastMsg = messages[messages.length - 1]; + expect(lastMsg.role).toBe("tool"); + expect(lastMsg.text).toContain("Error"); + return { choices: [{ message: { role: "assistant", content: "Handled error" } }] }; + }, + formatToolMessages( + _rawResponse: unknown, + toolCalls: { id: string; name: string; arguments: string }[], + toolResults: string[], + textContent = "", + ): Message[] { + const messages: Message[] = []; + const rawToolCalls = toolCalls.map((tc) => ({ + id: tc.id, type: "function", function: { name: tc.name, arguments: tc.arguments }, + })); + messages.push(new Message("assistant", textContent ? [text(textContent)] : [], { tool_calls: rawToolCalls })); + for (let i = 0; i < toolCalls.length; i++) { + messages.push(new Message("tool", [text(toolResults[i])], { tool_call_id: toolCalls[i].id, name: toolCalls[i].name })); + } + return messages; + }, + }; + + registerExecutor("retrymock", toolCallExecutor); + registerProcessor("retrymock", new MockProcessor()); + + const agent = makeAgent(); + const tools = { + crashing_tool: () => { throw new Error("BOOM"); }, + }; + + // Should NOT throw — error is returned as string to the LLM + const result = await turn(agent, { name: "Test" }, { + tools: tools as any, + maxLlmRetries: 1, + }); + expect(result).toBe("Handled error"); + expect(callCount).toBe(2); + }); +}); diff --git a/runtime/typescript/packages/core/tests/spec-vectors.test.ts b/runtime/typescript/packages/core/tests/spec-vectors.test.ts index 315356fe..015938e0 100644 --- a/runtime/typescript/packages/core/tests/spec-vectors.test.ts +++ b/runtime/typescript/packages/core/tests/spec-vectors.test.ts @@ -990,6 +990,7 @@ describe("Spec Vectors: Agent", () => { await expect( turn(agent, input.parent_inputs ?? {}, { tools: toolFunctions, + maxLlmRetries: 1, }), ).rejects.toThrow("maxIterations"); } else if (expected.error.includes("not registered")) { @@ -998,6 +999,7 @@ describe("Spec Vectors: Agent", () => { try { await turn(agent, input.parent_inputs ?? {}, { tools: toolFunctions, + maxLlmRetries: 1, }); } catch { // Mock may run out of responses — that's fine @@ -1007,6 +1009,7 @@ describe("Spec Vectors: Agent", () => { // Success vectors const result = await turn(agent, input.parent_inputs ?? {}, { tools: toolFunctions, + maxLlmRetries: 1, }); // Validate result @@ -1168,7 +1171,7 @@ describe("Spec Vectors: Agent Extensions (§13)", () => { try { // --- Build extension options --- - const opts: Record = { tools: toolFunctions }; + const opts: Record = { tools: toolFunctions, maxLlmRetries: 1 }; // Events const collectedEvents: Array<{ type: string; data: Record }> = []; diff --git a/runtime/typescript/packages/core/tests/structured-pipeline.test.ts b/runtime/typescript/packages/core/tests/structured-pipeline.test.ts new file mode 100644 index 00000000..5ce27c4b --- /dev/null +++ b/runtime/typescript/packages/core/tests/structured-pipeline.test.ts @@ -0,0 +1,405 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { + invoke, + run, + process, +} from "../src/core/pipeline.js"; +import { + registerRenderer, + registerParser, + registerExecutor, + registerProcessor, +} from "../src/core/registry.js"; +import { Message, text } from "../src/core/types.js"; +import { Prompty, Property } from "@prompty/core"; +import type { Renderer, Parser, Executor, Processor } from "../src/core/interfaces.js"; +import { + createStructuredResult, + isStructuredResult, + StructuredResultSymbol, + cast, +} from "../src/core/structured.js"; + +// --------------------------------------------------------------------------- +// Test data: the structured JSON an LLM would return +// --------------------------------------------------------------------------- + +const WEATHER_JSON = '{"city":"Seattle","temperature":72.5,"unit":"F"}'; +const WEATHER_DATA = { city: "Seattle", temperature: 72.5, unit: "F" }; + +const PERSON_JSON = '{"name":"Jane","age":30,"email":"jane@example.com"}'; +const PERSON_DATA = { name: "Jane", age: 30, email: "jane@example.com" }; + +// --------------------------------------------------------------------------- +// Mock implementations that simulate real providers returning structured output +// --------------------------------------------------------------------------- + +class MockRenderer implements Renderer { + async render(_agent: Prompty, template: string, inputs: Record): Promise { + let result = template; + for (const [key, val] of Object.entries(inputs)) { + result = result.replace(`{{${key}}}`, String(val)); + } + return result; + } +} + +class MockParser implements Parser { + async parse(_agent: Prompty, rendered: string): Promise { + return [new Message("user", [text(rendered)])]; + } +} + +/** + * Mock executor that returns a raw LLM response envelope containing + * structured JSON in the content field — mimicking what OpenAI returns + * when response_format is set. + */ +class StructuredExecutor implements Executor { + constructor(private jsonContent: string = WEATHER_JSON) {} + + async execute(_agent: Prompty, _messages: Message[]): Promise { + return { + choices: [{ + message: { + role: "assistant", + content: this.jsonContent, + }, + finish_reason: "stop", + }], + }; + } + + formatToolMessages( + _rawResponse: unknown, + toolCalls: { id: string; name: string; arguments: string }[], + toolResults: string[], + textContent = "", + ): Message[] { + const messages: Message[] = []; + const rawToolCalls = toolCalls.map((tc) => ({ + id: tc.id, + type: "function", + function: { name: tc.name, arguments: tc.arguments }, + })); + messages.push( + new Message("assistant", textContent ? [text(textContent)] : [], { + tool_calls: rawToolCalls, + }), + ); + for (let i = 0; i < toolCalls.length; i++) { + messages.push( + new Message("tool", [text(toolResults[i])], { + tool_call_id: toolCalls[i].id, + name: toolCalls[i].name, + }), + ); + } + return messages; + } +} + +/** + * Mock processor that behaves like real OpenAI/Anthropic processors: + * extracts content from the LLM response envelope and wraps structured + * JSON in a StructuredResult when the agent has outputs defined. + */ +class StructuredProcessor implements Processor { + async process(agent: Prompty, response: unknown): Promise { + const r = response as Record; + const choices = r.choices as Record[]; + const msg = choices[0].message as Record; + const content = msg.content as string; + + // Mirror real processor behavior: when agent has outputs, wrap in StructuredResult + if (agent.outputs && agent.outputs.length > 0) { + const parsed = JSON.parse(content) as Record; + return createStructuredResult(parsed, content); + } + + return content; + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeStructuredAgent(overrides?: { + name?: string; + instructions?: string; + jsonContent?: string; +}): Prompty { + const agent = new Prompty({ + name: overrides?.name ?? "structured-test", + instructions: overrides?.instructions ?? "Return weather data for {{city}}.", + outputs: [ + new Property({ name: "city", kind: "string", description: "City name" }), + new Property({ name: "temperature", kind: "float", description: "Temperature value" }), + new Property({ name: "unit", kind: "string", description: "Temperature unit" }), + ], + }); + agent.template = { format: { kind: "struct-mock" }, parser: { kind: "struct-mock" } } as any; + (agent as any).model = { provider: "struct-mock" }; + return agent; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("Structured output through pipeline", () => { + beforeEach(() => { + registerRenderer("struct-mock", new MockRenderer()); + registerParser("struct-mock", new MockParser()); + registerExecutor("struct-mock", new StructuredExecutor(WEATHER_JSON)); + registerProcessor("struct-mock", new StructuredProcessor()); + }); + + // ------------------------------------------------------------------------- + // invoke() + // ------------------------------------------------------------------------- + + describe("invoke() with structured output", () => { + it("returns a StructuredResult with expected data fields", async () => { + const agent = makeStructuredAgent(); + const result = await invoke(agent, { city: "Seattle" }); + + // The result should be a StructuredResult — not a raw envelope + expect(isStructuredResult(result)).toBe(true); + + // Data fields are directly accessible (no .choices[0].message wrapper) + const sr = result as Record; + expect(sr.city).toBe("Seattle"); + expect(sr.temperature).toBe(72.5); + expect(sr.unit).toBe("F"); + }); + + it("StructuredResult carries raw JSON via symbol", async () => { + const agent = makeStructuredAgent(); + const result = await invoke(agent, { city: "Seattle" }); + + expect(isStructuredResult(result)).toBe(true); + const sr = result as any; + expect(sr[StructuredResultSymbol]).toBe(WEATHER_JSON); + }); + + it("StructuredResult symbol is not enumerable — clean serialization", async () => { + const agent = makeStructuredAgent(); + const result = await invoke(agent, { city: "Seattle" }); + + // Object.keys and JSON.stringify should not include the symbol + const keys = Object.keys(result as object); + expect(keys).toEqual(["city", "temperature", "unit"]); + expect(JSON.stringify(result)).toBe(WEATHER_JSON); + }); + + it("cast() works on invoke() result", async () => { + const agent = makeStructuredAgent(); + const result = await invoke(agent, { city: "Seattle" }); + + interface Weather { + city: string; + temperature: number; + unit: string; + } + const weather = cast(result); + expect(weather.city).toBe("Seattle"); + expect(weather.temperature).toBe(72.5); + expect(weather.unit).toBe("F"); + }); + + it("invoke() with validator returns typed result directly", async () => { + const agent = makeStructuredAgent(); + + interface Weather { + city: string; + temperature: number; + unit: string; + } + const weather = await invoke(agent, { city: "Seattle" }, { + validator: (data: unknown) => { + const d = data as Weather; + if (typeof d.city !== "string") throw new Error("city must be string"); + return d; + }, + }); + + expect(weather.city).toBe("Seattle"); + expect(weather.temperature).toBe(72.5); + }); + + it("returns plain string when agent has no outputs", async () => { + // Agent without outputs — processor returns raw string content + const agent = new Prompty({ + name: "no-outputs", + instructions: "Hello {{name}}", + }); + agent.template = { format: { kind: "struct-mock" }, parser: { kind: "struct-mock" } } as any; + (agent as any).model = { provider: "struct-mock" }; + + const result = await invoke(agent, { name: "World" }); + // Without outputs, processor returns the raw content string + expect(typeof result).toBe("string"); + expect(isStructuredResult(result)).toBe(false); + }); + }); + + // ------------------------------------------------------------------------- + // run() + // ------------------------------------------------------------------------- + + describe("run() with structured output", () => { + it("returns a StructuredResult with accessible data fields", async () => { + const agent = makeStructuredAgent(); + const messages = [new Message("user", [text("What is the weather?")])]; + + const result = await run(agent, messages); + + expect(isStructuredResult(result)).toBe(true); + const sr = result as Record; + expect(sr.city).toBe("Seattle"); + expect(sr.temperature).toBe(72.5); + expect(sr.unit).toBe("F"); + }); + + it("cast() works on run() result", async () => { + const agent = makeStructuredAgent(); + const messages = [new Message("user", [text("What is the weather?")])]; + + const result = await run(agent, messages); + + interface Weather { + city: string; + temperature: number; + unit: string; + } + const weather = cast(result); + expect(weather.city).toBe("Seattle"); + expect(weather.temperature).toBe(72.5); + }); + + it("run() with raw=true returns executor envelope, not StructuredResult", async () => { + const agent = makeStructuredAgent(); + const messages = [new Message("user", [text("What is the weather?")])]; + + const result = await run(agent, messages, { raw: true }) as Record; + + // Raw mode skips the processor — returns the executor's raw response + expect(isStructuredResult(result)).toBe(false); + expect(result.choices).toBeDefined(); + }); + }); + + // ------------------------------------------------------------------------- + // process() + // ------------------------------------------------------------------------- + + describe("process() with structured output", () => { + it("wraps structured JSON in StructuredResult when agent has outputs", async () => { + const agent = makeStructuredAgent(); + const rawResponse = { + choices: [{ message: { role: "assistant", content: WEATHER_JSON } }], + }; + + const result = await process(agent, rawResponse); + + expect(isStructuredResult(result)).toBe(true); + const sr = result as Record; + expect(sr.city).toBe("Seattle"); + }); + + it("returns plain string when agent has no outputs", async () => { + const agent = new Prompty({ + name: "no-outputs", + instructions: "Hello", + }); + (agent as any).model = { provider: "struct-mock" }; + + const rawResponse = { + choices: [{ message: { role: "assistant", content: "plain text" } }], + }; + + const result = await process(agent, rawResponse); + expect(typeof result).toBe("string"); + expect(result).toBe("plain text"); + }); + }); + + // ------------------------------------------------------------------------- + // cast() integration with pipeline results + // ------------------------------------------------------------------------- + + describe("cast() with different data shapes", () => { + it("works with a different structured schema through invoke()", async () => { + // Register executor that returns person data + registerExecutor("person-mock", new StructuredExecutor(PERSON_JSON)); + registerProcessor("person-mock", new StructuredProcessor()); + + const agent = new Prompty({ + name: "person-test", + instructions: "Return person info", + outputs: [ + new Property({ name: "name", kind: "string" }), + new Property({ name: "age", kind: "integer" }), + new Property({ name: "email", kind: "string" }), + ], + }); + agent.template = { format: { kind: "struct-mock" }, parser: { kind: "struct-mock" } } as any; + (agent as any).model = { provider: "person-mock" }; + + const result = await invoke(agent, {}); + + interface Person { + name: string; + age: number; + email: string; + } + const person = cast(result); + expect(person.name).toBe("Jane"); + expect(person.age).toBe(30); + expect(person.email).toBe("jane@example.com"); + }); + + it("validator can transform the cast result", async () => { + const agent = makeStructuredAgent(); + const result = await invoke(agent, { city: "Seattle" }); + + // Validator that converts temperature to Celsius + const celsius = cast(result, (data: unknown) => { + const d = data as { city: string; temperature: number; unit: string }; + return { + city: d.city, + temperature: Math.round(((d.temperature - 32) * 5) / 9), + unit: "C", + }; + }); + + expect(celsius.city).toBe("Seattle"); + expect(celsius.temperature).toBe(23); // (72.5 - 32) * 5/9 ≈ 22.5 → 23 + expect(celsius.unit).toBe("C"); + }); + + it("cast uses raw JSON from StructuredResult, not round-tripped data", async () => { + // Use JSON with specific whitespace to verify raw JSON is preserved + const rawWithSpaces = '{"city": "Seattle", "temperature": 72.5, "unit": "F"}'; + registerExecutor("ws-mock", new StructuredExecutor(rawWithSpaces)); + registerProcessor("ws-mock", new StructuredProcessor()); + + const agent = makeStructuredAgent(); + (agent as any).model = { provider: "ws-mock" }; + + const result = await invoke(agent, { city: "Seattle" }); + expect(isStructuredResult(result)).toBe(true); + + // The StructuredResult should carry the original whitespace-heavy JSON + const sr = result as any; + expect(sr[StructuredResultSymbol]).toBe(rawWithSpaces); + + // cast still produces correct values + const weather = cast<{ city: string; temperature: number }>(result); + expect(weather.city).toBe("Seattle"); + expect(weather.temperature).toBe(72.5); + }); + }); +}); diff --git a/runtime/typescript/packages/foundry/tests/entra-id.test.ts b/runtime/typescript/packages/foundry/tests/entra-id.test.ts new file mode 100644 index 00000000..b461eb56 --- /dev/null +++ b/runtime/typescript/packages/foundry/tests/entra-id.test.ts @@ -0,0 +1,191 @@ +/** + * Foundry Entra ID (keyless / DefaultAzureCredential) integration tests. + * + * These tests verify the FoundryConnection path in the Foundry executor, + * which creates an AzureOpenAI client internally using DefaultAzureCredential + * instead of a pre-registered API-key client. + * + * Prerequisites: + * - `az login` (or another credential source for DefaultAzureCredential) + * - AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_CHAT_DEPLOYMENT env vars set + * - AZURE_TENANT_ID env var set (so DefaultAzureCredential picks the right tenant) + * - Identity must have 'Cognitive Services OpenAI User' role on the resource + * + * Skipped automatically when env vars or credentials are unavailable. + */ +import "dotenv/config"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { readFileSync, existsSync } from "fs"; +import { resolve } from "path"; +import { + Tracer, + invoke, + clearConnections, + registerExecutor, + registerProcessor, + Prompty, +} from "@prompty/core"; +import { FoundryExecutor } from "../src/executor.js"; +import { FoundryProcessor } from "../src/processor.js"; + +// --------------------------------------------------------------------------- +// Env loading (reads workspace-root .env, same pattern as integration.test.ts) +// --------------------------------------------------------------------------- +function loadEnv() { + const envPath = resolve(__dirname, "../../../.env"); + if (!existsSync(envPath)) return; + for (const line of readFileSync(envPath, "utf-8").split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eq = trimmed.indexOf("="); + if (eq < 0) continue; + const key = trimmed.slice(0, eq).trim(); + const value = trimmed.slice(eq + 1).trim(); + if (key && value && !process.env[key]) process.env[key] = value; + } +} +loadEnv(); + +const AZURE_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT; +const AZURE_CHAT_DEPLOYMENT = process.env.AZURE_OPENAI_CHAT_DEPLOYMENT; +const hasEntraId = !!(AZURE_ENDPOINT && AZURE_CHAT_DEPLOYMENT); + +// --------------------------------------------------------------------------- +// Helper — build an agent that uses FoundryConnection (Entra ID, no API key) +// --------------------------------------------------------------------------- +function makeEntraIdAgent(opts: { + apiType?: string; + instructions?: string; + deployment?: string; + options?: Record; + inputs?: Record[]; +} = {}): Prompty { + return Prompty.load({ + name: "entra-id-integration", + model: { + id: opts.deployment || AZURE_CHAT_DEPLOYMENT, + provider: "foundry", + apiType: opts.apiType || "chat", + connection: { kind: "foundry", endpoint: AZURE_ENDPOINT }, + options: { temperature: 0, maxOutputTokens: 200, ...(opts.options || {}) }, + }, + template: { format: { kind: "jinja2" }, parser: { kind: "prompty" } }, + instructions: + opts.instructions ?? + "system:\nYou are a helpful assistant. Be very brief.\nuser:\n{{question}}", + inputs: opts.inputs ?? [ + { name: "question", kind: "string", default: "Say hello in exactly 3 words." }, + ], + }); +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- +describe.skipIf(!hasEntraId)("Foundry Entra ID Integration", () => { + // The AzureOpenAI SDK auto-reads AZURE_OPENAI_API_KEY from the environment. + // When both apiKey and azureADTokenProvider are present, it throws. We must + // temporarily clear the API key env var so DefaultAzureCredential is the + // sole auth mechanism — which is exactly what these tests verify. + let savedApiKey: string | undefined; + let savedBaseURL: string | undefined; + + beforeEach(() => { + Tracer.clear(); + clearConnections(); + registerExecutor("foundry", new FoundryExecutor()); + registerProcessor("foundry", new FoundryProcessor()); + + savedApiKey = process.env.AZURE_OPENAI_API_KEY; + savedBaseURL = process.env.OPENAI_BASE_URL; + delete process.env.AZURE_OPENAI_API_KEY; + delete process.env.OPENAI_BASE_URL; + }); + + afterEach(() => { + clearConnections(); + Tracer.clear(); + + if (savedApiKey !== undefined) process.env.AZURE_OPENAI_API_KEY = savedApiKey; + else delete process.env.AZURE_OPENAI_API_KEY; + if (savedBaseURL !== undefined) process.env.OPENAI_BASE_URL = savedBaseURL; + else delete process.env.OPENAI_BASE_URL; + }); + + // --- Token acquisition --- + it("acquires a token via DefaultAzureCredential", { timeout: 30_000 }, async () => { + const { DefaultAzureCredential } = await import("@azure/identity"); + const credential = new DefaultAzureCredential(); + try { + const token = await credential.getToken( + "https://cognitiveservices.azure.com/.default", + ); + expect(token.token).toBeTruthy(); + expect(token.token.length).toBeGreaterThan(0); + expect(token.expiresOnTimestamp).toBeGreaterThan(Date.now()); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + // Skip gracefully if credentials unavailable + console.warn( + `Skipping: DefaultAzureCredential failed — run \`az login\` first. (${msg})`, + ); + return; + } + }); + + // --- Chat completion via Entra ID --- + it("chat completion via Entra ID auth", { timeout: 30_000 }, async () => { + const agent = makeEntraIdAgent(); + try { + const result = await invoke(agent, { question: "Say hello in exactly 3 words." }); + expect(typeof result).toBe("string"); + expect((result as string).length).toBeGreaterThan(0); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + if (msg.includes("authentication") || msg.includes("credential") || msg.includes("CredentialUnavailableError")) { + console.warn( + `Skipping: Entra ID credentials not available — run \`az login\` first. (${msg})`, + ); + return; + } + if (msg.includes("401") || msg.includes("403")) { + console.warn( + `Skipping: Entra ID authorization failed — ensure 'Cognitive Services OpenAI User' role. (${msg})`, + ); + return; + } + throw e; + } + }); + + // --- Streaming chat via Entra ID --- + it("streaming chat via Entra ID auth", { timeout: 30_000 }, async () => { + const agent = makeEntraIdAgent({ + options: { temperature: 0, maxOutputTokens: 200, additionalProperties: { stream: true } }, + }); + try { + const result = await invoke(agent, { question: "Say hello in exactly 3 words." }); + const chunks: string[] = []; + for await (const chunk of result as AsyncIterable) { + if (typeof chunk === "string" && chunk.length > 0) chunks.push(chunk); + } + expect(chunks.length).toBeGreaterThan(0); + expect(chunks.join("").length).toBeGreaterThan(0); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + if (msg.includes("authentication") || msg.includes("credential") || msg.includes("CredentialUnavailableError")) { + console.warn( + `Skipping: Entra ID credentials not available — run \`az login\` first. (${msg})`, + ); + return; + } + if (msg.includes("401") || msg.includes("403")) { + console.warn( + `Skipping: Entra ID authorization failed — ensure 'Cognitive Services OpenAI User' role. (${msg})`, + ); + return; + } + throw e; + } + }); +}); diff --git a/spec/spec.md b/spec/spec.md index dbf36cb5..f646ca7e 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -2179,6 +2179,7 @@ the entire loop including all inner `execute` and `execute_tool` spans. | Constant | Default | Notes | | ------------------ | ------- | ------------------------------ | | `MAX_ITERATIONS` | `10` | MAY be configurable at runtime | +| `MAX_LLM_RETRIES` | `3` | MAY be configurable at runtime (§9.10) | ### §9.2 Algorithm @@ -2209,8 +2210,21 @@ function turn(agent_or_path, inputs, tools=null) → result: "Agent loop exceeded " + MAX_ITERATIONS + " iterations" ) - // 5b. Call the LLM - response = execute_llm(agent, messages) // raw API call + // 5b. Call the LLM (with retry — see §9.10) + llm_attempts = 0 + loop: + try: + response = execute_llm(agent, messages) // raw API call + break + catch error: + llm_attempts += 1 + if llm_attempts >= MAX_LLM_RETRIES: + raise ExecuteError( + message: str(error), + messages: messages // MUST include conversation state + ) + backoff = min(2^llm_attempts + jitter(), 60) + sleep(backoff) // 5c. Process response result = process(agent, response) @@ -2230,24 +2244,30 @@ function turn(agent_or_path, inputs, tools=null) → result: // Layer 1: explicit name override handler = get_tool(tool_call.name) - // Parse arguments - args = json_parse(tool_call.arguments) + // Parse arguments (with resilient fallback — see §9.8) + args = resilient_json_parse(tool_call.arguments) // Apply bindings (inject bound values from inputs) args = apply_bindings(tool_def, args, inputs) - if handler is not null: - // Name registry hit — direct call - tool_result = handler(args) - else: - // Layer 2: kind handler fallback - kind_handler = get_tool_handler(tool_def.kind) - if kind_handler is null: - raise ValueError( - "No handler registered for tool: " + tool_call.name - + " (kind: " + tool_def.kind + ")" - ) - tool_result = kind_handler(tool_def, args, agent, inputs) + // Execute tool handler (with error safety — see §9.9) + try: + if handler is not null: + // Name registry hit — direct call + tool_result = handler(args) + else: + // Layer 2: kind handler fallback + kind_handler = get_tool_handler(tool_def.kind) + if kind_handler is null: + raise ValueError( + "No handler registered for tool: " + tool_call.name + + " (kind: " + tool_def.kind + ")" + ) + tool_result = kind_handler(tool_def, args, agent, inputs) + catch error: + // Tool handler failures MUST NOT kill the agent loop (§9.9) + tool_result = "Error: Tool '" + tool_call.name + "' failed: " + str(error) + emit event("error", { message: tool_result }) tool_results.append({ tool_call_id: tool_call.id, result: str(tool_result) }) @@ -2462,6 +2482,133 @@ function execute_prompty_tool(tool, args, parent_inputs) → result: Child `PromptyTool` execution MUST inherit the parent's tracer registry, producing nested trace spans that show the full call hierarchy. +### §9.8 Resilient Argument Parsing + +LLMs frequently produce malformed JSON in tool call arguments — markdown +code fences wrapping JSON, trailing commas, or JSON embedded in prose text. +Implementations SHOULD attempt recovery using the following fallback chain +when `json_parse` fails on the raw argument string: + +``` +function resilient_json_parse(raw_arguments) → dict: + // Strategy 1: Direct parse + try: + return json_parse(raw_arguments) + catch: pass + + // Strategy 2: Strip markdown code fences + stripped = regex_replace(raw_arguments, + /^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$/s, "$1") + if stripped != raw_arguments: + try: + return json_parse(stripped) + catch: pass + + // Strategy 3: Extract first balanced JSON block + block = extract_first_json_block(raw_arguments) // find first { to matching } + if block is not null: + try: + return json_parse(block) + catch: pass + + // Strategy 4: Strip trailing commas before } or ] + cleaned = regex_replace(raw_arguments, /,\s*([}\]])/g, "$1") + try: + return json_parse(cleaned) + catch: pass + + // All strategies failed — return error as tool result + return null // caller MUST convert to error tool result +``` + +**Requirements:** + +- Implementations SHOULD attempt all four strategies in order. +- When a non-direct strategy succeeds, implementations SHOULD log a warning + indicating which fallback was used (e.g., "Parsed tool arguments after + stripping markdown fences"). +- If all strategies fail, implementations MUST NOT substitute a silent empty + object (`{}`). The parse failure MUST be reported as a string tool result + (e.g., `"Error: Invalid JSON in tool arguments: {error}"`) so the LLM + can see the error and retry. +- `extract_first_json_block` MUST respect string escapes (do not match + braces inside quoted strings). + +### §9.9 Tool Execution Error Safety + +Tool handlers are user-provided code. Implementations MUST catch exceptions +(or panics, in languages that distinguish them) raised by tool handlers +during execution. This applies to all tool dispatch paths: direct name +handlers (§9.2 Layer 1), kind handlers (§9.2 Layer 2), and PromptyTool +execution (§9.7). + +**Requirements:** + +- Caught errors MUST be converted to a string tool result: + `"Error: Tool '{name}' failed: {message}"` +- An `error` event (§13.1) MUST be emitted with the error details. +- The agent loop MUST NOT terminate due to a tool handler failure — the + error result is fed back to the LLM as the tool's response, allowing + the model to recover or try an alternative approach. +- For languages with both exceptions and panics (e.g., Rust), both MUST + be caught. For languages with only exceptions (e.g., Python, TypeScript), + catching exceptions is sufficient. +- `ValueError` for "Tool not registered" (§12.4) is NOT subject to this + rule — a missing handler indicates a configuration error, not a runtime + failure, and SHOULD still raise. + +### §9.10 LLM Call Retry + +Long-running agent loops accumulate valuable state across iterations. A +transient LLM failure at iteration N should not discard the work from +iterations 1 through N-1. Implementations SHOULD retry the `execute_llm` +call within the agent loop before raising to the caller. + +**Constants:** + +| Constant | Default | Notes | +| ------------------ | ------- | ------------------------------ | +| `MAX_LLM_RETRIES` | `3` | MAY be configurable at runtime | + +**Algorithm:** + +``` +// Inside the agent loop, replacing the direct execute_llm call: +llm_attempts = 0 +loop: + try: + response = execute_llm(agent, messages) + break // success + catch error: + llm_attempts += 1 + if llm_attempts >= MAX_LLM_RETRIES: + raise ExecuteError( + message: str(error), + messages: messages // MUST include conversation state + ) + backoff = min(2^llm_attempts + jitter(), 60) // exponential backoff, capped at 60s + sleep(backoff) +``` + +**Requirements:** + +- This retry is independent of any HTTP-level retry inside the executor. + The loop retries calling the executor as a black box — this also gives + the runtime an opportunity to refresh credentials, failover connections, + or recover from transient executor state between attempts. +- Retry SHOULD apply to all executor failures, not just specific HTTP + status codes. The loop does not inspect the error type — it simply + retries the call. +- When all retries are exhausted, the raised error MUST include the + accumulated `messages` list so the caller can resume by passing them + back as thread input on a subsequent `turn()` call. +- Implementations SHOULD emit a `status` event (§13.1) before each retry + (e.g., `"LLM call failed, retrying (attempt 2/3)..."`). +- Retry MUST respect the cancellation token (§13.2) — if cancellation is + signaled during a backoff wait, the loop MUST stop retrying immediately. +- During non-agent `invoke()` calls (single LLM call, no tool loop), + implementations MAY apply the same retry logic but are not required to. + --- ## §10 Streaming @@ -2925,8 +3072,21 @@ specific exception class SHOULD follow language idioms. | Agent Loop | Tool not registered | ValueError | `"Tool not registered: {name}"` | | Agent Loop | Max iterations exceeded | RuntimeError | `"Agent loop exceeded {n} iterations"` | | Agent Loop | Model refusal | ValueError | `"Model refused: {message}"` | +| Agent Loop | LLM call failed (retries exhausted) | ExecuteError | `"LLM call failed after {n} retries: {message}"` — MUST include `messages` | +| Agent Loop | Tool handler exception/panic | *(not raised)* | Converted to tool result string (§9.9) | | Discovery | No implementation for key | InvokerError | `"No {component} registered for key: {key}"` | +### §12.5 Executor Retry + +Executors SHOULD handle transient HTTP errors (status codes 429 and 5xx) +internally with retry and exponential backoff. This is independent of the +agent loop's LLM call retry (§9.10) — the executor retries the HTTP +request, while the loop retries calling the executor. + +When retrying, executors SHOULD respect `Retry-After` headers when present. +The maximum number of internal retries is an implementation choice. +Executors MUST NOT retry on 4xx errors other than 429 by default. + ### §12.6 Rich Kinds `RICH_KINDS = { thread, image, file, audio }` @@ -3371,6 +3531,7 @@ function turn( tools = null, // tool handlers *, // keyword-only below max_iterations = 10, // iteration cap + max_llm_retries = 3, // retry execute_llm on failure (§9.10) on_event = null, // event callback cancel = null, // cancellation token context_budget = null, // character budget for context window @@ -3389,15 +3550,17 @@ function turn( 2. Drain steering messages 3. Trim context window (if budget set) 4. Check input guardrail - 5. Call LLM (§9.2 step 5b) + 5. Call LLM with retry (§9.10 — up to max_llm_retries attempts) 6. Process response (§9.2 step 5c) 7. Check output guardrail 8. If tool calls: - a. Check tool guardrails (per tool) - b. Execute tools (parallel or sequential), applying bindings (§9.6) - c. Format tool messages via executor.FormatToolMessages (§9.4) - d. Append to messages, emit messages_updated - e. Continue loop + a. Parse arguments with resilient fallback (§9.8) + b. Check tool guardrails (per tool) + c. Execute tools with error safety (§9.9), parallel or sequential + d. Apply bindings (§9.6) + e. Format tool messages via executor.FormatToolMessages (§9.4) + f. Append to messages, emit messages_updated + g. Continue loop 9. Emit done, return result ``` diff --git a/web/src/content/docs/core-concepts/agent-mode.mdx b/web/src/content/docs/core-concepts/agent-mode.mdx index 081438be..cf24e080 100644 --- a/web/src/content/docs/core-concepts/agent-mode.mdx +++ b/web/src/content/docs/core-concepts/agent-mode.mdx @@ -85,6 +85,7 @@ result = turn( inputs={"question": "What's the weather in Seattle?"}, tools=tools, max_iterations=10, + max_llm_retries=3, ) print(result) # "It's currently 72°F and sunny in Seattle!" @@ -123,7 +124,7 @@ const tools = bindTools(agent, [getWeather, getTime]); const result = await turn( agent, { question: "What's the weather in Seattle?" }, - { tools, maxIterations: 10 }, + { tools, maxIterations: 10, maxLlmRetries: 3 }, ); console.log(result); // "It's currently 72°F and sunny in Seattle!" @@ -161,7 +162,8 @@ var result = await Pipeline.TurnAsync( agent, new() { ["question"] = "What's the weather in Seattle?" }, tools: tools, - maxIterations: 10 + maxIterations: 10, + maxLlmRetries: 3 ); Console.WriteLine(result); // "It's currently 72°F and sunny in Seattle!" @@ -193,6 +195,7 @@ let agent = prompty::load("agent.prompty")?; // 3. Run the agent loop let options = TurnOptions { max_iterations: Some(10), + max_llm_retries: Some(3), ..Default::default() }; @@ -309,32 +312,133 @@ asyncio.run(main()) coroutines. -## Error Recovery +## Error Recovery & Resilience -The agent loop is designed to be resilient. Instead of crashing on tool -execution errors, it feeds error information back to the LLM so the model can -retry or adjust its approach. +The agent loop is designed to be resilient at three levels: malformed tool +arguments, tool execution failures, and transient LLM errors. Instead of +crashing, the loop recovers and feeds error information back to the LLM so +the model can retry or adjust its approach. -### Bad JSON in Tool Arguments +### §9.8 — Resilient Argument Parsing -If the LLM returns malformed JSON in a tool call's `arguments` field, Prompty -catches the `json.JSONDecodeError` and sends the error string back as the tool -result. The model typically corrects the JSON on the next attempt. +LLMs sometimes return malformed JSON in tool call arguments — markdown code +fences wrapping JSON, trailing commas, or JSON embedded in prose. Prompty +uses a four-strategy fallback chain before giving up: + +1. **Direct parse** — try `JSON.parse` as-is +2. **Strip markdown fences** — remove `` ```json ... ``` `` wrappers +3. **Extract first JSON block** — find the first `{` to its matching `}` +4. **Strip trailing commas** — remove `,` before `}` or `]` + +If all four strategies fail, the parse error is sent back to the LLM as a +tool result string (never a silent empty `{}`). The model typically corrects +the JSON on the next attempt. ``` -tool message → "Error parsing arguments: Expecting ',' delimiter: line 1 column 42" +tool message → "Error: Invalid JSON in tool arguments: Expecting ',' delimiter: line 1 column 42" ``` -### Tool Function Throws an Exception +### §9.9 — Tool Execution Error Safety -If your tool function raises any exception, Prompty catches it and sends the -error message back to the LLM as the tool result. This lets the model decide -whether to retry with different arguments or inform the user. +If your tool function raises any exception (or panics in Rust), Prompty +catches it and sends the error message back to the LLM as the tool result. +The agent loop **never** terminates due to a tool handler failure — the +model decides whether to retry with different arguments or inform the user. ``` -tool message → "Error executing get_weather: ConnectionTimeout: API unreachable" +tool message → "Error: Tool 'get_weather' failed: ConnectionTimeout: API unreachable" ``` +### §9.10 — LLM Call Retry + +Transient LLM failures (429 rate limits, 500 server errors) can derail a +long and expensive agent loop. Prompty retries the LLM call with exponential +backoff before giving up — preserving the conversation state accumulated +across iterations. + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `max_llm_retries` | `3` | Maximum retry attempts per LLM call | + +The backoff formula is `min(2^attempt + jitter, 60s)` — exponential with +random jitter, capped at 60 seconds. + +When all retries are exhausted, Prompty raises an `ExecuteError` that +**includes the full conversation history**. This lets you resume a failed +agent loop without losing work: + + + +```python +from prompty import turn, ExecuteError + +try: + result = turn( + "agent.prompty", + inputs={"question": "Plan my trip"}, + tools=tools, + max_llm_retries=3, + ) +except ExecuteError as e: + print(f"Failed after retries: {e}") + # e.messages contains the full conversation — resume later + saved_messages = e.messages +``` + + +```typescript +import { turn, ExecuteError } from "@prompty/core"; + +try { + const result = await turn(agent, inputs, { + tools, + maxLlmRetries: 3, + }); +} catch (e) { + if (e instanceof ExecuteError) { + console.log(`Failed after retries: ${e.message}`); + // e.messages contains the full conversation — resume later + const savedMessages = e.messages; + } +} +``` + + +```csharp +try +{ + var result = await Pipeline.TurnAsync( + agent, inputs, tools: tools, maxLlmRetries: 3); +} +catch (ExecuteError e) +{ + Console.WriteLine($"Failed after retries: {e.Message}"); + // e.Messages contains the full conversation — resume later + var savedMessages = e.Messages; +} +``` + + +```rust +use prompty::{TurnOptions, InvokerError}; + +let options = TurnOptions { + max_llm_retries: Some(3), + ..Default::default() +}; + +match prompty::turn(&agent, Some(&inputs), Some(options)).await { + Ok(result) => println!("{result}"), + Err(InvokerError::ExecuteRetryExhausted { message, messages }) => { + eprintln!("Failed after retries: {message}"); + // messages contains the full conversation — resume later + } + Err(e) => eprintln!("Other error: {e}"), +} +``` + + + ### Missing Tool Name If the LLM requests a tool that doesn't exist in the `tools` dict, an error @@ -383,7 +487,7 @@ const tools = bindTools(agent, [getWeather]); const result = await turn(agent, { question: "What's the weather in London?", -}, { tools, maxIterations: 10 }); +}, { tools, maxIterations: 10, maxLlmRetries: 3 }); console.log(result); ``` @@ -406,7 +510,8 @@ var result = await Pipeline.TurnAsync( agent, new() { ["question"] = "What's the weather in London?" }, tools: tools, - maxIterations: 10 + maxIterations: 10, + maxLlmRetries: 3 ); Console.WriteLine(result); @@ -428,6 +533,7 @@ let agent = prompty::load("agent.prompty")?; let options = TurnOptions { max_iterations: Some(10), + max_llm_retries: Some(3), ..Default::default() }; @@ -452,8 +558,9 @@ Under the hood, the agent loop in the executor follows these steps: off, it reads tool calls directly from the response. Either way, tool calls are fully collected before any are executed. -2. **Call the LLM** — send the current message list plus tool definitions via - the chat completions API. +2. **Call the LLM (with retry)** — send the current message list plus tool + definitions via the chat completions API. If the call fails, retry with + exponential backoff up to `max_llm_retries` times (§9.10). 3. **Check `finish_reason`** — if the response's `finish_reason` is `"tool_calls"`, the model wants to invoke tools. If it's `"stop"`, the model @@ -462,35 +569,53 @@ Under the hood, the agent loop in the executor follows these steps: 4. **Extract tool calls** — each tool call has an `id`, a `function.name`, and `function.arguments` (a JSON string). -5. **Look up & execute** — for each tool call, find the matching function in the - `tools` dict (or `agent.metadata["tool_functions"]`), parse the arguments, - and call the function. +5. **Parse arguments (resilient)** — parse the JSON arguments using the + four-strategy fallback chain (§9.8). If all strategies fail, send the + error back to the LLM as a tool result. + +6. **Execute (with error safety)** — for each tool call, find the matching + function and call it. If the function throws, catch the error and send it + back to the LLM as a tool result (§9.9) — the loop continues. -6. **Append results** — add the assistant's tool-call message and one `tool` +7. **Append results** — add the assistant's tool-call message and one `tool` role message per call result back to the conversation. -7. **Repeat** — go back to step 2 with the updated message list. +8. **Repeat** — go back to step 2 with the updated message list. -8. **Return** — when the model produces a final response (no tool calls), pass +9. **Return** — when the model produces a final response (no tool calls), pass it through the processor and return the result. ```python -# Simplified pseudocode of the agent loop +# Simplified pseudocode of the agent loop (with resilience) +from prompty import ExecuteError +from prompty.core.tool_dispatch import resilient_json_parse + messages = prepare(agent, inputs) for i in range(max_iterations): - response = client.chat.completions.create( - model=agent.model.id, - messages=messages, - tools=tool_definitions, - ) + # LLM call with retry (§9.10) + for attempt in range(max_llm_retries): + try: + response = client.chat.completions.create( + model=agent.model.id, messages=messages, tools=tool_defs) + break + except Exception as e: + if attempt + 1 >= max_llm_retries: + raise ExecuteError(str(e), messages=messages) + time.sleep(min(2 ** (attempt + 1) + random(), 60)) + if response.finish_reason != "tool_calls": return process(response) - # Execute each tool call messages.append(response.message) for tool_call in response.tool_calls: - result = tools[tool_call.function.name](**tool_call.args) - messages.append({"role": "tool", "tool_call_id": tool_call.id, "content": result}) + # Resilient parsing (§9.8) + args = resilient_json_parse(tool_call.function.arguments) + try: + # Error safety (§9.9) — catch tool failures + result = tools[tool_call.function.name](**args) + except Exception as e: + result = f"Error: Tool '{tool_call.function.name}' failed: {e}" + messages.append({"role": "tool", "tool_call_id": tool_call.id, "content": str(result)}) raise ValueError(f"Agent loop exceeded max_iterations ({max_iterations})") ``` diff --git a/web/src/content/docs/implementation/csharp.mdx b/web/src/content/docs/implementation/csharp.mdx index 20b23648..8577218b 100644 --- a/web/src/content/docs/implementation/csharp.mdx +++ b/web/src/content/docs/implementation/csharp.mdx @@ -271,11 +271,16 @@ var tools = new Dictionary>> }; var result = await Pipeline.TurnAsync( - agent, inputs, tools: tools, maxIterations: 10); + agent, inputs, tools: tools, maxIterations: 10, maxLlmRetries: 3); Console.WriteLine(result); ``` +The agent loop includes built-in resilience: +- **Resilient JSON parsing** — `ParseArguments()` recovers from malformed tool arguments (markdown fences, trailing commas) +- **Tool error safety** — tool exceptions are caught and fed back to the LLM +- **LLM call retry** — transient failures are retried with exponential backoff; `ExecuteError` carries the full conversation for resumption +