fix: 6 infrastructure fixes (LLM retry, cache, token counting, tests)

libs/context/include/merak/token_counter.hpp

@@ -28,9 +28,19 @@ class TokenCounter {
     int fit_in_budget(const std::vector<Message>& messages,
         int token_limit) const;
 
+    // Update authoritative token count from API response.
+    // Subsequent count() calls use this as baseline, only estimating
+    // messages beyond the authoritative count.
+    void update_authoritative(int prompt_tokens, int message_count) {
+        authoritative_total_ = prompt_tokens;
+        authoritative_message_count_ = message_count;
+    }
+
 private:
     std::string model_;
     double chars_per_token_;
+    int authoritative_total_ = 0;
+    int authoritative_message_count_ = 0;
 };
 
 } // namespace merak

libs/context/src/token_counter.cpp

@@ -22,6 +22,15 @@ int TokenCounter::count(const Message& msg) const {
 }
 
 int TokenCounter::count(const std::vector<Message>& messages) const {
+    // Hybrid: authoritative baseline from API + heuristic for new messages
+    if (authoritative_total_ > 0 && (int)messages.size() >= authoritative_message_count_) {
+        int incremental = 0;
+        for (int i = authoritative_message_count_; i < (int)messages.size(); i++) {
+            incremental += count(messages[i]);
+        }
+        return authoritative_total_ + incremental;
+    }
+    // Cold start: pure heuristic
     int total = 0;
     for (auto& msg : messages) {
         total += count(msg);

libs/llm/include/merak/anthropic_provider.hpp

@@ -23,11 +23,11 @@ class AnthropicProvider : public LlmProvider {
     bool test_connection() override;
     const CacheStats& cache_stats() const { return stats_; }
 
+    nlohmann::json build_request_body(const ChatRequest& request) const;
+
 private:
     LLMConfig config_;
     CacheStats stats_;
-
-    nlohmann::json build_request_body(const ChatRequest& request) const;
 };
 
 } // namespace merak

libs/llm/include/merak/llm_provider.hpp

@@ -28,6 +28,11 @@ struct ChatRequest {
     bool enable_thinking = true;
 };
 
+struct RetryConfig {
+    int max_retries = 3;
+    int base_delay_ms = 1000;
+};
+
 class LlmProvider {
 public:
     virtual ~LlmProvider() = default;

libs/llm/include/merak/openai_provider.hpp

@@ -23,12 +23,12 @@ class OpenAIProvider : public LlmProvider {
     bool test_connection() override;
     const CacheStats& cache_stats() const { return stats_; }
 
+    nlohmann::json build_messages(const std::vector<Message>& msgs) const;
+    nlohmann::json build_tools(const std::vector<ToolSpec>& tools) const;
+
 private:
     LLMConfig config_;
     CacheStats stats_;
-
-    nlohmann::json build_messages(const std::vector<Message>& msgs) const;
-    nlohmann::json build_tools(const std::vector<ToolSpec>& tools) const;
 };
 
 } // namespace merak

libs/llm/src/anthropic_provider.cpp

@@ -2,6 +2,8 @@
 #include <nlohmann/json.hpp>
 #include <spdlog/spdlog.h>
 #include <future>
+#include <chrono>
+#include <thread>
 
 namespace merak {
 
@@ -130,28 +132,6 @@ std::future<AgentResponse> AnthropicProvider::chat(
         std::string body_str = body.dump();
         spdlog::debug("Anthropic request: url={}, body_size={}", url, body_str.size());
 
-        CURL* curl = curl_easy_init();
-        struct curl_slist* headers = nullptr;
-        headers = curl_slist_append(headers,
-            ("x-api-key: " + config_.api_key).c_str());
-        headers = curl_slist_append(headers,
-            "anthropic-version: 2023-06-01");
-        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-        curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, body_str.c_str());
-        headers = curl_slist_append(headers, "Content-Type: application/json");
-        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
-        curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 10000L);
-        curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
-        curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L);
-        curl_easy_setopt(curl, CURLOPT_TIMEOUT, 300L);
-        curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
-        curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION,
-            +[](void* userdata, curl_off_t, curl_off_t, curl_off_t, curl_off_t) -> int {
-                auto* token = static_cast<CancellationToken*>(userdata);
-                return token && token->cancelled() ? 1 : 0;
-            });
-        curl_easy_setopt(curl, CURLOPT_XFERINFODATA, cancellation.get());
-
         // SSE 累积状态
         std::string response_text;
         int input_tokens = 0, output_tokens = 0;
@@ -279,49 +259,106 @@ std::future<AgentResponse> AnthropicProvider::chat(
             }
         };
 
-        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
-            +[](char* ptr, size_t size, size_t nmemb, void* userdata) -> size_t {
-                auto* cb = static_cast<decltype(&write_callback)>(userdata);
-                std::string data(ptr, size * nmemb);
-                (*cb)(data);
-                return size * nmemb;
-            });
+        RetryConfig retry;
+        int delay = retry.base_delay_ms;
+        CURLcode res = CURLE_OK;
+        long http_code = 0;
 
-        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &write_callback);
+        for (int attempt = 0; attempt <= retry.max_retries; attempt++) {
+            response_text.clear();
+            input_tokens = 0;
+            output_tokens = 0;
+            has_usage = false;
+            pending_tools.clear();
+            preserved_content_blocks.clear();
+            accumulated_tool_calls.clear();
+            current_event.clear();
+            current_data.clear();
+            line_buffer.clear();
 
-        CURLcode res = curl_easy_perform(curl);
+            CURL* curl = curl_easy_init();
+            if (!curl) {
+                res = CURLE_OUT_OF_MEMORY;
+                http_code = 0;
+                if (attempt == retry.max_retries) {
+                    throw AgentError(ErrorType::LLM_ERROR, "Failed to initialize curl handle");
+                }
+                spdlog::warn("Provider: retry {}/{} after {}ms (curl_easy_init failed)",
+                    attempt + 1, retry.max_retries, delay);
+                std::this_thread::sleep_for(std::chrono::milliseconds(delay));
+                delay *= 2;
+                continue;
+            }
+            struct curl_slist* hdrs = nullptr;
+            hdrs = curl_slist_append(hdrs,
+                ("x-api-key: " + config_.api_key).c_str());
+            hdrs = curl_slist_append(hdrs,
+                "anthropic-version: 2023-06-01");
+            curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+            curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, body_str.c_str());
+            hdrs = curl_slist_append(hdrs, "Content-Type: application/json");
+            curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hdrs);
+            curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 10000L);
+            curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
+            curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L);
+            curl_easy_setopt(curl, CURLOPT_TIMEOUT, 300L);
+            curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
+            curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION,
+                +[](void* userdata, curl_off_t, curl_off_t, curl_off_t, curl_off_t) -> int {
+                    auto* token = static_cast<CancellationToken*>(userdata);
+                    return token && token->cancelled() ? 1 : 0;
+                });
+            curl_easy_setopt(curl, CURLOPT_XFERINFODATA, cancellation.get());
 
-        long http_code = 0;
-        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+                +[](char* ptr, size_t size, size_t nmemb, void* userdata) -> size_t {
+                    auto* cb = static_cast<decltype(&write_callback)>(userdata);
+                    std::string data(ptr, size * nmemb);
+                    (*cb)(data);
+                    return size * nmemb;
+                });
+            curl_easy_setopt(curl, CURLOPT_WRITEDATA, &write_callback);
 
-        if (res != CURLE_OK) {
-            spdlog::error("curl error: {}", curl_easy_strerror(res));
+            res = curl_easy_perform(curl);
+            curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
             curl_easy_cleanup(curl);
-            curl_slist_free_all(headers);
-            throw AgentError(
-                cancellation && cancellation->cancelled()
-                    ? ErrorType::LLM_TIMEOUT : ErrorType::LLM_ERROR,
-                cancellation && cancellation->cancelled()
-                    ? "LLM request cancelled" : curl_easy_strerror(res));
-        }
+            curl_slist_free_all(hdrs);
 
-        if (http_code >= 400) {
-            curl_easy_cleanup(curl);
-            curl_slist_free_all(headers);
+            // Success
+            if (res == CURLE_OK && http_code < 400) break;
+
+            // Never retry cancellation
+            if (cancellation && cancellation->cancelled()) {
+                throw AgentError(ErrorType::LLM_TIMEOUT, "LLM request cancelled");
+            }
+
+            // Never retry auth errors
             if (http_code == 401 || http_code == 403) {
                 throw AgentError(ErrorType::LLM_ERROR,
                     "LLM authentication failed (HTTP " + std::to_string(http_code) + ")");
             }
-            if (http_code == 429) {
+
+            // Not retryable: other 4xx
+            if (http_code >= 400 && http_code < 500 && http_code != 429) {
                 throw AgentError(ErrorType::LLM_ERROR,
-                    "Rate limited (HTTP 429)");
+                    "LLM API error (HTTP " + std::to_string(http_code) + ")");
+            }
+
+            // Exhausted retries
+            if (attempt == retry.max_retries) {
+                if (res != CURLE_OK) {
+                    throw AgentError(ErrorType::LLM_ERROR, curl_easy_strerror(res));
+                }
+                throw AgentError(ErrorType::LLM_ERROR,
+                    "LLM API error after retries (HTTP " + std::to_string(http_code) + ")");
             }
-            throw AgentError(ErrorType::LLM_ERROR,
-                "LLM API error (HTTP " + std::to_string(http_code) + ")");
-        }
 
-        curl_easy_cleanup(curl);
-        curl_slist_free_all(headers);
+            // Backoff and retry
+            spdlog::warn("Provider: retry {}/{} after {}ms (HTTP {}, curl {})",
+                attempt + 1, retry.max_retries, delay, http_code, (int)res);
+            std::this_thread::sleep_for(std::chrono::milliseconds(delay));
+            delay *= 2;
+        }
 
         AgentResponse response;
         response.tool_calls = std::move(accumulated_tool_calls);