feat[bc]: move prompt cache reconciliation into llm-llamacpp

packages/llm-llamacpp/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [0.54.0] - 2026-09-18
+
+### Breaking
+
+- Cached requests now use addon-owned full-prompt reconciliation. Every request
+  with `cacheKey` must resend the complete message history and complete tool
+  list. The addon persists a versioned token/media ledger in the sequence-state
+  file, reuses the longest matching prefix, and treats pre-ledger cache files as
+  cold misses. Generated reasoning is retained until the next authoritative
+  render omits it; `generationParams.remove_thinking_from_context` has therefore
+  been removed from the addon API, together with the obsolete
+  `RuntimeStats.thinkingBlockDiscards` counter. Current SDK releases still send
+  delta prompts/tools and still expose that option, so they are intentionally
+  incompatible with this addon until the SDK migration lands.
+
 ## [0.53.1] - 2026-09-16
 
 ### Fixed

packages/llm-llamacpp/CMakeLists.txt

@@ -92,14 +92,13 @@ endif()
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/MtmdLlmContext.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/MultiRequestBatcher.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/TextLlmContext.cpp
-    ${PROJECT_SOURCE_DIR}/addon/src/model-interface/ReasoningBlockCompactor.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/ModelMetadata.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/LoggingMacros.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/BackendSelection.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/ChatTemplateUtils.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/ReasoningUtils.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/RecurrentStateSnapshot.cpp
-    ${PROJECT_SOURCE_DIR}/addon/src/utils/ReasoningRollbackState.cpp
+    ${PROJECT_SOURCE_DIR}/addon/src/utils/RequestRollbackState.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/QwenTemplate.cpp
   )
 
@@ -150,14 +149,13 @@ if(BUILD_CLI)
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/MtmdLlmContext.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/MultiRequestBatcher.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/TextLlmContext.cpp
-    ${PROJECT_SOURCE_DIR}/addon/src/model-interface/ReasoningBlockCompactor.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/model-interface/ModelMetadata.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/LoggingMacros.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/BackendSelection.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/ChatTemplateUtils.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/ReasoningUtils.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/RecurrentStateSnapshot.cpp
-    ${PROJECT_SOURCE_DIR}/addon/src/utils/ReasoningRollbackState.cpp
+    ${PROJECT_SOURCE_DIR}/addon/src/utils/RequestRollbackState.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/QwenTemplate.cpp
   )
 
@@ -205,4 +203,4 @@ if(BUILD_TESTING)
   # Integration tests for model classes (includes backend selection tests)
   # Pass ENABLE_COVERAGE option to test subdirectory
   add_subdirectory(test/unit)
-endif()
+endif()

packages/llm-llamacpp/README.md

@@ -191,7 +191,7 @@ The addon picks a safe KV-cache type when `cache-type-k`/`cache-type-v` are unse
 
 - **Auto-default:** on a **Metal / Vulkan GPU** (with flash attention on) both K and V default to **`q8_0`** — quality-neutral vs `f16` and ~47% smaller KV cache. **CPU** and **OpenCL (Adreno)** keep **`f16`** (ARM CPU `q8_0` has a quality/throughput cost; quantized KV is unsafe on OpenCL — see below). Finetuning manages its own KV types and is left untouched.
 - **`flash-attn: 'auto'` keeps `f16`.** "Flash attention on" above means a truthy `flash-attn` — `on`, `enabled`, `true` or `1`, or the `on` default when the key is unset. `'auto'` is deliberately excluded: quantizing the V cache forces qvac-fabric to promote AUTO to ENABLED, which skips the runtime capability probe that `'auto'` exists to run. So `'auto'` trades the ~47% KV-cache saving for letting qvac-fabric decide. To get both, set `cache-type-k`/`-v` explicitly alongside `'auto'`, or use `'on'`. **`split-mode: 'tensor'` is the exception:** qvac-fabric promotes AUTO to ENABLED unconditionally for that mode, so there is no probe to preserve and `'auto'` takes the q8_0 default there exactly as `'on'` does.
-- **OpenCL (Adreno) accepts only `f16`/`f32`/`bf16`:** any other cache type — quantized (`q8_0`, `q4_0`, `q4_1`, `q5_0`, …) or unrecognized — throws a `StatusError`. A quantized K or V cache aborts in `llama_kv_cache::update` on cache management (reasoning-block compaction, state restore) because ggml-opencl has no `F32→quantized` requantize kernel. Use `f16`/`f32`/`bf16`, or a Vulkan GPU / CPU.
+- **OpenCL (Adreno) accepts only `f16`/`f32`/`bf16`:** any other cache type — quantized (`q8_0`, `q4_0`, `q4_1`, `q5_0`, …) or unrecognized — throws a `StatusError`. A quantized K or V cache aborts in `llama_kv_cache::update` during state restore because ggml-opencl has no `F32→quantized` requantize kernel. Use `f16`/`f32`/`bf16`, or a Vulkan GPU / CPU.
 - **Mixed K≠V is a warning, not an error:** if K and V differ and at least one is quantized, the addon logs a warning (asymmetric quantized K/V falls off the fused flash-attention path — a notable GPU decode penalty — for no quality benefit, and is unsupported on Adreno OpenCL) but proceeds. Prefer a symmetric type. (This may be relaxed once qvac-fabric handles asymmetric quantized K/V efficiently.)
 
 

packages/llm-llamacpp/addon/src/handlers/GenerationParamHandlers.cpp

@@ -80,14 +80,6 @@ const GenerationParamHandlerList GENERATION_PARAM_HANDLERS = {
          p.reasoning_budget = parsers::validateReasoningBudgetOverride(*value);
        }
      }},
-    {"remove_thinking_from_context",
-     [](js_env_t* env, js::Object& obj, GenerationParams& p) {
-       auto value = obj.getOptionalPropertyAs<js::Boolean, bool>(
-           env, "remove_thinking_from_context");
-       if (value.has_value()) {
-         p.remove_thinking_from_context = *value;
-       }
-     }},
 };
 
 void applyGenerationParamHandlers(

packages/llm-llamacpp/addon/src/model-interface/CacheLedger.hpp

@@ -0,0 +1,218 @@
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <llama.h>
+
+namespace qvac_lib_inference_addon_llama::cache {
+
+// The token array embedded by llama_state_seq_save_file is also our cache
+// manifest.  Keep the marker positive: llama_token is signed on some builds.
+inline constexpr llama_token LEDGER_MAGIC = 0x514c4447; // "QLDG"
+inline constexpr llama_token LEDGER_VERSION = 1;
+inline constexpr size_t LEDGER_HEADER_WORDS = 8;
+inline constexpr size_t LEDGER_ENTRY_WORDS = 5;
+inline constexpr size_t MAX_PROCESS_CHECKPOINTS = 32;
+
+enum class EntryKind : int32_t { Token = 1, Media = 2 };
+
+struct Entry {
+  EntryKind kind = EntryKind::Token;
+  int64_t identity = 0;
+  llama_pos positions = 1;
+  llama_pos cacheTokens = 1;
+
+  friend bool operator==(const Entry& a, const Entry& b) {
+    return a.kind == b.kind && a.identity == b.identity &&
+           a.positions == b.positions && a.cacheTokens == b.cacheTokens;
+  }
+};
+
+struct Ledger {
+  std::vector<Entry> entries;
+
+  [[nodiscard]] llama_pos positions(size_t end) const {
+    llama_pos value = 0;
+    end = std::min(end, entries.size());
+    for (size_t i = 0; i < end; ++i) {
+      value += entries[i].positions;
+    }
+    return value;
+  }
+
+  [[nodiscard]] llama_pos cacheTokens(size_t end) const {
+    llama_pos value = 0;
+    end = std::min(end, entries.size());
+    for (size_t i = 0; i < end; ++i) {
+      value += entries[i].cacheTokens;
+    }
+    return value;
+  }
+
+  [[nodiscard]] llama_pos positions() const {
+    return positions(entries.size());
+  }
+  [[nodiscard]] llama_pos cacheTokens() const {
+    return cacheTokens(entries.size());
+  }
+
+  void truncate(size_t count) {
+    entries.resize(std::min(count, entries.size()));
+  }
+
+  void appendToken(llama_token token) {
+    entries.push_back(
+        {.kind = EntryKind::Token,
+         .identity = static_cast<int64_t>(token),
+         .positions = 1,
+         .cacheTokens = 1});
+  }
+};
+
+inline Ledger fromTokens(const std::vector<llama_token>& tokens) {
+  Ledger result;
+  result.entries.reserve(tokens.size());
+  for (llama_token token : tokens) {
+    result.appendToken(token);
+  }
+  return result;
+}
+
+inline size_t commonPrefix(const Ledger& a, const Ledger& b) {
+  const size_t limit = std::min(a.entries.size(), b.entries.size());
+  size_t i = 0;
+  while (i < limit && a.entries[i] == b.entries[i]) {
+    ++i;
+  }
+  return i;
+}
+
+template <typename T>
+void appendProcessCheckpoint(std::deque<T>& checkpoints, T checkpoint) {
+  checkpoints.push_back(std::move(checkpoint));
+  while (checkpoints.size() > MAX_PROCESS_CHECKPOINTS) {
+    checkpoints.pop_front();
+  }
+}
+
+inline uint64_t hashBytes(const void* data, size_t size) {
+  // Stable FNV-1a identity. This is not a security boundary; it prevents a
+  // media span from being reused for different content.
+  constexpr uint64_t basis = 1469598103934665603ULL;
+  constexpr uint64_t prime = 1099511628211ULL;
+  uint64_t hash = basis;
+  const auto* bytes = static_cast<const uint8_t*>(data);
+  for (size_t i = 0; i < size; ++i) {
+    hash ^= bytes[i];
+    hash *= prime;
+  }
+  return hash;
+}
+
+inline uint64_t checksum(const std::vector<llama_token>& words, size_t begin) {
+  return hashBytes(
+      words.data() + begin, (words.size() - begin) * sizeof(llama_token));
+}
+
+inline std::vector<llama_token>
+serialize(const Ledger& ledger, llama_pos nPast, llama_pos cacheTokens) {
+  std::vector<llama_token> out(
+      LEDGER_HEADER_WORDS + ledger.entries.size() * LEDGER_ENTRY_WORDS);
+  out[0] = LEDGER_MAGIC;
+  out[1] = LEDGER_VERSION;
+  out[2] = static_cast<llama_token>(nPast);
+  out[3] = static_cast<llama_token>(cacheTokens);
+  out[4] = static_cast<llama_token>(ledger.entries.size());
+  out[5] = 0;
+  out[6] = 0;
+  out[7] = 0;
+  size_t cursor = LEDGER_HEADER_WORDS;
+  for (const Entry& entry : ledger.entries) {
+    const uint64_t id = static_cast<uint64_t>(entry.identity);
+    out[cursor++] = static_cast<llama_token>(entry.kind);
+    out[cursor++] = static_cast<llama_token>(id & 0xffffffffULL);
+    out[cursor++] = static_cast<llama_token>(id >> 32U);
+    out[cursor++] = static_cast<llama_token>(entry.positions);
+    out[cursor++] = static_cast<llama_token>(entry.cacheTokens);
+  }
+  const uint64_t sum = checksum(out, LEDGER_HEADER_WORDS);
+  out[5] = static_cast<llama_token>(sum & 0xffffffffULL);
+  out[6] = static_cast<llama_token>(sum >> 32U);
+  return out;
+}
+
+struct DecodedLedger {
+  Ledger ledger;
+  llama_pos nPast = 0;
+  llama_pos cacheTokens = 0;
+};
+
+inline bool hasMarker(const llama_token* words, size_t count) {
+  return count > 0 && words != nullptr && words[0] == LEDGER_MAGIC;
+}
+
+inline DecodedLedger deserialize(const llama_token* words, size_t count) {
+  if (!hasMarker(words, count)) {
+    throw std::runtime_error("cache ledger marker is missing");
+  }
+  if (count < LEDGER_HEADER_WORDS) {
+    throw std::runtime_error("cache ledger header is truncated");
+  }
+  if (words[1] != LEDGER_VERSION) {
+    throw std::runtime_error("unsupported cache ledger version");
+  }
+  if (words[2] < 0 || words[3] < 0 || words[4] < 0) {
+    throw std::runtime_error("cache ledger contains a negative size");
+  }
+  const size_t entryCount = static_cast<size_t>(words[4]);
+  if (entryCount > (SIZE_MAX - LEDGER_HEADER_WORDS) / LEDGER_ENTRY_WORDS ||
+      count != LEDGER_HEADER_WORDS + entryCount * LEDGER_ENTRY_WORDS) {
+    throw std::runtime_error("cache ledger length does not match its header");
+  }
+  std::vector<llama_token> owned(words, words + count);
+  const uint64_t expected =
+      static_cast<uint32_t>(words[5]) |
+      (static_cast<uint64_t>(static_cast<uint32_t>(words[6])) << 32U);
+  if (checksum(owned, LEDGER_HEADER_WORDS) != expected) {
+    throw std::runtime_error("cache ledger checksum mismatch");
+  }
+
+  DecodedLedger result;
+  result.nPast = static_cast<llama_pos>(words[2]);
+  result.cacheTokens = static_cast<llama_pos>(words[3]);
+  result.ledger.entries.reserve(entryCount);
+  size_t cursor = LEDGER_HEADER_WORDS;
+  for (size_t i = 0; i < entryCount; ++i) {
+    const int32_t rawKind = words[cursor++];
+    if (rawKind != static_cast<int32_t>(EntryKind::Token) &&
+        rawKind != static_cast<int32_t>(EntryKind::Media)) {
+      throw std::runtime_error("cache ledger contains an unknown entry kind");
+    }
+    const uint64_t lo = static_cast<uint32_t>(words[cursor++]);
+    const uint64_t hi = static_cast<uint32_t>(words[cursor++]);
+    const llama_pos positions = static_cast<llama_pos>(words[cursor++]);
+    const llama_pos kv = static_cast<llama_pos>(words[cursor++]);
+    if (positions <= 0 || kv <= 0) {
+      throw std::runtime_error("cache ledger contains an invalid span");
+    }
+    result.ledger.entries.push_back(
+        {.kind = static_cast<EntryKind>(rawKind),
+         .identity = static_cast<int64_t>(lo | (hi << 32U)),
+         .positions = positions,
+         .cacheTokens = kv});
+  }
+  if (result.ledger.positions() != result.nPast ||
+      result.ledger.cacheTokens() != result.cacheTokens) {
+    throw std::runtime_error("cache ledger totals do not match cache state");
+  }
+  return result;
+}
+
+} // namespace qvac_lib_inference_addon_llama::cache