Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/common/AutoModel/automodel.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector<int>& to
clear_context();
skip_count = 0;
}
// A fully cached prompt leaves nothing to prefill, and _chunked_insert then
// computes zero chunks and returns a default-constructed (empty) logits
// buffer straight into sampler->sample(). Re-prefill instead.
if (skip_count == tokens.size()) {
clear_context();
skip_count = 0;
}
tokens.erase(tokens.begin(), tokens.begin() + skip_count);

if (this->total_tokens + tokens.size() >= this->MAX_L){
Expand Down
6 changes: 4 additions & 2 deletions src/include/prompt_cache.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -173,9 +173,11 @@ class PromptCache {
// (in order) at the start of the new conversation, allowing rounds
// produced by other backends (cloud) to be appended without
// invalidating the locally-built KV cache prefix.
const size_t prefix_len = messages.size() - 2;
// Bound by the full incoming length, not length-2. Requiring two *new*
// messages made a resend of an unchanged conversation miss every time,
// which is exactly what a client sends after a timeout.
const bool can_use_message =
message_checksums_.size() <= prefix_len &&
message_checksums_.size() <= messages.size() &&
matched == message_checksums_.size();
const bool can_use_tools = tool_checksums_ == new_tool_checksums;
info.tools_matched = can_use_tools;
Expand Down