diff --git a/Makefile b/Makefile index 2167a9d..e37d3d5 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,16 @@ else ifneq ($(shell $(CXX) -fopenmp -E - < /dev/null 2>/dev/null && echo yes),) LDFLAGS += -lgomp endif + # ggml-cuda.a leaves the CUDA runtime, cuBLAS, and driver API unresolved. + # The stubs dir lets -lcuda link on hosts without a driver (e.g. release CI); + # the real libcuda.so.1 is picked up from the driver at load time. + ifneq (,$(filter -DGGML_CUDA=ON,$(CMAKE_FLAGS))) + CUDA_HOME ?= $(patsubst %/bin/nvcc,%,$(shell which nvcc 2>/dev/null)) + ifneq ($(CUDA_HOME),) + LDFLAGS += -L$(CUDA_HOME)/lib64 -L$(CUDA_HOME)/lib64/stubs + endif + LDFLAGS += -lcudart -lcublas -lcublasLt -lcuda + endif endif # CPU count for parallel builds diff --git a/c_src/llama_cpp_ex/llama_nif.cpp b/c_src/llama_cpp_ex/llama_nif.cpp index 31ffd67..35716c6 100644 --- a/c_src/llama_cpp_ex/llama_nif.cpp +++ b/c_src/llama_cpp_ex/llama_nif.cpp @@ -1794,11 +1794,19 @@ fine::Ok<> generate_mtp_tokens( // Re-decode the accepted tokens on the target so the next // iteration's draft starts from a consistent state. + // + // The KV must be rebuilt with the token that was `sampled` at + // the top of this iteration (batch element 0, at pos n_past), + // followed by all but the LAST emitted token. The last emitted + // token becomes the next iteration's `sampled` and is decoded + // then — including it here would duplicate it in the context. + // `sampled` sits at prompt[size - n_accepted_total - 1], since + // the accept loop pushed n_accepted_total tokens after it. if (n_accepted_total > 0) { llama_batch redo = llama_batch_init(n_accepted_total, 0, 1); for (int i = 0; i < n_accepted_total; i++) { llama_token tok = - prompt[prompt.size() - n_accepted_total + i]; + prompt[prompt.size() - n_accepted_total - 1 + i]; common_batch_add(redo, tok, n_past + static_cast(i), { seq_id },