diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a94fb13 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,54 @@ +# Changelog + +All notable changes to this project are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project aims to +follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] — 2026-08-21 + +First tagged release: an offline LLM server on an unrooted Android phone, plus a +retrieval-augmented notes assistant, both reachable from a laptop or other +devices on the same network. + +### Added +- **Chat server** — `llama-server` serving an OpenAI-compatible API on `:8081`, + API-key protected, autostarting on boot via Termux:Boot and tmux. +- **Notes assistant (RAG)** — a pipeline that embeds a corpus of markdown notes, + retrieves the most relevant chunks for a question by cosine similarity (pure + standard library, no numpy), and answers from them with cited sources. +- **Browser UI + OpenAI-compatible RAG endpoint** (`rag/bin/rag-web.py`, `:8083`) + — a chat page and a `/v1/chat/completions` endpoint that apply retrieval before + answering and add a `sources` field. Binds LAN-wide behind the bearer token, or + loopback-only with `RAG_WEB_HOST=127.0.0.1`. +- **Shared core** (`rag/bin/ragcore.py`) — chunking, scoring, prompt assembly used + by the indexer, the CLI, and the web server so they cannot drift apart. Vectors + held as pre-normalized float32 arrays (~4.8 MB vs ~38 MB of Python floats). +- **Test suite** (`tests/`) — 60 tests covering the whole path against stand-in + model servers, so they run on a laptop and in CI with no phone and no model + weights. +- **CI** (`.github/workflows/ci.yml`) — pytest on Python 3.11–3.13, ruff, and + shellcheck. +- **Benchmark harness** (`bench/`) — `probe.py` reports live prompt/generation + throughput; `RESULTS.md` records the full matrix and the GPU story. + +### Performance +- **GPU offload** — prompt processing runs on the Adreno GPU through Mesa's turnip + Vulkan driver, taking prompt-eval from **18 to 70 tokens/sec**. Generation runs + on the six pinned performance cores at **12 tokens/sec**. A 605-token retrieval + prompt starts answering in **8.9 s instead of 36 s**. +- Retrieved context is bounded by a character budget rather than a fixed chunk + count, since time-to-first-token is prompt-eval bound. + +### Security +- API key minted with python's `secrets` module (Termux ships no openssl) and + stored `chmod 600`; never committed. +- The notes corpus and built index are gitignored — the index contains full chunk + text, so publishing it would publish the notes. + +### Setup +- `install.sh` is idempotent, runs a preflight (Termux, aarch64, free space), + installs the Vulkan GPU packages, and generates the key if absent. +- `bin/fetch-model.sh` pins and verifies the model's SHA256, skipping the + download when the file is already present and correct. + +[0.1.0]: https://github.com/ankamteja/android-llm-server/releases/tag/v0.1.0 diff --git a/bin/fetch-model.sh b/bin/fetch-model.sh index 597d58c..4d44657 100755 --- a/bin/fetch-model.sh +++ b/bin/fetch-model.sh @@ -1,15 +1,43 @@ #!/data/data/com.termux/files/usr/bin/bash -# Download the GGUF model. Resumable — safe to re-run after a dropped connection. -# Runs on the DEVICE (inside Termux). +# Download and verify the chat model (GGUF). Runs on the DEVICE (inside Termux). +# Resumable — safe to re-run after a dropped connection. set -euo pipefail +# Pin the exact build so a rebuild is reproducible. Override URL/DEST/SHA to use +# a different quant or model. URL="${LLM_MODEL_URL:-https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/Qwen3-4B-Instruct-2507-Q4_K_M.gguf}" DEST="${LLM_MODEL:-$HOME/models/qwen3-4b.gguf}" +SHA256="${LLM_MODEL_SHA256:-3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597}" mkdir -p "$(dirname "$DEST")" termux-wake-lock -# -C - resumes a partial file; --retry-all-errors survives flaky campus Wi-Fi. +verify() { + [ -r "$DEST" ] || return 1 + [ -n "$SHA256" ] || return 0 # nothing to check against + printf '%s %s\n' "$SHA256" "$DEST" | sha256sum -c --status +} + +if verify; then + echo "model already present and verified at $DEST" + exit 0 +fi + +# -C - resumes a partial file; --retry-all-errors survives a flaky connection. +echo "downloading $(basename "$URL") ..." curl -4 -L -C - --retry 999 --retry-delay 5 --retry-all-errors -o "$DEST" "$URL" +if [ -n "$SHA256" ]; then + echo "verifying checksum ..." + if verify; then + echo "checksum OK" + else + echo "CHECKSUM MISMATCH for $DEST" >&2 + echo " expected: $SHA256" >&2 + echo " got: $(sha256sum "$DEST" | cut -d' ' -f1)" >&2 + echo "delete the file and re-run, or set LLM_MODEL_SHA256 if you changed the model." >&2 + exit 1 + fi +fi + echo "model at $DEST ($(du -h "$DEST" | cut -f1))" diff --git a/install.sh b/install.sh index c556416..6ba6eb3 100755 --- a/install.sh +++ b/install.sh @@ -1,28 +1,86 @@ #!/data/data/com.termux/files/usr/bin/bash # One-shot setup. Runs on the DEVICE (inside Termux) on a fresh install. +# Idempotent: safe to re-run — it only fills in what is missing. set -euo pipefail -echo "==> installing packages" +say() { printf '==> %s\n' "$*"; } +die() { printf 'error: %s\n' "$*" >&2; exit 1; } + +# --- preflight ------------------------------------------------------------- +# Fail early with a clear message rather than halfway through a package install. +[ -n "${PREFIX:-}" ] && [ -d "$PREFIX" ] || die "this must run inside Termux (\$PREFIX not set)" +case "$(uname -m)" in + aarch64|arm64) : ;; + *) die "built for aarch64 phones; this device is $(uname -m)" ;; +esac +# The model alone is 2.3 GB; ask for a little headroom on top. +avail_kb=$(df -Pk "$HOME" | awk 'NR==2{print $4}') +[ "${avail_kb:-0}" -ge 3500000 ] || die "need ~3.5 GB free under \$HOME, have $((avail_kb/1024)) MB" + +# --- packages -------------------------------------------------------------- +# llama-cpp is the server. The Vulkan backend plus Mesa's turnip driver are what +# let prompt processing run on the Adreno GPU (4x faster than CPU); without them +# everything silently falls back to CPU. python runs the RAG pipeline and mints +# the API key. See bench/RESULTS.md for why the GPU packages matter. +say "installing packages (this pulls the Vulkan GPU backend)" pkg update -y -pkg install -y llama-cpp openssh tmux termux-api +pkg install -y \ + llama-cpp \ + llama-cpp-backend-vulkan \ + mesa-vulkan-icd-freedreno \ + python \ + openssh \ + tmux \ + termux-api -echo "==> creating directories" -mkdir -p "$HOME/bin" "$HOME/models" "$HOME/.config" "$HOME/.termux/boot" +# --- directories ----------------------------------------------------------- +say "creating directories" +mkdir -p "$HOME/bin" "$HOME/models" "$HOME/.config" "$HOME/.termux/boot" \ + "$HOME/rag/bin" "$HOME/rag/corpus" -echo "==> installing scripts" -install -m 700 bin/llm-server.sh "$HOME/bin/llm-server.sh" -install -m 700 bin/fetch-model.sh "$HOME/bin/fetch-model.sh" -install -m 755 boot/start-lab.sh "$HOME/.termux/boot/start-lab.sh" +# --- scripts --------------------------------------------------------------- +say "installing scripts" +install -m 700 bin/llm-server.sh "$HOME/bin/llm-server.sh" +install -m 700 bin/fetch-model.sh "$HOME/bin/fetch-model.sh" +install -m 755 boot/start-lab.sh "$HOME/.termux/boot/start-lab.sh" +# RAG pipeline: shared core plus the three entry points. +install -m 644 rag/bin/ragcore.py "$HOME/rag/bin/ragcore.py" +install -m 700 rag/bin/rag-embed-server.sh "$HOME/rag/bin/rag-embed-server.sh" +install -m 755 rag/bin/rag-index.py "$HOME/rag/bin/rag-index.py" +install -m 755 rag/bin/rag-ask.py "$HOME/rag/bin/rag-ask.py" +install -m 755 rag/bin/rag-web.py "$HOME/rag/bin/rag-web.py" +# --- API key --------------------------------------------------------------- +# Termux ships no openssl, so the key is minted with python's secrets module, +# which is always present. Never overwrite an existing key on a re-run. KEYFILE="$HOME/.config/llm-api-key" if [ ! -r "$KEYFILE" ]; then - echo "==> generating API key" - openssl rand -hex 24 > "$KEYFILE" + say "generating API key" + python3 -c "import secrets; print(secrets.token_hex(24))" > "$KEYFILE" chmod 600 "$KEYFILE" fi +# --- GPU sanity check ------------------------------------------------------ +# Not fatal: the server still runs on CPU if the GPU is unavailable, just slower. +if command -v vulkaninfo >/dev/null 2>&1; then + if vulkaninfo --summary 2>/dev/null | grep -qi adreno; then + say "GPU detected: $(vulkaninfo --summary 2>/dev/null | grep -i adreno | head -1 | sed 's/^[[:space:]]*//')" + fi +fi + echo -echo "setup done. API key:" +say "setup done. API key (also at $KEYFILE):" cat "$KEYFILE" echo -echo "next: ~/bin/fetch-model.sh then ~/bin/llm-server.sh" +cat <<'NEXT' +next steps: + ~/bin/fetch-model.sh download + verify the chat model (~2.3 GB, resumable) + ~/bin/llm-server.sh start the chat server on :8081 + +for the notes assistant (RAG), additionally: + - put an embedding model at ~/models/nomic-embed.gguf (see rag/README.md) + - put your notes (markdown) under ~/rag/corpus/ + - ~/rag/bin/rag-embed-server.sh then python3 ~/rag/bin/rag-index.py + +everything autostarts on reboot via ~/.termux/boot/start-lab.sh +NEXT