diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 4e5391301755..aefdac17ab8d 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=db21cbdd60f3d2ff62114bc863781ff8073ac39b +AUDIO_CPP_VERSION?=17751c0e8c48a3d56dcf05eeb60464409ecc69ce AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index 34fab9e6bec6..5f2ad4d310a2 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=ef40550042973817ac391ca95a2ff041f512257b +IK_LLAMA_VERSION?=7cff686d3732bfef5ce18bc4a6115fbceda29c14 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= diff --git a/backend/go/depth-anything-cpp/Makefile b/backend/go/depth-anything-cpp/Makefile index 79a1598e2b02..14fa58ec9c06 100644 --- a/backend/go/depth-anything-cpp/Makefile +++ b/backend/go/depth-anything-cpp/Makefile @@ -14,7 +14,7 @@ JOBS?=$(shell nproc --ignore=1) # It is kept alive by the upstream tag da2-support (survives a squash-merge); # repoint to the master merge commit once mudler/depth-anything.cpp PR #1 lands. DEPTHANYTHING_REPO?=https://github.com/mudler/depth-anything.cpp.git -DEPTHANYTHING_VERSION?=54abd5c0abfd1f394e01cb3c38f2e3af4daedf85 +DEPTHANYTHING_VERSION?=739992d10bf9472c46dcd4622b14d2b20766c58d ifeq ($(NATIVE),false) CMAKE_ARGS+=-DGGML_NATIVE=OFF diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 4aa9b09d0f49..89e533059ccf 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=50d640568388f876b0d63ee6ddb6bc86d997ec64 +STABLEDIFFUSION_GGML_VERSION?=be0e34480dada95f8ce9a021bbb95c5de85d67c7 CMAKE_ARGS+=-DGGML_MAX_NAME=128 diff --git a/docs/content/advanced/reverse-proxy-tls.md b/docs/content/advanced/reverse-proxy-tls.md index 29dc20dcae26..ccc0fb8284c9 100644 --- a/docs/content/advanced/reverse-proxy-tls.md +++ b/docs/content/advanced/reverse-proxy-tls.md @@ -1,6 +1,6 @@ --- title: TLS Reverse Proxy Configuration -description: Configure LocalAI behind a TLS termination reverse proxy (HAProxy, Apache, Nginx) +description: Configure LocalAI behind a TLS termination reverse proxy (HAProxy, Apache, Nginx, APISIX) weight: 100 --- @@ -147,6 +147,116 @@ the proxy stops waiting before LocalAI finishes, clients receive a proxy-generat Nginx Proxy Manager, Caddy, Traefik, HAProxy, and ingress controllers have equivalent upstream response timeout settings. +## Apache APISIX Configuration + +[Apache APISIX](https://apisix.apache.org/) is an open source API and AI +gateway. Put it between clients and LocalAI when you want to manage the +OpenAI-compatible API with the same gateway used for other services. In +addition to TLS termination, APISIX can add authentication, rate limiting, +load balancing, observability, and other policies through plugins without +changing LocalAI. + +This is useful when LocalAI runs on a private network but its API must be +available to several applications or teams. Clients keep using the standard +OpenAI-compatible API while APISIX provides one public entry point where you +can apply access and traffic policies. The configuration below starts with a +transparent route so it works with LocalAI clients before you add those +policies. + +The request path is: + +```text +OpenAI-compatible client -> APISIX -> LocalAI (:8080) +``` + +### Create the LocalAI Route + +Start LocalAI with the model configuration you want to serve, and confirm that +APISIX can reach `http://localai:8080`. When both services run in containers, +attach them to the same container network and use the LocalAI service name as +the upstream hostname. + +Install APISIX using its +[getting started guide](https://apisix.apache.org/docs/apisix/getting-started/README/), +then create a route that forwards the external scheme and host, allows +long-running inference, and disables response buffering for streaming +completions. This example assumes APISIX can resolve `localai` and reach it on +port `8080`: + +```bash +curl http://127.0.0.1:9180/apisix/admin/routes/localai \ + --request PUT \ + --header "X-API-KEY: ${admin_key}" \ + --data '{ + "uri": "/*", + "plugins": { + "proxy-rewrite": { + "headers": { + "set": { + "X-Forwarded-Proto": "$scheme", + "X-Forwarded-Host": "$host" + } + } + }, + "proxy-buffering": { + "disable_proxy_buffering": true + } + }, + "timeout": { + "connect": 60, + "send": 3600, + "read": 3600 + }, + "upstream": { + "type": "roundrobin", + "pass_host": "pass", + "nodes": { + "localai:8080": 1 + } + } + }' +``` + +Set `X-Forwarded-Prefix` in the `proxy-rewrite` header map as well if LocalAI +is exposed under a sub-path. Adjust the timeout values, in seconds, for the +slowest request you expect to serve. Keep the APISIX Admin API private and +replace `${admin_key}` with the key configured for your deployment. + +### Use LocalAI Through APISIX + +Clients continue to use LocalAI's OpenAI-compatible paths; only the base URL +changes. First, check that model discovery reaches LocalAI through the APISIX +data-plane port (port `9080` by default): + +```bash +curl http://127.0.0.1:9080/v1/models +``` + +Then test a streaming chat completion. Replace `your-model` with an ID returned +by `/v1/models`: + +```bash +curl --no-buffer http://127.0.0.1:9080/v1/chat/completions \ + --header "Content-Type: application/json" \ + --data '{ + "model": "your-model", + "messages": [{"role": "user", "content": "Hello from APISIX"}], + "stream": true + }' +``` + +With TLS configured on APISIX, applications use a base URL such as +`https://localai.example.com/v1`. Keep LocalAI's own API key in the usual +`Authorization: Bearer ...` header if LocalAI authentication is enabled. If +you enable an APISIX authentication plugin as well, configure its consumer +credential separately and send the header required by that plugin. + +For policies beyond this transparent route, see the +[APISIX AI Gateway overview](https://apisix.apache.org/ai-gateway/) and its +authentication, traffic management, load balancing, and observability +plugins. A gateway-specific LocalAI guide maintained by the APISIX project can +also be linked here when one is available. + For bulk jobs on a trusted private network, you can also bypass the public reverse proxy and connect directly to LocalAI, for example `http://localai-host:8080/v1`.