diff --git a/.requirements b/.requirements index 8bd4cf8ae875..9c76f3ee01d3 100644 --- a/.requirements +++ b/.requirements @@ -17,5 +17,5 @@ APISIX_PACKAGE_NAME=apisix -APISIX_RUNTIME=1.3.14 +APISIX_RUNTIME=1.3.16 APISIX_DASHBOARD_COMMIT=c8d3466d3c36386d3888efbc8250cd8183c77298 diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index ab34d8469a2b..4315794f21c7 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -309,6 +309,9 @@ local _M = { }, stream_plugins = { "ip-restriction", "limit-conn", "mqtt-proxy", "syslog", "traffic-split" }, plugin_attr = { + ["ai-proxy"] = { + http_client = "ngx_http_ffi_client" + }, ["log-rotate"] = { timeout = 10000, interval = 3600, diff --git a/apisix/plugins/ai-transport/http.lua b/apisix/plugins/ai-transport/http.lua index a2075aa3f4c3..ee72fd7a333b 100644 --- a/apisix/plugins/ai-transport/http.lua +++ b/apisix/plugins/ai-transport/http.lua @@ -19,7 +19,7 @@ -- Provides HTTP client lifecycle management for AI provider requests. local core = require("apisix.core") -local http = require("resty.http") +local http_client = require("apisix.utils.http") local ngx_now = ngx.now local pairs = pairs local ipairs = ipairs @@ -28,8 +28,41 @@ local type = type local str_lower = string.lower local tostring = tostring +local attr_schema = { + type = "object", + properties = { + http_client = http_client.client_schema, + }, +} + local _M = {} +local client_name + + +--- Which client this transport should use. +-- `plugin_attr.ai-proxy.http_client` names it; the shared module owns the +-- names, validation and loading. Read on first request, because local_conf is +-- not readable while this module is still loading. +local function resolve_client_name() + if client_name then + return client_name + end + + local local_conf = core.config.local_conf() + local attr = core.table.try_read_attr(local_conf, "plugin_attr", "ai-proxy") or {} + + local ok, err = core.schema.check(attr_schema, attr) + if not ok then + core.log.error("invalid plugin_attr.ai-proxy: ", err) + return nil, "invalid plugin_attr.ai-proxy: " .. err + end + + client_name = attr.http_client or http_client.DEFAULT_CLIENT + + return client_name +end + --- Map network errors to HTTP status codes. -- Cosocket timers report "timeout"; OS errno (ETIMEDOUT) and the resolver @@ -100,7 +133,12 @@ end -- @return string|nil Error message -- @return table|nil Upstream metadata on failure (for recording failed attempts) function _M.request(params, timeout) - local httpc, err = http.new() + local name, name_err = resolve_client_name() + if not name then + return nil, "failed to create http client: " .. name_err + end + + local httpc, err = http_client.new(name) if not httpc then return nil, "failed to create http client: " .. (err or "unknown") end diff --git a/apisix/utils/http.lua b/apisix/utils/http.lua new file mode 100644 index 000000000000..d63be0e13f2d --- /dev/null +++ b/apisix/utils/http.lua @@ -0,0 +1,125 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +--- Outbound HTTP client selection. +-- Shared by any module that makes outbound HTTP calls, so the choice between +-- `ngx_http_ffi_client` and `lua-resty-http` is made in one place rather than +-- per plugin. The caller decides where the preference comes from (its own +-- config key) and passes the name in. + +local core = require("apisix.core") +local pcall = pcall +local require = require +local tostring = tostring +local type = type + +local FFI_CLIENT = "ngx_http_ffi_client" +local LUA_RESTY_HTTP = "lua-resty-http" + +-- the client name used in configuration is not the module name +local CLIENT_MODULES = { + [FFI_CLIENT] = "resty.ngx_http_ffi_client", + [LUA_RESTY_HTTP] = "resty.http", +} + +local loaded = {} + + +local _M = { + version = 0.1, + FFI_CLIENT = FFI_CLIENT, + LUA_RESTY_HTTP = LUA_RESTY_HTTP, + DEFAULT_CLIENT = FFI_CLIENT, +} + + +--- Schema fragment for a client-name config field. +-- Callers embed this in their own attribute schema so every module validates +-- the name the same way. +_M.client_schema = { + type = "string", + enum = {FFI_CLIENT, LUA_RESTY_HTTP}, + default = FFI_CLIENT, +} + + +--- Load the module for a client name. +-- `ngx_http_ffi_client` is a C client with the same object API as +-- lua-resty-http and around half its outbound CPU cost, and it exists only +-- when the gateway runtime was built with the module. A name that cannot be +-- loaded is an error, never a silent switch to the other client. +-- Cached per name once loaded, so a failure is retried rather than remembered. +local function load_client(name) + local cached = loaded[name] + if cached then + return cached + end + + local module_name = CLIENT_MODULES[name] + if not module_name then + return nil, "unknown http client: " .. tostring(name) + end + + local ok, mod = pcall(require, module_name) + if not ok or type(mod) ~= "table" then + core.log.error(module_name, " is not available: ", mod) + return nil, module_name .. " is not available: " .. tostring(mod) + end + + -- Cosockets have their names resolved by apisix/patch.lua, which routes + -- them through core.resolver and so honours dns_resolver, /etc/hosts and + -- the search domains. The C client dials from C and never touches a + -- cosocket, so without this it would see only nginx's `resolver`. Handing + -- it the same resolver keeps every outbound name on one set of rules. + -- A client too old to take one is an error rather than a client quietly + -- resolving names by different rules than the rest of the gateway. + if name == FFI_CLIENT then + if type(mod.set_resolver) ~= "function" then + core.log.error(module_name, " does not support set_resolver, ", + "the runtime is older than the pinned one") + return nil, module_name .. " does not support set_resolver, " + .. "the runtime is older than the pinned one" + end + + mod.set_resolver(core.resolver.parse_domain) + end + + loaded[name] = mod + + return mod +end + + +--- Create an HTTP client. +-- @tparam string|nil name client name; defaults to DEFAULT_CLIENT +-- @treturn table|nil the client +-- @treturn string|nil error message +function _M.new(name) + name = name or _M.DEFAULT_CLIENT + + local mod, err = load_client(name) + if not mod then + return nil, err + end + + -- The Lua half of `ngx_http_ffi_client` loads even when the C module is + -- not compiled into the runtime; new() is what reports that. + return mod.new() +end + + +return _M diff --git a/ci/linux-install-openresty.sh b/ci/linux-install-openresty.sh index 07f05e82b84b..ffdab29ad570 100755 --- a/ci/linux-install-openresty.sh +++ b/ci/linux-install-openresty.sh @@ -61,7 +61,7 @@ else sudo apt-get -y update --fix-missing sudo apt-get install -y build-essential gcc g++ cpanminus libxml2-dev libxslt-dev - if [ "$APISIX_RUNTIME" != "1.3.14" ]; then + if [ "$APISIX_RUNTIME" != "1.3.16" ]; then echo "Please update the apisix-runtime-debug checksum for APISIX_RUNTIME=$APISIX_RUNTIME" >&2 exit 1 fi @@ -69,11 +69,11 @@ else case "$ARCH" in x86_64|amd64) DEB_ARCH="amd64" - EXPECTED_SHA256="2d2350347c982e4467ff9326b5b93fcb9af2089b33b02bcd426e84a5adacf6f2" + EXPECTED_SHA256="a56f0adc9bf6f6a491f7548df4f8e45fa3df3dd5e209d4a2ba5341b66eb7e060" ;; arm64|aarch64) DEB_ARCH="arm64" - EXPECTED_SHA256="495320e6377b96ab8d8a80a980a845e346c88160e2798ba46113a4da9042af4d" + EXPECTED_SHA256="b645ee4f5ea36d26aaacb1b1c8278d89756b0b8448701ec561ab982b4768204a" ;; *) echo "Unsupported architecture: $ARCH" >&2 diff --git a/conf/config.yaml.example b/conf/config.yaml.example index 26954e2797a2..6f6c791ea12e 100644 --- a/conf/config.yaml.example +++ b/conf/config.yaml.example @@ -659,6 +659,9 @@ stream_plugins: # stream plugin list (sorted by priority) # protocols: # - name: pingpong plugin_attr: # Plugin attributes + ai-proxy: # Plugin: ai-proxy, ai-proxy-multi + http_client: ngx_http_ffi_client # HTTP client the AI plugins use to reach the + # LLM upstream: ngx_http_ffi_client or lua-resty-http. log-rotate: # Plugin: log-rotate timeout: 10000 # maximum wait time for a log rotation(unit: millisecond) interval: 3600 # Set the log rotate interval in seconds. diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index dc58ca565fbf..ea2640236014 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -152,6 +152,23 @@ By default, `ai-proxy-multi` forwards the incoming client request headers to the Because the LLM upstream is often a third-party service, be aware that any header the client sends (for example `Authorization`, `Cookie`, or internal application headers) is forwarded to that provider unless it is overridden by `auth.header`. If the client should not expose certain headers to the LLM provider, strip them before the request reaches `ai-proxy-multi`, for example with the [`proxy-rewrite`](./proxy-rewrite.md) plugin. +## Upstream HTTP Client + +Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client that costs around half the outbound CPU time of `lua-resty-http`. Both clients behave the same on the wire. + +`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client: + +```yaml +plugin_attr: + ai-proxy: + http_client: ngx_http_ffi_client # or lua-resty-http +``` + +- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime built with the module, which the runtime pinned in `.requirements` is. On a runtime without it, the request fails and the error names the missing module; the plugin never silently switches clients. +- `lua-resty-http`: the Lua client, on every runtime. + +The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`, which share the same transport. + ## Upstream Error Responses When the selected LLM upstream returns a `429` or `5xx` status, `ai-proxy-multi` reads the upstream error body before deciding whether to fall back: diff --git a/docs/en/latest/plugins/ai-proxy.md b/docs/en/latest/plugins/ai-proxy.md index b93702ed00af..c9f132bbb611 100644 --- a/docs/en/latest/plugins/ai-proxy.md +++ b/docs/en/latest/plugins/ai-proxy.md @@ -145,6 +145,23 @@ By default, `ai-proxy` forwards the incoming client request headers to the confi Because the LLM upstream is often a third-party service, be aware that any header the client sends (for example `Authorization`, `Cookie`, or internal application headers) is forwarded to that provider unless it is overridden by `auth.header`. If the client should not expose certain headers to the LLM provider, strip them before the request reaches `ai-proxy`, for example with the [`proxy-rewrite`](./proxy-rewrite.md) plugin. +## Upstream HTTP Client + +Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client that costs around half the outbound CPU time of `lua-resty-http`. Both clients behave the same on the wire. + +`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client: + +```yaml +plugin_attr: + ai-proxy: + http_client: ngx_http_ffi_client # or lua-resty-http +``` + +- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime built with the module, which the runtime pinned in `.requirements` is. On a runtime without it, the request fails and the error names the missing module; the plugin never silently switches clients. +- `lua-resty-http`: the Lua client, on every runtime. + +The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`, which share the same transport. + ## Upstream Error Responses When the LLM upstream returns a `429` or `5xx` status, `ai-proxy` reads the upstream error body and returns it to the client together with the upstream status code and `Content-Type`, so provider-side error details (such as rate-limit information or validation errors) are not discarded. diff --git a/t/plugin/ai-transport-http.t b/t/plugin/ai-transport-http.t index b475164939d3..58e23a6cf7c7 100644 --- a/t/plugin/ai-transport-http.t +++ b/t/plugin/ai-transport-http.t @@ -33,6 +33,10 @@ run_tests; __DATA__ === TEST 1: AI transport encodes upstream request body with sorted keys and preserves empty arrays +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: lua-resty-http --- config location /t { content_by_lua_block { @@ -96,6 +100,10 @@ __DATA__ === TEST 2: AI transport falls back to cjson when rapidjson encode fails +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: lua-resty-http --- config location /t { content_by_lua_block { @@ -197,6 +205,10 @@ rapidjson nested empty table: \{\} === TEST 4: AI transport preserves JSON null values from cjson decode +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: lua-resty-http --- config location /t { content_by_lua_block { @@ -242,6 +254,10 @@ failed to encode AI request body with rapidjson: === TEST 5: AI transport preserves manually constructed arrays +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: lua-resty-http --- config location /t { content_by_lua_block { @@ -292,6 +308,10 @@ failed to encode AI request body with rapidjson: === TEST 6: connect timeout ("Operation timed out") maps to 504 +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: lua-resty-http --- config location /t { content_by_lua_block { @@ -353,3 +373,673 @@ connect: operation timed out => 504 connect: Operation timed out => 504 request: connection refused => 500 request: connection reset by peer => 500 + + + +=== TEST 8: ngx_http_ffi_client is the default client +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + ngx.say("lua-resty-http client created") + return { + set_timeout = function() end, + connect = function() return true end, + request = function() return {headers = {}, status = 200} end, + } + end, + } + + package.loaded["resty.ngx_http_ffi_client"] = { + set_resolver = function() end, + new = function() + return { + set_timeout = function() end, + connect = function() return 1 end, + request = function(_, params) + ngx.say("ffi client request: ", params.body) + return {headers = {}, status = 200} + end, + } + end, + } + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {model = "m"}, + }, 1000) + ngx.say("status: ", res and res.status or err) + + package.loaded["resty.http"] = orig_http + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +ffi client request: {"model":"m"} +status: 200 + + + +=== TEST 9: a runtime without the C module fails the request +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + ngx.say("lua-resty-http client created") + return { + set_timeout = function() end, + connect = function() return true end, + request = function() return {headers = {}, status = 200} end, + } + end, + } + + -- the Lua half loads even when the C module is not built in + package.loaded["resty.ngx_http_ffi_client"] = { + set_resolver = function() end, + new = function() + return nil, "ngx_http_ffi_client_module is not loaded" + end, + } + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + for _ = 1, 2 do + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {model = "m"}, + }, 1000) + ngx.say("status: ", res and res.status or err) + end + + package.loaded["resty.http"] = orig_http + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +status: failed to create http client: ngx_http_ffi_client_module is not loaded +status: failed to create http client: ngx_http_ffi_client_module is not loaded + + + +=== TEST 10: plugin_attr.ai-proxy.http_client selects lua-resty-http +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: lua-resty-http +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + return { + set_timeout = function() end, + connect = function() return true end, + request = function(_, params) + ngx.say("lua-resty-http request: ", params.body) + return {headers = {}, status = 200} + end, + } + end, + } + + package.loaded["resty.ngx_http_ffi_client"] = { + new = function() + return { + set_timeout = function() end, + connect = function() return 1 end, + request = function() + ngx.say("ffi client request") + return {headers = {}, status = 200} + end, + } + end, + } + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {model = "m"}, + }, 1000) + ngx.say("status: ", res and res.status or err) + + package.loaded["resty.http"] = orig_http + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +lua-resty-http request: {"model":"m"} +status: 200 + + + +=== TEST 11: the lua-resty-http path reaches a real upstream +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: lua-resty-http +--- config + location = /mock-llm { + content_by_lua_block { + ngx.req.read_body() + ngx.header["Content-Type"] = "application/json" + ngx.print('{"echo":', ngx.req.get_body_data(), '}') + } + } + + location /t { + content_by_lua_block { + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + + -- lua-resty-http stays real; only the client that must not be + -- picked is stubbed, so a regression in the selection shows up + package.loaded["resty.ngx_http_ffi_client"] = { + new = function() + ngx.log(ngx.ERR, "unexpected ngx_http_ffi_client selection") + return nil, "unexpected ngx_http_ffi_client selection" + end, + } + + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + method = "POST", + scheme = "http", + host = "127.0.0.1", + port = 1984, + path = "/mock-llm", + headers = {["content-type"] = "application/json"}, + body = {model = "m"}, + }, 2000) + + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + + if not res then + ngx.say("err: ", err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-type: ", res.headers["Content-Type"]) + ngx.say("body: ", res:read_body()) + transport.set_keepalive(res, 60000, 30) + } + } +--- response_body +status: 200 +content-type: application/json +body: {"echo":{"model":"m"}} +--- no_error_log +[error] +unexpected ngx_http_ffi_client selection + + + +=== TEST 12: a module that loads but is not a table fails the request +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + ngx.say("lua-resty-http client created") + return { + set_timeout = function() end, + connect = function() return true end, + request = function() return {headers = {}, status = 200} end, + } + end, + } + + -- require() returns this instead of the module table + package.loaded["resty.ngx_http_ffi_client"] = "not a module" + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {model = "m"}, + }, 1000) + ngx.say("status: ", res and res.status or err) + + package.loaded["resty.http"] = orig_http + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +status: failed to create http client: resty.ngx_http_ffi_client is not available: not a module +--- error_log +resty.ngx_http_ffi_client is not available: not a module + + + +=== TEST 13: an unknown plugin_attr.ai-proxy.http_client fails the request +--- extra_yaml_config +plugin_attr: + ai-proxy: + http_client: curl +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + ngx.say("lua-resty-http client created") + return { + set_timeout = function() end, + connect = function() return true end, + request = function() return {headers = {}, status = 200} end, + } + end, + } + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {model = "m"}, + }, 1000) + ngx.say("status: ", res and res.status or err) + + package.loaded["resty.http"] = orig_http + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +status: failed to create http client: invalid plugin_attr.ai-proxy: property "http_client" validation failed: matches none of the enum values +--- error_log +invalid plugin_attr.ai-proxy: property "http_client" validation failed + + + +=== TEST 14: the C client is given the gateway's resolver +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + local orig_utils = package.loaded["apisix.utils.http"] + + local installed + package.loaded["resty.ngx_http_ffi_client"] = { + set_resolver = function(fn) + installed = fn + end, + new = function() + return {} + end, + } + + package.loaded["apisix.utils.http"] = nil + local http_client = require("apisix.utils.http") + http_client.new(http_client.FFI_CLIENT) + + -- the C client dials on its own, so what it resolves names with has + -- to be the resolver every other socket in the gateway uses + ngx.say("gateway resolver installed: ", + installed == core.resolver.parse_domain) + ngx.say("resolves through it: ", installed("localhost")) + + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + package.loaded["apisix.utils.http"] = orig_utils + } + } +--- response_body +gateway resolver installed: true +resolves through it: 127.0.0.1 +--- no_error_log +[error] + + + +=== TEST 15: a module whose loader raises fails the request +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + local orig_preload = package.preload["resty.ngx_http_ffi_client"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + ngx.say("lua-resty-http client created") + return { + set_timeout = function() end, + connect = function() return true end, + request = function() return {headers = {}, status = 200} end, + } + end, + } + + -- an unloaded module whose loader raises: this is what a require() + -- failure looks like, as opposed to one that loads the wrong thing + package.loaded["resty.ngx_http_ffi_client"] = nil + package.preload["resty.ngx_http_ffi_client"] = function() + error("simulated loader failure", 0) + end + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {model = "m"}, + }, 1000) + ngx.say("status: ", res and res.status or err) + + package.loaded["resty.http"] = orig_http + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + package.preload["resty.ngx_http_ffi_client"] = orig_preload + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +status: failed to create http client: resty.ngx_http_ffi_client is not available: simulated loader failure +--- error_log +resty.ngx_http_ffi_client is not available: simulated loader failure + + + +=== TEST 16: the C client carries a buffered request to a real upstream +--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 | grep -q ngx_http_ffi_client")) != 0 +--- config + location = /mock-buffered { + content_by_lua_block { + ngx.req.read_body() + ngx.header["Content-Type"] = "application/json" + ngx.print('{"echo":', ngx.req.get_body_data(), '}') + } + } + + location /t { + content_by_lua_block { + -- nothing stubbed: this is the real C client on the default setting + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + method = "POST", + scheme = "http", + host = "127.0.0.1", + port = 1984, + path = "/mock-buffered", + headers = {["content-type"] = "application/json"}, + body = {model = "m"}, + }, 2000) + if not res then + ngx.say("err: ", err) + return + end + ngx.say("status: ", res.status) + ngx.say("content-type: ", res.headers["Content-Type"]) + ngx.say("body: ", res:read_body()) + transport.set_keepalive(res, 60000, 30) + } + } +--- response_body +status: 200 +content-type: application/json +body: {"echo":{"model":"m"}} +--- no_error_log +[error] + + + +=== TEST 17: the C client streams an SSE response through body_reader +--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 | grep -q ngx_http_ffi_client")) != 0 +--- config + location = /mock-sse { + content_by_lua_block { + ngx.header["Content-Type"] = "text/event-stream" + for i = 1, 3 do + ngx.print("data: {\"n\":", i, "}\n\n") + ngx.flush(true) + end + ngx.print("data: [DONE]\n\n") + ngx.flush(true) + } + } + + location /t { + content_by_lua_block { + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + method = "POST", + scheme = "http", + host = "127.0.0.1", + port = 1984, + path = "/mock-sse", + headers = {["content-type"] = "application/json"}, + body = {model = "m", stream = true}, + }, 2000) + if not res then + ngx.say("err: ", err) + return + end + ngx.say("status: ", res.status) + ngx.say("content-type: ", res.headers["Content-Type"]) + + local reader = res.body_reader + if not reader then + ngx.say("no body_reader") + return + end + local buf = {} + while true do + local chunk, rerr = reader(4096) + if rerr then + ngx.say("read err: ", rerr) + break + end + if not chunk then + break + end + buf[#buf + 1] = chunk + end + local body = table.concat(buf) + local n = select(2, body:gsub("data: ", "")) + ngx.say("sse events: ", n) + ngx.say("saw done: ", body:find("[DONE]", 1, true) ~= nil) + } + } +--- response_body +status: 200 +content-type: text/event-stream +sse events: 4 +saw done: true +--- no_error_log +[error] + + + +=== TEST 18: the C client reuses a pooled connection across requests +--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 | grep -q ngx_http_ffi_client")) != 0 +--- config + location = /mock-keepalive { + # counts requests served on this connection: 1,2,3 proves one + # pooled connection carried all three + content_by_lua_block { ngx.print("req ", ngx.var.connection_requests) } + } + + location /t { + content_by_lua_block { + local transport = require("apisix.plugins.ai-transport.http") + for i = 1, 3 do + local res, err = transport.request({ + method = "POST", + scheme = "http", + host = "127.0.0.1", + port = 1984, + path = "/mock-keepalive", + headers = {["content-type"] = "application/json"}, + body = {model = "m"}, + }, 2000) + if not res then + ngx.say(i, ": err ", err) + return + end + local body = res:read_body() + transport.set_keepalive(res, 60000, 30) + ngx.say(i, ": ", res.status, " ", body) + end + } + } +--- response_body +1: 200 req 1 +2: 200 req 2 +3: 200 req 3 +--- no_error_log +[error] + + + +=== TEST 19: the C client reaches an upstream named by hostname +--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 | grep -q ngx_http_ffi_client")) != 0 +--- config + location = /mock-host { + content_by_lua_block { + ngx.print("host header: ", ngx.var.http_host) + } + } + + location /t { + content_by_lua_block { + -- "localhost" only resolves through core.resolver, so this proves + -- the real client is going through the resolver it was given + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + method = "POST", + scheme = "http", + host = "localhost", + port = 1984, + path = "/mock-host", + headers = {["content-type"] = "application/json"}, + body = {model = "m"}, + }, 2000) + if not res then + ngx.say("err: ", err) + return + end + ngx.say("status: ", res.status) + ngx.say(res:read_body()) + } + } +--- response_body +status: 200 +host header: localhost:1984 +--- no_error_log +[error] + + + +=== TEST 20: the C client verifies TLS against the configured trust store +--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 | grep -q ngx_http_ffi_client")) != 0 +--- http_config + server { + listen 21981 ssl; + ssl_certificate cert/apisix.crt; + ssl_certificate_key cert/apisix.key; + server_name test.com; + location = /mock-tls { + content_by_lua_block { ngx.print("tls ok") } + } + } +--- config + location /t { + content_by_lua_block { + -- no per-call CA: the trust store comes from + -- lua_ssl_trusted_certificate, as it does for every cosocket + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + method = "POST", + scheme = "https", + host = "127.0.0.1", + port = 21981, + path = "/mock-tls", + ssl_verify = true, + ssl_server_name = "test.com", + headers = {["content-type"] = "application/json"}, + body = {model = "m"}, + }, 2000) + if not res then + ngx.say("err: ", err) + return + end + ngx.say("status: ", res.status) + ngx.say(res:read_body()) + } + } +--- response_body +status: 200 +tls ok +--- no_error_log +[error] + + + +=== TEST 21: a client that cannot take a resolver fails the request +--- config + location /t { + content_by_lua_block { + local orig_ffi = package.loaded["resty.ngx_http_ffi_client"] + local orig_utils = package.loaded["apisix.utils.http"] + + -- a runtime older than the pinned one: the module is there, but it + -- resolves names by nginx's rules rather than the gateway's + package.loaded["resty.ngx_http_ffi_client"] = { + new = function() + return {} + end, + } + + package.loaded["apisix.utils.http"] = nil + local http_client = require("apisix.utils.http") + for _ = 1, 2 do + local client, err = http_client.new(http_client.FFI_CLIENT) + ngx.say("client: ", client and "created" or err) + end + + package.loaded["resty.ngx_http_ffi_client"] = orig_ffi + package.loaded["apisix.utils.http"] = orig_utils + } + } +--- response_body +client: resty.ngx_http_ffi_client does not support set_resolver, the runtime is older than the pinned one +client: resty.ngx_http_ffi_client does not support set_resolver, the runtime is older than the pinned one +--- error_log +does not support set_resolver