diff --git a/.requirements b/.requirements index 215b4f93b283..b484c4955792 100644 --- a/.requirements +++ b/.requirements @@ -17,5 +17,5 @@ APISIX_PACKAGE_NAME=apisix -APISIX_RUNTIME=1.3.16 +APISIX_RUNTIME=1.3.17 APISIX_DASHBOARD_COMMIT=045e3142867e3b7d5d1b8ec40bf8f66a7ce24a64 diff --git a/apisix/balancer.lua b/apisix/balancer.lua index ff2f8fb8dcaf..35f015da1b45 100644 --- a/apisix/balancer.lua +++ b/apisix/balancer.lua @@ -27,6 +27,8 @@ local set_more_tries = balancer.set_more_tries local get_last_failure = balancer.get_last_failure local set_timeouts = balancer.set_timeouts local ngx_now = ngx.now +local ngx_md5 = ngx.md5 +local tostring = tostring local module_name = "balancer" local pickers = {} @@ -369,6 +371,18 @@ end _M.pick_server = pick_server +-- Keyed by the `ca_certs` array itself: a config update always rebuilds that +-- table, so a stale digest can never outlive the certificates it was made from. +local ca_certs_digest_cache = core.lrucache.new({ + ttl = 300, count = 256, +}) + + +local function ca_certs_digest(ca_certs) + return ngx_md5(core.table.concat(ca_certs, "\n")) +end + + local set_current_peer do local pool_opt = {} @@ -409,12 +423,25 @@ do local sni = ctx.var.upstream_host pool = pool .. "#" .. sni + local tls = up_conf.tls -- separate the pool by client cert so referenced SSL objects -- don't share a connection - if up_conf.tls and up_conf.tls.client_cert then - pool = pool .. "#" .. up_conf.tls.client_cert - elseif up_conf.tls and up_conf.tls.client_cert_id then - pool = pool .. "#" .. up_conf.tls.client_cert_id + if tls and tls.client_cert then + pool = pool .. "#" .. tls.client_cert + elseif tls and tls.client_cert_id then + pool = pool .. "#" .. tls.client_cert_id + end + + -- and by the verification policy, which is applied while the + -- connection is being established: a pooled connection keeps + -- whatever policy it was handshaked under, so reusing it across + -- policies would skip the verification the config asks for + if tls and (tls.verify ~= nil or tls.ca_certs) then + pool = pool .. "#" .. tostring(tls.verify) + if tls.ca_certs then + pool = pool .. "#" .. ca_certs_digest_cache(tls.ca_certs, nil, + ca_certs_digest, tls.ca_certs) + end end end pool_opt.pool = pool diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index df2641106de1..218ffefa4982 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -1002,6 +1002,9 @@ http { grpc_set_header Content-Type application/grpc; grpc_set_header TE trailers; grpc_socket_keepalive on; + # only consulted once upstream.tls.verify turns verification on; + # without it the certificate would be checked against "apisix_backend" + grpc_ssl_name $upstream_host; grpc_pass $upstream_scheme://apisix_backend; {% if enabled_plugins["proxy-mirror"] then %} diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua index ba4010bef2e3..3687cbcf5f3f 100644 --- a/apisix/schema_def.lua +++ b/apisix/schema_def.lua @@ -444,9 +444,14 @@ local upstream_schema = { client_key = private_key_schema, verify = { type = "boolean", - description = "Turn on server certificate verification, ".. - "currently only kafka upstream is supported", - default = false, + description = "enable or disable upstream certificate verification, " .. + "fall back to the nginx configuration when not set", + }, + ca_certs = { + type = "array", + description = "CA certificates used to verify the upstream certificate", + minItems = 1, + items = certificate_scheme, }, }, dependencies = { diff --git a/apisix/upstream.lua b/apisix/upstream.lua index 232b0eb0dd48..7ed61faf9214 100644 --- a/apisix/upstream.lua +++ b/apisix/upstream.lua @@ -20,6 +20,8 @@ local discovery = require("apisix.discovery.init").discovery local upstream_util = require("apisix.utils.upstream") local apisix_ssl = require("apisix.ssl") local resource = require("apisix.resource") +local openssl_x509 = require("resty.openssl.x509") +local openssl_x509_store = require("resty.openssl.x509.store") local error = error local tostring = tostring local ipairs = ipairs @@ -32,14 +34,95 @@ local upstreams local healthcheck_manager local set_upstream_tls_client_param +local set_upstream_ssl_verify +local set_upstream_ssl_trusted_store local ok, apisix_ngx_upstream = pcall(require, "resty.apisix.upstream") if ok then set_upstream_tls_client_param = apisix_ngx_upstream.set_cert_and_key -else + set_upstream_ssl_verify = apisix_ngx_upstream.set_ssl_verify + set_upstream_ssl_trusted_store = apisix_ngx_upstream.set_ssl_trusted_store +end +-- guard each function independently: an older runtime may expose the module +-- (set_cert_and_key) without the newer upstream TLS C-APIs +if not set_upstream_tls_client_param then set_upstream_tls_client_param = function () return nil, "need to build APISIX-Runtime to support upstream mTLS" end end +if not set_upstream_ssl_verify then + set_upstream_ssl_verify = function () + return nil, "need to build APISIX-Runtime to support upstream certificate verification" + end +end +if not set_upstream_ssl_trusted_store then + set_upstream_ssl_trusted_store = function () + return nil, "need to build APISIX-Runtime to support upstream CA certificates" + end +end + + +-- Keyed by the `ca_certs` array itself: a config update always rebuilds that +-- table, so a stale store can never outlive the certificates it was built from. +local trusted_store_cache = core.lrucache.new({ + ttl = 300, count = 256, +}) + + +local function create_trusted_store(ca_certs) + local store, err = openssl_x509_store.new() + if not store then + return nil, err + end + + for _, ca_cert in ipairs(ca_certs) do + local x509, err = openssl_x509.new(ca_cert, "PEM") + if not x509 then + return nil, err + end + + local ok, err = store:add(x509) + if not ok then + return nil, err + end + end + + return store +end + + +-- Apply upstream.tls.verify / upstream.tls.ca_certs to the current request. +-- Both settings live in the apisix-nginx-module request context, which is wiped +-- by the internal redirect to @grpc_pass, so grpcs has to apply them again from +-- `grpc_access_phase` just like the client certificate does. +local function set_upstream_ssl_opts(up_conf) + local tls = up_conf.tls + if not tls then + return true + end + + if tls.verify ~= nil then + local ok, err = set_upstream_ssl_verify(tls.verify) + if not ok then + return nil, err + end + end + + if tls.ca_certs then + local store, err = trusted_store_cache(tls.ca_certs, up_conf.resource_version, + create_trusted_store, tls.ca_certs) + if not store then + return nil, err + end + + local ok, err = set_upstream_ssl_trusted_store(store) + if not ok then + return nil, err + end + end + + return true +end + local set_stream_upstream_tls local set_stream_upstream_cert_and_key @@ -356,6 +439,15 @@ function _M.set_by_route(route, api_ctx) local checker = healthcheck_manager.fetch_checker(up_conf.resource_key, resource_version) api_ctx.up_checker = checker local scheme = up_conf.scheme + if scheme == "https" then + -- grpcs applies these in `set_grpcs_upstream_param` instead, after the + -- internal redirect that drops the settings made here + local ok, err = set_upstream_ssl_opts(up_conf) + if not ok then + return 503, err + end + end + local tls_has_cert = up_conf.tls and (up_conf.tls.client_cert or up_conf.tls.client_cert_id) if (scheme == "https" or scheme == "grpcs") and tls_has_cert then local client_cert, client_key @@ -395,6 +487,13 @@ end function _M.set_grpcs_upstream_param(ctx) + if ctx.upstream_conf and ctx.upstream_conf.scheme == "grpcs" then + local ok, err = set_upstream_ssl_opts(ctx.upstream_conf) + if not ok then + return 503, err + end + end + if ctx.upstream_grpcs_cert then local cert = ctx.upstream_grpcs_cert local key = ctx.upstream_grpcs_key @@ -517,6 +616,15 @@ local function check_upstream_conf(in_dp, conf) end end + if conf.tls and conf.tls.ca_certs then + for _, ca_cert in ipairs(conf.tls.ca_certs) do + local ok, err = apisix_ssl.validate(ca_cert) + if not ok then + return false, err + end + end + end + if conf.type ~= "chash" then return true end diff --git a/ci/linux-install-openresty.sh b/ci/linux-install-openresty.sh index ffdab29ad570..fbc11b2671bd 100755 --- a/ci/linux-install-openresty.sh +++ b/ci/linux-install-openresty.sh @@ -61,7 +61,7 @@ else sudo apt-get -y update --fix-missing sudo apt-get install -y build-essential gcc g++ cpanminus libxml2-dev libxslt-dev - if [ "$APISIX_RUNTIME" != "1.3.16" ]; then + if [ "$APISIX_RUNTIME" != "1.3.17" ]; then echo "Please update the apisix-runtime-debug checksum for APISIX_RUNTIME=$APISIX_RUNTIME" >&2 exit 1 fi @@ -69,11 +69,11 @@ else case "$ARCH" in x86_64|amd64) DEB_ARCH="amd64" - EXPECTED_SHA256="a56f0adc9bf6f6a491f7548df4f8e45fa3df3dd5e209d4a2ba5341b66eb7e060" + EXPECTED_SHA256="d60067ba7a89cab6fca8e70994e4158fa8c414a569406e1692998be2567832a6" ;; arm64|aarch64) DEB_ARCH="arm64" - EXPECTED_SHA256="b645ee4f5ea36d26aaacb1b1c8278d89756b0b8448701ec561ab982b4768204a" + EXPECTED_SHA256="2db6619c6fa31128e7ea45b2cdcc56dd26d2b11d7b4a68600f4f04746cb34766" ;; *) echo "Unsupported architecture: $ARCH" >&2 diff --git a/docs/en/latest/FAQ.md b/docs/en/latest/FAQ.md index 96c3f3a0c542..7c4db8d85985 100644 --- a/docs/en/latest/FAQ.md +++ b/docs/en/latest/FAQ.md @@ -728,7 +728,7 @@ The `ssls` is managed through the `/apisix/admin/ssls` API. It's used for managi The `tls.client_cert`, `tls.client_key`, and `tls.client_cert_id` in upstream are used for mTLS communication with the upstream. -The `ssl_trusted_certificate` in `config.yaml` configures a trusted CA certificate. It is used for verifying some certificates signed by private authorities within APISIX, to avoid APISIX rejects the certificate. Note that it is not used to trust the certificates of APISIX upstream, because APISIX does not verify the legality of the upstream certificates. Therefore, even if the upstream uses an invalid TLS certificate, it can still be accessed without configuring a root certificate. +The `ssl_trusted_certificate` in `config.yaml` configures a trusted CA certificate. It is used for verifying some certificates signed by private authorities within APISIX, to avoid APISIX rejects the certificate. Note that APISIX does not verify the certificate of an upstream unless that upstream sets `tls.verify` to `true`, so by default an upstream using an invalid TLS certificate can still be accessed. Once `tls.verify` is enabled, the certificate is checked against the upstream's own `tls.ca_certs` if set, and against `ssl_trusted_certificate` otherwise. ## Where can I find more answers? diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 389b1b0a924e..abd9965d3703 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -1019,7 +1019,8 @@ In addition to the equalization algorithm selections, Upstream also supports pas | tls.client_cert | False, can't be used with `tls.client_cert_id` | HTTPS certificate | Sets the client certificate while connecting to a TLS Upstream. | | | tls.client_key | False, can't be used with `tls.client_cert_id` | HTTPS certificate private key | Sets the client private key while connecting to a TLS Upstream. | | | tls.client_cert_id | False, can't be used with `tls.client_cert` and `tls.client_key` | SSL | Set the referenced [SSL](#ssl) id. | | -| tls.verify | False, currently only kafka upstream is supported | Boolean | Turn on server certificate verification, currently only kafka upstream is supported. | | +| tls.verify | False | Boolean | Enables or disables verification of the Upstream certificate. Falls back to the nginx configuration when unset. Also used by the `kafka` scheme. | | +| tls.ca_certs | False | Array of HTTPS certificates | CA certificates used to verify the Upstream certificate, replacing the ones loaded from `ssl_trusted_certificate`. | | | keepalive_pool.size | False | Auxiliary | Sets `keepalive` directive dynamically. | | | keepalive_pool.idle_timeout | False | Auxiliary | Sets `keepalive_timeout` directive dynamically. | | | keepalive_pool.requests | False | Auxiliary | Sets `keepalive_requests` directive dynamically. | | @@ -1048,6 +1049,22 @@ To use mTLS to communicate with Upstream, you can use the `tls.client_cert/key` Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is `client`, otherwise the request will be rejected by APISIX. In addition, only `cert` and `key` will be used in the SSL object. +To verify the certificate presented by the Upstream, set `tls.verify` to `true`. Leaving it unset keeps the behaviour configured in nginx, which is off unless `proxy_ssl_verify` is turned on. The certificate is checked against the CA certificates in `tls.ca_certs`, or against `ssl_trusted_certificate` from `config.yaml` when `tls.ca_certs` is not set: + +```json +{ + "scheme": "https", + "type": "roundrobin", + "nodes": { + "127.0.0.1:8443": 1 + }, + "tls": { + "verify": true, + "ca_certs": [""] + } +} +``` + To allow Upstream to have a separate connection pool, use `keepalive_pool`. It can be configured by modifying its child fields. Example Configuration: diff --git a/docs/zh/latest/FAQ.md b/docs/zh/latest/FAQ.md index 2d58dec59a6e..ca8fad57f628 100644 --- a/docs/zh/latest/FAQ.md +++ b/docs/zh/latest/FAQ.md @@ -732,7 +732,7 @@ Admin API 中 `/apisix/admin/ssls` 用于管理 SSL 对象，如果 APISIX 需 Upstream 对象中的 `tls.client_cert`、`tls.client_key` 与 `tls.client_cert_id` 用于存放客户端的证书，适用于需要与上游进行 [mTLS 通信](https://apisix.apache.org/zh/docs/apisix/tutorials/client-to-apisix-mtls/)的情况。 -`config.yaml` 中的 `ssl_trusted_certificate` 用于配置一个受信任的根证书。它仅用于在 APISIX 内部访问某些具有自签名证书的服务时，避免提示拒绝对方的 SSL 证书。注意：它不用于信任 APISIX 上游的证书，因为 APISIX 不会验证上游证书的合法性。因此，即使上游使用了无效的 TLS 证书，APISIX 仍然可以与其通信，而无需配置根证书。 +`config.yaml` 中的 `ssl_trusted_certificate` 用于配置一个受信任的根证书。它仅用于在 APISIX 内部访问某些具有自签名证书的服务时，避免提示拒绝对方的 SSL 证书。注意：APISIX 默认不会验证上游证书的合法性，只有在 Upstream 上设置了 `tls.verify` 为 `true` 时才会验证，因此在默认情况下即使上游使用了无效的 TLS 证书，APISIX 仍然可以与其通信。开启 `tls.verify` 后，如果 Upstream 设置了 `tls.ca_certs` 则使用它校验，否则使用 `ssl_trusted_certificate`。 ## 如果在使用 APISIX 过程中遇到问题，我可以在哪里寻求更多帮助？ diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md index a57d30188176..aae200b88f85 100644 --- a/docs/zh/latest/admin-api.md +++ b/docs/zh/latest/admin-api.md @@ -1027,7 +1027,8 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 | tls.client_cert | 否，不能和 `tls.client_cert_id` 一起使用 | https 证书 | 设置跟上游通信时的客户端证书，详细信息请参考下文。 | | | tls.client_key | 否，不能和 `tls.client_cert_id` 一起使用 | https 证书私钥 | 设置跟上游通信时的客户端私钥，详细信息请参考下文。 | | | tls.client_cert_id | 否，不能和 `tls.client_cert`、`tls.client_key` 一起使用 | SSL | 设置引用的 SSL id，详见 [SSL](#ssl)。 | | -| tls.verify |否，目前仅支持 Kafka 上游。 | Boolean | 开启服务器证书验证功能，目前仅支持 Kafka 上游。 | | +| tls.verify | 否 | Boolean | 开启或关闭上游证书校验，不设置时沿用 nginx 的配置，详细信息请参考下文。Kafka 上游同样使用该字段。 | | +| tls.ca_certs | 否 | https 证书数组 | 用于校验上游证书的 CA 证书，设置后将取代 `ssl_trusted_certificate` 中加载的证书，详细信息请参考下文。 | | |keepalive_pool.size | 否 | 辅助 | 动态设置 `keepalive` 指令，详细信息请参考下文。 | |keepalive_pool.idle_timeout | 否 | 辅助 | 动态设置 `keepalive_timeout` 指令，详细信息请参考下文。 | |keepalive_pool.requests | 否 | 辅助 | 动态设置 `keepalive_requests` 指令，详细信息请参考下文。 | @@ -1052,6 +1053,22 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 - `scheme` 可以设置成 `tls`，表示 `TLS over TCP`。 - `tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。他们的格式和 SSL 对象的 `cert` 和 `key` 一样。 - `tls.client_cert_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 client 时才能被引用，否则请求会被 APISIX 拒绝。另外，SSL 对象中只有 `cert` 和 `key` 会被使用。 +- `tls.verify` 设置为 `true` 时校验上游返回的证书；不设置时沿用 nginx 的行为，即只有开启了 `proxy_ssl_verify` 才会校验。校验使用 `tls.ca_certs` 中的 CA 证书，未设置 `tls.ca_certs` 时使用 `config.yaml` 中的 `ssl_trusted_certificate`： + + ```json + { + "scheme": "https", + "type": "roundrobin", + "nodes": { + "127.0.0.1:8443": 1 + }, + "tls": { + "verify": true, + "ca_certs": [""] + } + } + ``` + - `keepalive_pool` 允许 Upstream 有自己单独的连接池。它下属的字段，比如 `requests`，可以用于配置上游连接保持的参数。 Upstream 对象 JSON 配置示例： diff --git a/t/APISIX.pm b/t/APISIX.pm index 677812bf8d64..c1d9a90584e1 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -207,6 +207,7 @@ $grpc_location .= <<_EOC_; grpc_set_header Content-Type application/grpc; grpc_set_header TE trailers; grpc_socket_keepalive on; + grpc_ssl_name \$upstream_host; grpc_pass \$upstream_scheme://apisix_backend; mirror /proxy_mirror_grpc; diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index 35b20e7a6d97..c2ec7e5477c7 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -758,7 +758,7 @@ failed to get ssl cert: ssl id [1] not exits ["127.0.0.1:1983"] = 1, }, tls = { - verify = true + verify = false } }, uri = "/hello" @@ -809,7 +809,7 @@ hello world tls = { client_cert = ssl_cert, client_key = ssl_key, - verify = true + verify = false } }, uri = "/hello" diff --git a/t/node/upstream-mtls2.t b/t/node/upstream-mtls2.t new file mode 100644 index 000000000000..5d0bbdb3da72 --- /dev/null +++ b/t/node/upstream-mtls2.t @@ -0,0 +1,355 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX; + +my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx'; +my $version = eval { `$nginx_binary -V 2>&1` }; + +if ($version !~ m/\/apisix-nginx-module/) { + plan(skip_all => "apisix-nginx-module not installed"); +} else { + plan('no_plan'); +} + +repeat_each(1); +log_level('info'); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->http_config) { + my $http_config = <<'_EOC_'; + +proxy_ssl_trusted_certificate ../../certs/mtls_ca.crt; +proxy_ssl_verify on; + +server { + listen 8777 ssl; + ssl_certificate ../../certs/mtls_server.crt; + ssl_certificate_key ../../certs/mtls_server.key; + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; + + location /hello { + return 200 'ok\n'; + } +} + +_EOC_ + $block->set_value("http_config", $http_config); + } + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: reject a ca_certs entry that isn't a certificate +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:8777"] = 1, + }, + tls = { + client_cert = t.read_file("t/certs/mtls_client.crt"), + client_key = t.read_file("t/certs/mtls_client.key"), + verify = true, + ca_certs = {string.rep("not a certificate", 16)}, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + ngx.status = code + ngx.print(body) + } + } +--- error_code: 400 +--- response_body +{"error_msg":"failed to parse cert: PEM_read_bio_X509_AUX() failed"} + + + +=== TEST 2: verify against the CA trusted by nginx +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:8777"] = 1, + }, + tls = { + client_cert = t.read_file("t/certs/mtls_client.crt"), + client_key = t.read_file("t/certs/mtls_client.key"), + verify = true, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 3: verification passes for the SNI the certificate is issued for +--- request +GET /hello +--- more_headers +host: admin.apisix.dev +--- response_body +ok + + + +=== TEST 4: verification rejects a certificate that doesn't match the SNI +--- request +GET /hello +--- more_headers +host: invalid.apisix.dev +--- error_code: 502 +--- error_log +upstream SSL certificate does not match "invalid.apisix.dev" + + + +=== TEST 5: ca_certs replaces the CA trusted by nginx +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:8777"] = 1, + }, + tls = { + client_cert = t.read_file("t/certs/mtls_client.crt"), + client_key = t.read_file("t/certs/mtls_client.key"), + verify = true, + ca_certs = {t.read_file("t/certs/apisix.crt")}, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 6: the upstream certificate no longer chains to a trusted CA +--- request +GET /hello +--- more_headers +host: admin.apisix.dev +--- error_code: 502 +--- error_log +upstream SSL certificate verify error: (21:unable to verify the first certificate) + + + +=== TEST 7: turning verification off ignores ca_certs +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:8777"] = 1, + }, + tls = { + client_cert = t.read_file("t/certs/mtls_client.crt"), + client_key = t.read_file("t/certs/mtls_client.key"), + verify = false, + ca_certs = {t.read_file("t/certs/apisix.crt")}, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 8: request succeeds despite the untrusted CA +--- request +GET /hello +--- more_headers +host: admin.apisix.dev +--- response_body +ok + + + +=== TEST 9: verify against a CA given in ca_certs +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:8777"] = 1, + }, + tls = { + client_cert = t.read_file("t/certs/mtls_client.crt"), + client_key = t.read_file("t/certs/mtls_client.key"), + verify = true, + ca_certs = {t.read_file("t/certs/mtls_ca.crt")}, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 10: request succeeds +--- request +GET /hello +--- more_headers +host: admin.apisix.dev +--- response_body +ok + + + +=== TEST 11: any CA in ca_certs can anchor the chain +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:8777"] = 1, + }, + tls = { + client_cert = t.read_file("t/certs/mtls_client.crt"), + client_key = t.read_file("t/certs/mtls_client.key"), + verify = true, + ca_certs = { + t.read_file("t/certs/apisix.crt"), + t.read_file("t/certs/mtls_ca.crt"), + }, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 12: request succeeds +--- request +GET /hello +--- more_headers +host: admin.apisix.dev +--- response_body +ok diff --git a/t/node/upstream-tls2.t b/t/node/upstream-tls2.t new file mode 100644 index 000000000000..d7af9c3ec425 --- /dev/null +++ b/t/node/upstream-tls2.t @@ -0,0 +1,433 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX; + +my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx'; +my $version = eval { `$nginx_binary -V 2>&1` }; + +if ($version !~ m/\/apisix-nginx-module/) { + plan(skip_all => "apisix-nginx-module not installed"); +} else { + plan('no_plan'); +} + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); +log_level("info"); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->http_config) { + my $http_config = <<'_EOC_'; + +proxy_ssl_trusted_certificate ../../certs/mtls_ca.crt; +proxy_ssl_verify on; + +server { + listen 8767 ssl; + ssl_certificate ../../certs/mtls_server.crt; + ssl_certificate_key ../../certs/mtls_server.key; + + location /hello { + return 200 'ok\n'; + } +} + +server { + listen 8768 ssl; + ssl_certificate ../../certs/apisix.crt; + ssl_certificate_key ../../certs/apisix.key; + + location /hello { + return 200 'ok\n'; + } +} + +_EOC_ + $block->set_value("http_config", $http_config); + } + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: set tls.verify to true +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "methods": ["GET"], + "upstream": { + "scheme": "https", + "type": "roundrobin", + "nodes": { + "127.0.0.1:8767": 1 + }, + "tls": { + "verify": true + } + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 2: verification passes for the SNI the certificate is issued for +--- request +GET /hello +--- more_headers +host: admin.apisix.dev +--- response_body +ok + + + +=== TEST 3: verification rejects a certificate that doesn't match the SNI +--- request +GET /hello +--- more_headers +host: invalid.apisix.dev +--- error_code: 502 +--- error_log +upstream SSL certificate does not match "invalid.apisix.dev" + + + +=== TEST 4: set tls.verify to false +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "methods": ["GET"], + "upstream": { + "scheme": "https", + "type": "roundrobin", + "nodes": { + "127.0.0.1:8767": 1 + }, + "tls": { + "verify": false + } + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 5: mismatched SNI is accepted once verification is off +--- request +GET /hello +--- more_headers +host: invalid.apisix.dev +--- response_body +ok + + + +=== TEST 6: an upstream the trusted certificate does not cover is rejected +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "methods": ["GET"], + "upstream": { + "scheme": "https", + "type": "roundrobin", + "nodes": { + "127.0.0.1:8768": 1 + }, + "pass_host": "rewrite", + "upstream_host": "test.com", + "tls": { + "verify": true + } + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 7: hit +--- request +GET /hello +--- error_code: 502 +--- error_log +upstream SSL certificate verify error + + + +=== TEST 8: ca_certs alone, without a client certificate +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:8768"] = 1, + }, + pass_host = "rewrite", + upstream_host = "test.com", + tls = { + verify = true, + ca_certs = {t.read_file("t/certs/apisix.crt")}, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 9: hit +--- request +GET /hello +--- response_body +ok + + + +=== TEST 10: a pooled connection does not carry a stale verification policy +--- config + location /t1 { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "methods": ["GET"], + "upstream": { + "scheme": "https", + "type": "roundrobin", + "nodes": { + "127.0.0.1:8768": 1 + }, + "pass_host": "rewrite", + "upstream_host": "test.com", + "tls": { + "verify": false + } + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.sleep(0.5) + ngx.say(body) + } + } + + location /t2 { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "methods": ["GET"], + "upstream": { + "scheme": "https", + "type": "roundrobin", + "nodes": { + "127.0.0.1:8768": 1 + }, + "pass_host": "rewrite", + "upstream_host": "test.com", + "tls": { + "verify": true + } + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.sleep(0.5) + ngx.say(body) + } + } +--- request eval +["GET /t1", "GET /hello", "GET /t2", "GET /hello"] +--- error_code eval +[200, 200, 200, 502] +--- response_body eval +["passed\n", "ok\n", "passed\n", qr/502 Bad Gateway/] +--- no_error_log +[alert] + + + +=== TEST 11: grpcs upstream is verified by default +--- http2 +--- http_config +grpc_ssl_trusted_certificate ../../certs/mtls_ca.crt; +grpc_ssl_verify on; +--- apisix_yaml +routes: + - + id: 1 + uris: + - /helloworld.Greeter/SayHello + methods: [ + POST + ] + upstream: + scheme: grpcs + nodes: + "127.0.0.1:10052": 1 + type: roundrobin +#END +--- exec +grpcurl -import-path ./t/grpc_server_example/proto -proto helloworld.proto -plaintext -d '{"name":"apisix"}' 127.0.0.1:1984 helloworld.Greeter.SayHello 2>&1 || true +--- error_log +upstream SSL certificate verify error + + + +=== TEST 12: tls.verify survives the internal redirect to @grpc_pass +--- http2 +--- http_config +grpc_ssl_trusted_certificate ../../certs/mtls_ca.crt; +grpc_ssl_verify on; +--- apisix_yaml +routes: + - + id: 1 + uris: + - /helloworld.Greeter/SayHello + methods: [ + POST + ] + upstream: + scheme: grpcs + nodes: + "127.0.0.1:10052": 1 + type: roundrobin + tls: + verify: false +#END +--- exec +grpcurl -import-path ./t/grpc_server_example/proto -proto helloworld.proto -plaintext -d '{"name":"apisix"}' 127.0.0.1:1984 helloworld.Greeter.SayHello +--- response_body +{ + "message": "Hello apisix" +} + + + +=== TEST 13: tls.ca_certs survives the internal redirect to @grpc_pass +The upstream serves a certificate that grpc_ssl_trusted_certificate does not +trust, so the request only succeeds if ca_certs reached the handshake. +--- http2 +--- http_config +grpc_ssl_trusted_certificate ../../certs/mtls_ca.crt; +grpc_ssl_verify on; +--- apisix_yaml +routes: + - + id: 1 + uris: + - /helloworld.Greeter/SayHello + methods: [ + POST + ] + upstream: + scheme: grpcs + nodes: + "127.0.0.1:10052": 1 + type: roundrobin + pass_host: rewrite + upstream_host: test.com + tls: + client_cert: "-----BEGIN CERTIFICATE-----\nMIIDUzCCAjugAwIBAgIURw+Rc5FSNUQWdJD+quORtr9KaE8wDQYJKoZIhvcNAQEN\nBQAwWDELMAkGA1UEBhMCY24xEjAQBgNVBAgMCUd1YW5nRG9uZzEPMA0GA1UEBwwG\nWmh1SGFpMRYwFAYDVQQDDA1jYS5hcGlzaXguZGV2MQwwCgYDVQQLDANvcHMwHhcN\nMjIxMjAxMTAxOTU3WhcNNDIwODE4MTAxOTU3WjBOMQswCQYDVQQGEwJjbjESMBAG\nA1UECAwJR3VhbmdEb25nMQ8wDQYDVQQHDAZaaHVIYWkxGjAYBgNVBAMMEWNsaWVu\ndC5hcGlzaXguZGV2MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzypq\nkrsJ8MaqpS0kr2SboE9aRKOJzd6mY3AZLq3tFpio5cK5oIHkQLfeaaLcd4ycFcZw\nFTpxc+Eth6I0X9on+j4tEibc5IpDnRSAQlzHZzlrOG6WxcOza4VmfcrKqj27oodr\noqXv05r/5yIoRrEN9ZXfA8n2OnjhkP+C3Q68L6dBtPpv+e6HaAuw8MvcsEo+MQwu\ncTZyWqWT2UzKVzToW29dHRW+yZGuYNWRh15X09VSvx+E0s+uYKzN0Cyef2C6VtBJ\nKmJ3NtypAiPqw7Ebfov2Ym/zzU9pyWPi3P1mYPMKQqUT/FpZSXm4iSy0a5qTYhkF\nrFdV1YuYYZL5YGl9aQIDAQABox8wHTAbBgNVHREEFDASghBhZG1pbi5hcGlzaXgu\nZGV2MA0GCSqGSIb3DQEBDQUAA4IBAQBepRpwWdckZ6QdL5EuufYwU7p5SIqkVL/+\nN4/l5YSjPoAZf/M6XkZu/PsLI9/kPZN/PX4oxjZSDH14dU9ON3JjxtSrebizcT8V\naQ13TeW9KSv/i5oT6qBmj+V+RF2YCUhyzXdYokOfsSVtSlA1qMdm+cv0vkjYcImV\nl3L9nVHRPq15dY9sbmWEtFBWvOzqNSuQYax+iYG+XEuL9SPaYlwKRC6eS/dbXa1T\nPPWDQad2X/WmhxPzEHvjSl2bsZF1u0GEdKyhXWMOLCLiYIJo15G7bMz8cTUvkDN3\n6WaWBd6bd2g13Ho/OOceARpkR/ND8PU78Y8cq+zHoOSqH+1aly5H\n-----END CERTIFICATE-----\n" + client_key: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAzypqkrsJ8MaqpS0kr2SboE9aRKOJzd6mY3AZLq3tFpio5cK5\noIHkQLfeaaLcd4ycFcZwFTpxc+Eth6I0X9on+j4tEibc5IpDnRSAQlzHZzlrOG6W\nxcOza4VmfcrKqj27oodroqXv05r/5yIoRrEN9ZXfA8n2OnjhkP+C3Q68L6dBtPpv\n+e6HaAuw8MvcsEo+MQwucTZyWqWT2UzKVzToW29dHRW+yZGuYNWRh15X09VSvx+E\n0s+uYKzN0Cyef2C6VtBJKmJ3NtypAiPqw7Ebfov2Ym/zzU9pyWPi3P1mYPMKQqUT\n/FpZSXm4iSy0a5qTYhkFrFdV1YuYYZL5YGl9aQIDAQABAoIBAD7tUG//lnZnsj/4\nJXONaORaFj5ROrOpFPuRemS+egzqFCuuaXpC2lV6RHnr+XHq6SKII1WfagTb+lt/\nvs760jfmGQSxf1mAUidtqcP+sKc/Pr1mgi/SUTawz8AYEFWD6PHmlqBSLTYml+La\nckd+0pGtk49wEnYSb9n+cv640hra9AYpm9LXUFaypiFEu+xJhtyKKWkmiVGrt/X9\n3aG6MuYeZplW8Xq1L6jcHsieTOB3T+UBfG3O0bELBgTVexOQYI9O4Ejl9/n5/8WP\nAbIw7PaAYc7fBkwOGh7/qYUdHnrm5o9MiRT6dPxrVSf0PZVACmA+JoNjCPv0Typf\n3MMkHoECgYEA9+3LYzdP8j9iv1fP5hn5K6XZAobCD1mnzv3my0KmoSMC26XuS71f\nvyBhjL7zMxGEComvVTF9SaNMfMYTU4CwOJQxLAuT69PEzW6oVEeBoscE5hwhjj6o\n/lr5jMbt807J9HnldSpwllfj7JeiTuqRcCu/cwqKQQ1aB3YBZ7h5pZkCgYEA1ejo\nKrR1hN2FMhp4pj0nZ5+Ry2lyIVbN4kIcoteaPhyQ0AQ0zNoi27EBRnleRwVDYECi\nXAFrgJU+laKsg1iPjvinHibrB9G2p1uv3BEh6lPl9wPFlENTOjPkqjR6eVVZGP8e\nVzxYxIo2x/QLDUeOpxySdG4pdhEHGfvmdGmr2FECgYBeknedzhCR4HnjcTSdmlTA\nwI+p9gt6XYG0ZIewCymSl89UR9RBUeh++HQdgw0z8r+CYYjfH3SiLUdU5R2kIZeW\nzXiAS55OO8Z7cnWFSI17sRz+RcbLAr3l4IAGoi9MO0awGftcGSc/QiFwM1s3bSSz\nPAzYbjHUpKot5Gae0PCeKQKBgQCHfkfRBQ2LY2WDHxFc+0+Ca6jF17zbMUioEIhi\n/X5N6XowyPlI6MM7tRrBsQ7unX7X8Rjmfl/ByschsTDk4avNO+NfTfeBtGymBYWX\nN6Lr8sivdkwoZZzKOSSWSzdos48ELlThnO/9Ti706Lg3aSQK5iY+aakJiC+fXdfT\n1TtsgQKBgQDRYvtK/Cpaq0W6wO3I4R75lHGa7zjEr4HA0Kk/FlwS0YveuTh5xqBj\nwQz2YyuQQfJfJs7kbWOITBT3vuBJ8F+pktL2Xq5p7/ooIXOGS8Ib4/JAS1C/wb+t\nuJHGva12bZ4uizxdL2Q0/n9ziYTiMc/MMh/56o4Je8RMdOMT5lTsRQ==\n-----END RSA PRIVATE KEY-----\n" + verify: true + ca_certs: + - "-----BEGIN CERTIFICATE-----\nMIIEojCCAwqgAwIBAgIJAK253pMhgCkxMA0GCSqGSIb3DQEBCwUAMFYxCzAJBgNV\nBAYTAkNOMRIwEAYDVQQIDAlHdWFuZ0RvbmcxDzANBgNVBAcMBlpodUhhaTEPMA0G\nA1UECgwGaXJlc3R5MREwDwYDVQQDDAh0ZXN0LmNvbTAgFw0xOTA2MjQyMjE4MDVa\nGA8yMTE5MDUzMTIyMTgwNVowVjELMAkGA1UEBhMCQ04xEjAQBgNVBAgMCUd1YW5n\nRG9uZzEPMA0GA1UEBwwGWmh1SGFpMQ8wDQYDVQQKDAZpcmVzdHkxETAPBgNVBAMM\nCHRlc3QuY29tMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAyCM0rqJe\ncvgnCfOw4fATotPwk5Ba0gC2YvIrO+gSbQkyxXF5jhZB3W6BkWUWR4oNFLLSqcVb\nVDPitz/Mt46Mo8amuS6zTbQetGnBARzPLtmVhJfoeLj0efMiOepOSZflj9Ob4yKR\n2bGdEFOdHPjm+4ggXU9jMKeLqdVvxll/JiVFBW5smPtW1Oc/BV5terhscJdOgmRr\nabf9xiIis9/qVYfyGn52u9452V0owUuwP7nZ01jt6iMWEGeQU6mwPENgvj1olji2\nWjdG2UwpUVp3jp3l7j1ekQ6mI0F7yI+LeHzfUwiyVt1TmtMWn1ztk6FfLRqwJWR/\nEvm95vnfS3Le4S2ky3XAgn2UnCMyej3wDN6qHR1onpRVeXhrBajbCRDRBMwaNw/1\n/3Uvza8QKK10PzQR6OcQ0xo9psMkd9j9ts/dTuo2fzaqpIfyUbPST4GdqNG9NyIh\n/B9g26/0EWcjyO7mYVkaycrtLMaXm1u9jyRmcQQI1cGrGwyXbrieNp63AgMBAAGj\ncTBvMB0GA1UdDgQWBBSZtSvV8mBwl0bpkvFtgyiOUUcbszAfBgNVHSMEGDAWgBSZ\ntSvV8mBwl0bpkvFtgyiOUUcbszAMBgNVHRMEBTADAQH/MB8GA1UdEQQYMBaCCHRl\nc3QuY29tggoqLnRlc3QuY29tMA0GCSqGSIb3DQEBCwUAA4IBgQAHGEul/x7ViVgC\ntC8CbXEslYEkj1XVr2Y4hXZXAXKd3W7V3TC8rqWWBbr6L/tsSVFt126V5WyRmOaY\n1A5pju8VhnkhYxYfZALQxJN2tZPFVeME9iGJ9BE1wPtpMgITX8Rt9kbNlENfAgOl\nPYzrUZN1YUQjX+X8t8/1VkSmyZysr6ngJ46/M8F16gfYXc9zFj846Z9VST0zCKob\nrJs3GtHOkS9zGGldqKKCj+Awl0jvTstI4qtS1ED92tcnJh5j/SSXCAB5FgnpKZWy\nhme45nBQj86rJ8FhN+/aQ9H9/2Ib6Q4wbpaIvf4lQdLUEcWAeZGW6Rk0JURwEog1\n7/mMgkapDglgeFx9f/XztSTrkHTaX4Obr+nYrZ2V4KOB4llZnK5GeNjDrOOJDk2y\nIJFgBOZJWyS93dQfuKEj42hA79MuX64lMSCVQSjX+ipR289GQZqFrIhiJxLyA+Ve\nU/OOcSRr39Kuis/JJ+DkgHYa/PWHZhnJQBxcqXXk1bJGw9BNbhM=\n-----END CERTIFICATE-----\n" +#END +--- exec +grpcurl -import-path ./t/grpc_server_example/proto -proto helloworld.proto -plaintext -d '{"name":"apisix"}' 127.0.0.1:1984 helloworld.Greeter.SayHello +--- response_body +{ + "message": "Hello apisix" +}