diff --git a/apisix/discovery/consul/client.lua b/apisix/discovery/consul/client.lua index a86f6356c3b9..3146dd7f917f 100644 --- a/apisix/discovery/consul/client.lua +++ b/apisix/discovery/consul/client.lua @@ -116,6 +116,16 @@ end -- ─── resty.consul options ───────────────────────────────────────────── +local function set_tls_opts(opts, consul_server) + if consul_server.ssl then + opts.ssl = true + opts.ssl_verify = consul_server.ssl_verify + opts.sni_host = consul_server.host + end + return opts +end + + local function get_opts(consul_server, is_catalog) local opts = { host = consul_server.host, @@ -126,6 +136,7 @@ local function get_opts(consul_server, is_catalog) token = consul_server.token, } } + set_tls_opts(opts, consul_server) if not consul_server.keepalive then return opts end @@ -251,15 +262,24 @@ function _M.format_consul_params(consul_conf) local consul_server_list = core.table.new(0, #servers) for _, v in pairs(servers) do - local scheme, host, port, path = unpack(http.parse_uri(nil, v)) - if scheme ~= "http" then - return nil, "only support consul http schema address, eg: http://address:port" + local parsed, err = http.parse_uri(nil, v) + if not parsed then + return nil, "invalid consul server address: " .. err + .. ", the valid format: http://address:port or https://address:port" + end + local scheme, host, port, path = unpack(parsed) + if scheme ~= "http" and scheme ~= "https" then + return nil, "only support consul http or https schema address, " + .. "eg: http://address:port or https://address:port" elseif path ~= "/" or core.string.has_suffix(v, '/') then - return nil, "invalid consul server address, the valid format: http://address:port" + return nil, "invalid consul server address, the valid format: " + .. "http://address:port or https://address:port" end core.table.insert(consul_server_list, { host = host, port = port, + ssl = scheme == "https", + ssl_verify = consul_conf.ssl_verify ~= false, token = consul_conf.token, connect_timeout = consul_conf.timeout.connect, read_timeout = consul_conf.timeout.read, @@ -302,7 +322,7 @@ function _M.fetch_services_from_server(consul_server, options) local preserve_metadata = options.preserve_metadata or false local key_builder = options.key_builder - local consul_client = resty_consul:new({ + local consul_client = resty_consul:new(set_tls_opts({ host = consul_server.host, port = consul_server.port, connect_timeout = consul_server.connect_timeout, @@ -310,7 +330,7 @@ function _M.fetch_services_from_server(consul_server, options) default_args = { token = consul_server.token } - }) + }, consul_server)) -- fetch catalog local catalog_success, catalog_res, catalog_err = pcall(function() diff --git a/apisix/discovery/consul/schema.lua b/apisix/discovery/consul/schema.lua index 57080ee7b717..0462974023b9 100644 --- a/apisix/discovery/consul/schema.lua +++ b/apisix/discovery/consul/schema.lua @@ -30,6 +30,7 @@ return { default = "64m", }, token = {type = "string", default = ""}, + ssl_verify = {type = "boolean", default = true}, fetch_interval = {type = "integer", minimum = 1, default = 3}, keepalive = { type = "boolean", diff --git a/conf/config.yaml.example b/conf/config.yaml.example index ded74f6ceb20..d07a8d0563b9 100644 --- a/conf/config.yaml.example +++ b/conf/config.yaml.example @@ -439,9 +439,10 @@ nginx_config: # Config for render the template to generate n # path: "logs/consul_kv.dump" # Location of the dump file. # expire: 2592000 # Specify the expiration time of the dump file in units of seconds. # consul: # Consul -# servers: # Consul address(es) +# servers: # Consul address(es), http or https # - "http://127.0.0.1:8500" # - "http://127.0.0.1:8600" +# ssl_verify: true # Default true. Verify the certificate of https servers. # skip_services: # Skip services during service discovery. # - "service_a" # timeout: diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md index fd6758bc9abb..a00c439020c5 100644 --- a/docs/en/latest/discovery/consul.md +++ b/docs/en/latest/discovery/consul.md @@ -38,6 +38,7 @@ discovery: - "http://127.0.0.1:8500" # `http://127.0.0.1:8500` and `http://127.0.0.1:8600` are different clusters - "http://127.0.0.1:8600" # `consul` service is default skip service token: "..." # if your consul cluster has enabled acl access control, you need to specify the token + ssl_verify: true # default true, verify the certificate of https consul servers skip_services: # if you need to skip special services - "service_a" timeout: @@ -69,6 +70,20 @@ discovery: - "http://127.0.0.1:8500" ``` +Each server address uses the `http` or `https` scheme, and the two can be mixed in one list. An `https` address without a port connects to port 443. By default APISIX verifies the certificate of an `https` Consul server against the host in the address, so configure the CA bundle that signed it with `apisix.ssl.ssl_trusted_certificate`: + +```yaml +apisix: + ssl: + ssl_trusted_certificate: /path/to/consul-ca.crt +discovery: + consul: + servers: + - "https://consul.example.com:8501" +``` + +Set `ssl_verify` to `false` to skip the certificate verification of `https` servers, for example while testing against a self-signed certificate. It has no effect on `http` servers. + The `keepalive` has two optional values: - `true`, default and recommend value, use the long pull way to query consul servers diff --git a/t/discovery/consul-tls.t b/t/discovery/consul-tls.t new file mode 100644 index 000000000000..3dd9308094c8 --- /dev/null +++ b/t/discovery/consul-tls.t @@ -0,0 +1,244 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +no_root_location(); +no_shuffle(); + + +add_block_preprocessor(sub { + my ($block) = @_; + + # a TLS endpoint fronting the consul agent on 8500, its certificate + # (localhost) is signed by apisix.crt; the trust store is the http level + # lua_ssl_trusted_certificate, as apisix.ssl.ssl_trusted_certificate renders it + my $http_config = $block->http_config // <<_EOC_; + lua_ssl_trusted_certificate ../../certs/apisix.crt; + + server { + listen 18501 ssl; + ssl_certificate ../../certs/localhost_slapd_cert.pem; + ssl_certificate_key ../../certs/localhost_slapd_key.pem; + + location / { + proxy_pass http://127.0.0.1:8500; + proxy_read_timeout 70s; + } + } + + server { + listen 30511; + + location /hello { + content_by_lua_block { + ngx.say("server 1") + } + } + } +_EOC_ + + $block->set_value("http_config", $http_config); +}); + + +sub main::yaml_config { + my ($servers, $keepalive, $ssl_verify) = @_; + my $ssl_verify_conf = defined $ssl_verify ? " ssl_verify: $ssl_verify\n" : ""; + return <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "$servers" + timeout: + connect: 1000 + read: 1000 + wait: 60 + weight: 1 + fetch_interval: 1 + keepalive: $keepalive +$ssl_verify_conf +_EOC_ +} + +our $apisix_yaml = <<_EOC_; +routes: + - + uri: /hello + upstream: + service_name: service_tls + discovery_type: consul + type: roundrobin +#END +_EOC_ + + +run_tests(); + +__DATA__ + +=== TEST 1: prepare consul catalog register nodes +--- config +location /v1/agent { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +"PUT /v1/agent/service/register\n" . "{\"ID\":\"service_tls1\",\"Name\":\"service_tls\",\"Address\":\"127.0.0.1\",\"Port\":30511}" +--- error_code: 200 + + + +=== TEST 2: https consul server (long connect type) +--- yaml_config eval: main::yaml_config("https://localhost:18501", "true") +--- apisix_yaml eval: $::apisix_yaml +--- request +GET /hello +--- response_body +server 1 +--- no_error_log +[error] + + + +=== TEST 3: https consul server (short connect type) +--- yaml_config eval: main::yaml_config("https://localhost:18501", "false") +--- apisix_yaml eval: $::apisix_yaml +--- request +GET /hello +--- response_body +server 1 +--- no_error_log +[error] + + + +=== TEST 4: the certificate of an https consul server is verified by default +--- http_config + # same endpoint as above, but its certificate is not signed by the trusted CA + lua_ssl_trusted_certificate ../../certs/mtls_ca.crt; + + server { + listen 18501 ssl; + ssl_certificate ../../certs/localhost_slapd_cert.pem; + ssl_certificate_key ../../certs/localhost_slapd_key.pem; + + location / { + proxy_pass http://127.0.0.1:8500; + } + } +--- yaml_config eval: main::yaml_config("https://localhost:18501", "false") +--- apisix_yaml eval: $::apisix_yaml +--- request +GET /hello +--- error_code: 503 +--- error_log eval +[ + qr/connect consul: https:\/\/localhost:18501\/v1 .* with error: \d+: unable to (get local issuer|verify the first) certificate/, + qr/consul service not found: default\/service_tls/, +] + + + +=== TEST 5: ssl_verify false accepts a certificate the DP does not trust +--- http_config + # same endpoint as above, but its certificate is not signed by the trusted CA + lua_ssl_trusted_certificate ../../certs/mtls_ca.crt; + + server { + listen 18501 ssl; + ssl_certificate ../../certs/localhost_slapd_cert.pem; + ssl_certificate_key ../../certs/localhost_slapd_key.pem; + + location / { + proxy_pass http://127.0.0.1:8500; + } + } + + server { + listen 30511; + + location /hello { + content_by_lua_block { + ngx.say("server 1") + } + } + } +--- yaml_config eval: main::yaml_config("https://localhost:18501", "false", "false") +--- apisix_yaml eval: $::apisix_yaml +--- request +GET /hello +--- response_body +server 1 +--- no_error_log +[error] + + + +=== TEST 6: scheme per server address, other schemes are still rejected +--- config + location /t { + content_by_lua_block { + local client = require("apisix.discovery.consul.client") + local timeout = {connect = 1000, read = 1000, wait = 60} + + local servers, err = client.format_consul_params({ + servers = {"http://127.0.0.1:8500", "https://consul.local"}, + timeout = timeout, + }) + for _, s in ipairs(servers or {}) do + ngx.say(s.host, ":", s.port, " ssl: ", s.ssl, " ssl_verify: ", s.ssl_verify) + end + + servers, err = client.format_consul_params({ + servers = {"https://consul.local:8501"}, + timeout = timeout, + ssl_verify = false, + }) + ngx.say(servers[1].host, ":", servers[1].port, " ssl_verify: ", servers[1].ssl_verify) + + servers, err = client.format_consul_params({ + servers = {"https://127.0.0.1:8501", "tcp://127.0.0.1:8500"}, + timeout = timeout, + }) + ngx.say(err) + } + } +--- request +GET /t +--- response_body +127.0.0.1:8500 ssl: false ssl_verify: true +consul.local:443 ssl: true ssl_verify: true +consul.local:8501 ssl_verify: false +invalid consul server address: bad uri: tcp://127.0.0.1:8500, the valid format: http://address:port or https://address:port + + + +=== TEST 7: clean nodes +--- config +location /v1/agent { + proxy_pass http://127.0.0.1:8500; +} +--- request +PUT /v1/agent/service/deregister/service_tls1 +--- error_code: 200