From 95152351979c70471c90d6ec9d940dea3a531e8b Mon Sep 17 00:00:00 2001 From: karthick udayakumar Date: Thu, 23 Apr 2026 16:13:11 -0400 Subject: [PATCH] sni tcp routing ai-assisted=yes TNZ-81099 --- README.md | 1 + cats_suite_helpers/cats_suite_helpers.go | 117 +++++++++++- helpers/config/config.go | 3 + helpers/config/config_struct.go | 21 +++ helpers/config/config_test.go | 6 + helpers/skip_messages/skip_messages.go | 2 + tcp_routing/tcp_sni_routing.go | 230 +++++++++++++++++++++++ 7 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 tcp_routing/tcp_sni_routing.go diff --git a/README.md b/README.md index f5ba56cb9..21965e2f8 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ include_app_syslog_tcp * `include_sso`: Flag to include the services tests that integrate with Single Sign On. * `include_tasks`: Flag to include the v3 task tests. `include_v3` must also be set for tests to run. The CC API task_creation feature flag must be enabled for these tests to pass. * `include_tcp_routing`: Flag to include the TCP Routing tests. These tests are equivalent to the [TCP Routing tests](https://github.com/cloudfoundry/routing-acceptance-tests/blob/master/tcp_routing/tcp_routing_test.go) from the Routing Acceptance Tests. +* `include_tcp_sni_routing`: Flag to include the SNI TCP Routing tests (TNZ-81099). These tests map multiple apps to the same external TCP port differentiated by SNI hostname (e.g. `cf map-route appB tcp. --port 18001 --hostname app-b`) and verify that a TLS client with the matching `ServerName` reaches the expected backend. Requires: (1) wildcard DNS so `*.` resolves to the TCP router load balancer; (2) `cf-tcp-router` deployed with frontend TLS termination enabled and a certificate that covers `*.` (self-signed is acceptable — the tests skip cert verification); (3) an org/space quota permitting at least two reserved route ports. * `tcp_domain`: Domain that will be used for apps with TCP routes * `include_user_provided_services`: Flag to include test for user-provided services. * `include_v3`: Flag to include tests for the v3 API. diff --git a/cats_suite_helpers/cats_suite_helpers.go b/cats_suite_helpers/cats_suite_helpers.go index a9f095315..942a3cf01 100644 --- a/cats_suite_helpers/cats_suite_helpers.go +++ b/cats_suite_helpers/cats_suite_helpers.go @@ -2,9 +2,12 @@ package cats_suite_helpers import ( "bytes" + "crypto/tls" "fmt" + "math/rand" "net" "regexp" + "strconv" "strings" "time" @@ -202,6 +205,17 @@ func TCPRoutingDescribe(description string, callback func()) bool { }) } +func TCPSNIRoutingDescribe(description string, callback func()) bool { + return Describe("[tcp sni routing]", func() { + BeforeEach(func() { + if !Config.GetIncludeTCPSNIRouting() { + Skip(skip_messages.SkipTCPSNIRoutingMessage) + } + }) + Describe(description, callback) + }) +} + func RoutingIsolationSegmentsDescribe(description string, callback func()) bool { return Describe("[routing_isolation_segments]", func() { BeforeEach(func() { @@ -475,8 +489,22 @@ func GetNServerResponses(n int, domainName, externalPort1 string) ([]string, err return responses, nil } +func GetTCPPort() int { + start := Config.GetTCPPortRangeStart() + end := Config.GetTCPPortRangeEnd() + if start > 0 && end >= start { + return rand.Intn(end-start+1) + start + } + return 0 +} + func MapTCPRoute(appName, domainName string) string { - createRouteSession := cf.Cf("map-route", appName, domainName).Wait() + port := GetTCPPort() + args := []string{"map-route", appName, domainName} + if port != 0 { + args = append(args, "--port", strconv.Itoa(port)) + } + createRouteSession := cf.Cf(args...).Wait() Expect(createRouteSession).To(Exit(0)) r := regexp.MustCompile(fmt.Sprintf(`.+%s:(\d+).+`, domainName)) @@ -529,3 +557,90 @@ func SendAndReceive(addr string, externalPort string) (string, error) { return string(buff[:i]), nil } + +// MapTCPRouteWithHostname maps a TCP route on `domainName` with the given hostname. +// If port == 0, a new external port is allocated by the TCP router; the chosen port +// is parsed from the CLI output and returned. +// If port != 0, the supplied port is reused so callers can map multiple apps on the +// same external port, differentiated by SNI hostname. +func MapTCPRouteWithHostname(appName, domainName, hostname string, port int) int { + if port == 0 { + port = GetTCPPort() + } + args := []string{"map-route", appName, domainName, "--hostname", hostname} + if port != 0 { + args = append(args, "--port", strconv.Itoa(port)) + } + + session := cf.Cf(args...).Wait() + Expect(session).To(Exit(0)) + + if port != 0 { + return port + } + + r := regexp.MustCompile(fmt.Sprintf(`.+%s:(\d+).+`, domainName)) + matches := r.FindStringSubmatch(string(session.Out.Contents())) + Expect(matches).To(HaveLen(2), "expected map-route output to contain an external port") + + chosen, err := strconv.Atoi(matches[1]) + Expect(err).ToNot(HaveOccurred()) + + return chosen +} + +// SendAndReceiveTLS opens a TLS connection to addr:port using the given SNI serverName, +// writes a small payload, and returns the server's response. insecureSkipVerify should +// typically be true in CATs since we only validate routing, not cert trust. +func SendAndReceiveTLS(addr string, port int, serverName string, insecureSkipVerify bool) (string, error) { + address := fmt.Sprintf("%s:%d", addr, port) + + dialer := &net.Dialer{Timeout: 10 * time.Second} + conn, err := tls.DialWithDialer(dialer, "tcp", address, &tls.Config{ + ServerName: serverName, + InsecureSkipVerify: insecureSkipVerify, + }) + if err != nil { + return "", err + } + defer conn.Close() + + if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + return "", err + } + + message := []byte(fmt.Sprintf("Time is %d", time.Now().Nanosecond())) + if _, err = conn.Write(message); err != nil { + return "", err + } + + // mirror SendAndReceive: small pause to let the server respond before the read. + time.Sleep(100 * time.Millisecond) + + buff := make([]byte, 1024) + n, err := conn.Read(buff) + if err != nil { + return "", err + } + + i := n + if j := bytes.IndexByte(buff[:n], 0); j > 0 { + i = j + } + + return string(buff[:i]), nil +} + +// GetNTLSResponses performs n TLS SNI round-trips against addr:port using the given +// serverName and returns all responses in order. +func GetNTLSResponses(n int, addr string, port int, serverName string, insecureSkipVerify bool) ([]string, error) { + responses := make([]string, 0, n) + for i := 0; i < n; i++ { + resp, err := SendAndReceiveTLS(addr, port, serverName, insecureSkipVerify) + if err != nil { + return nil, err + } + responses = append(responses, resp) + } + return responses, nil +} diff --git a/helpers/config/config.go b/helpers/config/config.go index 3a6fcbe19..b068acb08 100644 --- a/helpers/config/config.go +++ b/helpers/config/config.go @@ -36,6 +36,9 @@ type CatsConfig interface { GetIncludeTCPIsolationSegments() bool GetIncludeHTTP2Routing() bool GetIncludeTCPRouting() bool + GetIncludeTCPSNIRouting() bool + GetTCPPortRangeStart() int + GetTCPPortRangeEnd() int GetIncludeWindows() bool GetIncludeVolumeServices() bool GetShouldKeepUser() bool diff --git a/helpers/config/config_struct.go b/helpers/config/config_struct.go index a1540acca..870b48cc0 100644 --- a/helpers/config/config_struct.go +++ b/helpers/config/config_struct.go @@ -99,6 +99,9 @@ type config struct { IncludeTCPIsolationSegments *bool `json:"include_tcp_isolation_segments"` IncludeHTTP2Routing *bool `json:"include_http2_routing"` IncludeTCPRouting *bool `json:"include_tcp_routing"` + IncludeTCPSNIRouting *bool `json:"include_tcp_sni_routing"` + TCPPortRangeStart *int `json:"tcp_port_range_start"` + TCPPortRangeEnd *int `json:"tcp_port_range_end"` IncludeTasks *bool `json:"include_tasks"` IncludeV3 *bool `json:"include_v3"` IncludeVolumeServices *bool `json:"include_volume_services"` @@ -210,6 +213,9 @@ func getDefaults() config { defaults.IncludeServiceCredentialBindingRotation = ptrToBool(false) defaults.IncludeHTTP2Routing = ptrToBool(false) defaults.IncludeTCPRouting = ptrToBool(false) + defaults.IncludeTCPSNIRouting = ptrToBool(false) + defaults.TCPPortRangeStart = ptrToInt(0) + defaults.TCPPortRangeEnd = ptrToInt(0) defaults.IncludeVolumeServices = ptrToBool(false) defaults.IncludeIPv6 = ptrToBool(false) @@ -508,6 +514,9 @@ func validateConfig(config *config) error { if config.IncludeTCPRouting == nil { errs = errors.Join(errs, fmt.Errorf("* 'include_tcp_routing' must not be null")) } + if config.IncludeTCPSNIRouting == nil { + errs = errors.Join(errs, fmt.Errorf("* 'include_tcp_sni_routing' must not be null")) + } if config.IncludeV3 == nil { errs = errors.Join(errs, fmt.Errorf("* 'include_v3' must not be null")) } @@ -1068,6 +1077,18 @@ func (c *config) GetIncludeTCPRouting() bool { return *c.IncludeTCPRouting } +func (c *config) GetIncludeTCPSNIRouting() bool { + return *c.IncludeTCPSNIRouting +} + +func (c *config) GetTCPPortRangeStart() int { + return *c.TCPPortRangeStart +} + +func (c *config) GetTCPPortRangeEnd() int { + return *c.TCPPortRangeEnd +} + func (c *config) GetIncludeV3() bool { return *c.IncludeV3 } diff --git a/helpers/config/config_test.go b/helpers/config/config_test.go index ac729c227..0fe3f5cd7 100644 --- a/helpers/config/config_test.go +++ b/helpers/config/config_test.go @@ -84,6 +84,7 @@ type testConfig struct { IncludeTCPIsolationSegments *bool `json:"include_tcp_isolation_segments,omitempty"` IncludeHTTP2Routing *bool `json:"include_http2_routing,omitempty"` IncludeTCPRouting *bool `json:"include_tcp_routing,omitempty"` + IncludeTCPSNIRouting *bool `json:"include_tcp_sni_routing,omitempty"` IncludeTasks *bool `json:"include_tasks,omitempty"` IncludeV3 *bool `json:"include_v3,omitempty"` IncludeVolumeServices *bool `json:"include_volume_services,omitempty"` @@ -175,6 +176,7 @@ type nullConfig struct { IncludeRoutingIsolationSegments *bool `json:"include_routing_isolation_segments"` IncludeHTTP2Routing *bool `json:"include_http2_routing"` IncludeTCPRouting *bool `json:"include_tcp_routing"` + IncludeTCPSNIRouting *bool `json:"include_tcp_sni_routing"` IncludeServiceDiscovery *bool `json:"include_service_discovery"` IncludeVolumeServices *bool `json:"include_volume_services"` IncludeTCPIsolationSegments *bool `json:"include_tcp_isolation_segments"` @@ -307,6 +309,7 @@ var _ = Describe("Config", func() { Expect(config.GetIncludeServiceInstanceSharing()).To(BeFalse()) Expect(config.GetIncludeHTTP2Routing()).To(BeFalse()) Expect(config.GetIncludeTCPRouting()).To(BeFalse()) + Expect(config.GetIncludeTCPSNIRouting()).To(BeFalse()) Expect(config.GetIncludeVolumeServices()).To(BeFalse()) Expect(config.GetIncludeWindows()).To(BeFalse()) @@ -433,6 +436,7 @@ var _ = Describe("Config", func() { Expect(err.Error()).To(ContainSubstring("'include_tasks' must not be null")) Expect(err.Error()).To(ContainSubstring("'include_http2_routing' must not be null")) Expect(err.Error()).To(ContainSubstring("'include_tcp_routing' must not be null")) + Expect(err.Error()).To(ContainSubstring("'include_tcp_sni_routing' must not be null")) Expect(err.Error()).To(ContainSubstring("'include_v3' must not be null")) Expect(err.Error()).To(ContainSubstring("'include_zipkin' must not be null")) Expect(err.Error()).To(ContainSubstring("'include_isolation_segments' must not be null")) @@ -496,6 +500,7 @@ var _ = Describe("Config", func() { testCfg.IncludeTCPIsolationSegments = ptrToBool(true) testCfg.IncludeHTTP2Routing = ptrToBool(true) testCfg.IncludeTCPRouting = ptrToBool(true) + testCfg.IncludeTCPSNIRouting = ptrToBool(true) testCfg.IncludeTasks = ptrToBool(true) testCfg.IncludeV3 = ptrToBool(false) testCfg.IncludeVolumeServices = ptrToBool(true) @@ -561,6 +566,7 @@ var _ = Describe("Config", func() { Expect(config.GetIncludeTCPIsolationSegments()).To(BeTrue()) Expect(config.GetIncludeHTTP2Routing()).To(BeTrue()) Expect(config.GetIncludeTCPRouting()).To(BeTrue()) + Expect(config.GetIncludeTCPSNIRouting()).To(BeTrue()) Expect(config.GetIncludeTasks()).To(BeTrue()) Expect(config.GetIncludeV3()).To(BeFalse()) Expect(config.GetIncludeVolumeServices()).To(BeTrue()) diff --git a/helpers/skip_messages/skip_messages.go b/helpers/skip_messages/skip_messages.go index 0fba8322b..cdb0bfdd0 100644 --- a/helpers/skip_messages/skip_messages.go +++ b/helpers/skip_messages/skip_messages.go @@ -30,6 +30,8 @@ NOTE: Ensure that route services are enabled on your platform before running thi const SkipRoutingMessage = `Skipping this test because config.IncludeRouting is set to 'false'.` const SkipHTTP2RoutingMessage = `Skipping this test because config.IncludeHTTP2Routing is set to 'false'.` const SkipTCPRoutingMessage = `Skipping this test because config.IncludeTCPRouting is set to 'false'.` +const SkipTCPSNIRoutingMessage = `Skipping this test because config.IncludeTCPSNIRouting is set to 'false'. +NOTE: Ensure that TCP router frontend TLS termination is enabled and *. DNS resolves to the TCP router before enabling this test.` const SkipSecurityGroupsMessage = `Skipping this test because config.IncludeSecurityGroups is set to 'false'. NOTE: Ensure that your platform restricts internal network traffic by default in order to run this test.` const SkipCommaDelimitedSecurityGroupsMessage = `Skipping this test because config.CommaDelimitedASGsEnabled is set to 'false'. diff --git a/tcp_routing/tcp_sni_routing.go b/tcp_routing/tcp_sni_routing.go new file mode 100644 index 000000000..c199d76b2 --- /dev/null +++ b/tcp_routing/tcp_sni_routing.go @@ -0,0 +1,230 @@ +package tcp_routing + +import ( + "fmt" + "path/filepath" + + . "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/assets" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name" + "github.com/cloudfoundry/cf-test-helpers/v2/cf" + "github.com/cloudfoundry/cf-test-helpers/v2/workflowhelpers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gexec" +) + +// SNI TCP routing exercises the TNZ-81099 feature: multiple apps share the +// same external TCP port and are differentiated by an SNI hostname on the +// client TLS ClientHello. The TCP router terminates frontend TLS and forwards +// the plaintext stream to the correct backend based on SNI. +var _ = TCPSNIRoutingDescribe("TCP SNI Routing", func() { + var domainName string + + BeforeEach(func() { + domainName = Config.GetTCPDomain() + workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() { + routerGroupOutput := string(cf.Cf("router-groups").Wait().Out.Contents()) + Expect(routerGroupOutput).To( + MatchRegexp(fmt.Sprintf("%s\\s+tcp", DefaultRouterGroupName)), + fmt.Sprintf("Router group %s of type tcp doesn't exist", DefaultRouterGroupName), + ) + + Expect(cf.Cf("create-shared-domain", + domainName, + "--router-group", DefaultRouterGroupName, + ).Wait()).To(Exit()) + }) + }) + + Context("two apps sharing a port via SNI", func() { + var ( + appA, appB string + hostA = "app-a" + hostB = "app-b" + serverIdA = "server-a" + serverIdB = "server-b" + externalPort int + tcpDropletReceiver = assets.NewAssets().TCPListener + ) + + BeforeEach(func() { + appA = random_name.CATSRandomName("APP-A") + appB = random_name.CATSRandomName("APP-B") + + pushArgs := func(appName, serverId string) []string { + return []string{ + "push", + "--no-route", + "--no-start", + appName, + "-p", tcpDropletReceiver, + "-b", Config.GetGoBuildpackName(), + "-m", DEFAULT_MEMORY_LIMIT, + "-f", filepath.Join(tcpDropletReceiver, "manifest.yml"), + "-c", fmt.Sprintf("tcp-listener --serverId=%s", serverId), + } + } + + Expect(cf.Cf(pushArgs(appA, serverIdA)...).Wait()).To(Exit(0)) + Expect(cf.Cf(pushArgs(appB, serverIdB)...).Wait()).To(Exit(0)) + + // Allocate a port for appA by mapping its SNI route without an explicit port. + externalPort = MapTCPRouteWithHostname(appA, domainName, hostA, 0) + + // Reuse the same port for appB, differentiated only by SNI hostname. + MapTCPRouteWithHostname(appB, domainName, hostB, externalPort) + + Expect(cf.Cf("start", appA).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + Expect(cf.Cf("start", appB).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + }) + + AfterEach(func() { + app_helpers.AppReport(appA) + app_helpers.AppReport(appB) + Eventually(cf.Cf("delete", appA, "-f", "-r")).Should(Exit(0)) + Eventually(cf.Cf("delete", appB, "-f", "-r")).Should(Exit(0)) + }) + + It("routes to the correct backend based on the SNI hostname", func() { + sniA := fmt.Sprintf("%s.%s", hostA, domainName) + sniB := fmt.Sprintf("%s.%s", hostB, domainName) + + respA, err := SendAndReceiveTLS(domainName, externalPort, sniA, true) + Expect(err).ToNot(HaveOccurred()) + Expect(respA).To(ContainSubstring(serverIdA)) + + respB, err := SendAndReceiveTLS(domainName, externalPort, sniB, true) + Expect(err).ToNot(HaveOccurred()) + Expect(respB).To(ContainSubstring(serverIdB)) + }) + + It("does not cross-talk: repeated requests for one hostname only hit that app", func() { + sniA := fmt.Sprintf("%s.%s", hostA, domainName) + + responses, err := GetNTLSResponses(10, domainName, externalPort, sniA, true) + Expect(err).ToNot(HaveOccurred()) + Expect(responses).To(HaveLen(10)) + for _, r := range responses { + Expect(r).To(ContainSubstring(serverIdA)) + Expect(r).ToNot(ContainSubstring(serverIdB)) + } + }) + }) + + // Two apps mapped to the same port AND the same SNI hostname. HAProxy places both + // backends in a single pool and round-robins across them. This is the SNI analogue + // of the port-only "maps single external port to both applications" test. + Context("two apps sharing the same port and same SNI hostname", func() { + var ( + appA, appB string + sharedHost = "shared-sni" + serverIdA = "server-shared-a" + serverIdB = "server-shared-b" + externalPort int + tcpDropletReceiver = assets.NewAssets().TCPListener + ) + + BeforeEach(func() { + appA = random_name.CATSRandomName("APP-A") + appB = random_name.CATSRandomName("APP-B") + + pushArgs := func(appName, serverId string) []string { + return []string{ + "push", + "--no-route", + "--no-start", + appName, + "-p", tcpDropletReceiver, + "-b", Config.GetGoBuildpackName(), + "-m", DEFAULT_MEMORY_LIMIT, + "-f", filepath.Join(tcpDropletReceiver, "manifest.yml"), + "-c", fmt.Sprintf("tcp-listener --serverId=%s", serverId), + } + } + + Expect(cf.Cf(pushArgs(appA, serverIdA)...).Wait()).To(Exit(0)) + Expect(cf.Cf(pushArgs(appB, serverIdB)...).Wait()).To(Exit(0)) + + externalPort = MapTCPRouteWithHostname(appA, domainName, sharedHost, 0) + MapTCPRouteWithHostname(appB, domainName, sharedHost, externalPort) + + Expect(cf.Cf("start", appA).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + Expect(cf.Cf("start", appB).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + }) + + AfterEach(func() { + app_helpers.AppReport(appA) + app_helpers.AppReport(appB) + Eventually(cf.Cf("delete-route", domainName, + "--port", fmt.Sprintf("%d", externalPort), + "--hostname", sharedHost, "-f", + )).Should(Exit(0)) + Eventually(cf.Cf("delete", appA, "-f")).Should(Exit(0)) + Eventually(cf.Cf("delete", appB, "-f")).Should(Exit(0)) + }) + + It("load-balances TLS connections across both apps on the shared SNI hostname", func() { + sharedSNI := fmt.Sprintf("%s.%s", sharedHost, domainName) + + responses, err := GetNTLSResponses(10, domainName, externalPort, sharedSNI, true) + Expect(err).ToNot(HaveOccurred()) + Expect(responses).To(ContainElement(ContainSubstring(serverIdA))) + Expect(responses).To(ContainElement(ContainSubstring(serverIdB))) + }) + }) + + // One app reachable via two different SNI hostnames on the same port. HAProxy creates + // two independent use_backend rules that both resolve to the same app's containers. + // This is the SNI analogue of the port-only "maps both ports to the same application" test. + Context("one app reachable via two different SNI hostnames on the same port", func() { + var ( + appA string + hostOne = "sni-host-one" + hostTwo = "sni-host-two" + serverIdA = "server-multi-sni" + externalPort int + tcpDropletReceiver = assets.NewAssets().TCPListener + ) + + BeforeEach(func() { + appA = random_name.CATSRandomName("APP-A") + + Expect(cf.Cf( + "push", + "--no-route", + "--no-start", + appA, + "-p", tcpDropletReceiver, + "-b", Config.GetGoBuildpackName(), + "-m", DEFAULT_MEMORY_LIMIT, + "-f", filepath.Join(tcpDropletReceiver, "manifest.yml"), + "-c", fmt.Sprintf("tcp-listener --serverId=%s", serverIdA), + ).Wait()).To(Exit(0)) + + externalPort = MapTCPRouteWithHostname(appA, domainName, hostOne, 0) + MapTCPRouteWithHostname(appA, domainName, hostTwo, externalPort) + + Expect(cf.Cf("start", appA).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + }) + + AfterEach(func() { + app_helpers.AppReport(appA) + Eventually(cf.Cf("delete", appA, "-f", "-r")).Should(Exit(0)) + }) + + It("reaches the same app via both SNI hostnames", func() { + sniOne := fmt.Sprintf("%s.%s", hostOne, domainName) + sniTwo := fmt.Sprintf("%s.%s", hostTwo, domainName) + + respOne, err := SendAndReceiveTLS(domainName, externalPort, sniOne, true) + Expect(err).ToNot(HaveOccurred()) + Expect(respOne).To(ContainSubstring(serverIdA)) + + respTwo, err := SendAndReceiveTLS(domainName, externalPort, sniTwo, true) + Expect(err).ToNot(HaveOccurred()) + Expect(respTwo).To(ContainSubstring(serverIdA)) + }) + }) +})