diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 965b3735bf..0ff914a4d7 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -146,6 +146,32 @@ jobs: # TLS is intercepted, their passthrough assumptions # (TestActorEgressHTTPS's end-to-end TLS with the origin) no longer hold. run: hack/install-ate-kind.sh --deploy-atenet --atenet-dataplane=${{ matrix.dataplane }} --experimental-use-sdsmint + - name: Run E2E tests (egress certificate identity, sdsmint) + run: hack/run-e2e-kind.sh ./internal/e2e/suites/egressauthz -count=1 -v -args --no-color + - name: Verify egress certificate chain producer mutation (Envoy) + if: matrix.dataplane == 'envoy' + run: | + set -euo pipefail + kube=(kubectl --context kind-kind) + original_data="$(mktemp)" + leaf_data="$(mktemp)" + "${kube[@]}" -n ate-system get configmap atenet-egress -o json | jq '{data:.data}' > "$original_data" + jq -e '.data["envoy.yaml"] | contains("urlEncodedPemEncodedPeerCertificateChain()")' "$original_data" >/dev/null + jq '.data["envoy.yaml"] |= sub("urlEncodedPemEncodedPeerCertificateChain"; "urlEncodedPemEncodedPeerCertificate") | {data:.data}' "$original_data" > "$leaf_data" + restore() { + "${kube[@]}" -n ate-system patch configmap atenet-egress --type merge --patch-file "$original_data" >/dev/null + "${kube[@]}" -n ate-system rollout restart deployment/atenet-egress >/dev/null + "${kube[@]}" -n ate-system rollout status deployment/atenet-egress --timeout=180s + rm -f "$original_data" "$leaf_data" + } + trap restore EXIT + "${kube[@]}" -n ate-system patch configmap atenet-egress --type merge --patch-file "$leaf_data" >/dev/null + "${kube[@]}" -n ate-system rollout restart deployment/atenet-egress >/dev/null + "${kube[@]}" -n ate-system rollout status deployment/atenet-egress --timeout=180s + E2E_EGRESS_CERTIFICATE_LEAF_ONLY=1 hack/run-e2e-kind.sh ./internal/e2e/suites/egressauthz -run '^TestGatewayCertificateMetadataTransport$' -count=1 -v -args --no-color + restore + trap - EXIT + hack/run-e2e-kind.sh ./internal/e2e/suites/egressauthz -run '^TestGatewayCertificateMetadataTransport$' -count=1 -v -args --no-color - name: Run E2E tests (egress MITM trust) # The consumption half of the trust-bundle chain: an actor does TLS with # the MITM gateway's minted leaf using ONLY the projected bundle, plus a diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index b0b8f0f6a7..a5db23c141 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -56,21 +56,9 @@ import ( const ( // agentgatewayClientCertificateAttribute is the PEM peer certificate agentgateway // computes from the downstream TLS connection for ext_proc. - agentgatewayClientCertificateAttribute = "source.certificate" - // forwardedClientCertHeader is the header Envoy fills in with details of - // the mTLS peer, including the PEM chain it validated. The egress filter - // chain sets forward_client_cert_details: SANITIZE_SET, so whatever a - // client sends under this name is discarded and replaced by Envoy's own - // value. - // - // This is the only channel that can carry a whole certificate to ext_proc: - // the CEL request attributes Envoy exposes (subject, SANs, SHA-256 digest) - // cannot express the custom ActorIdentity X.509 extension this gateway - // authorizes on. - forwardedClientCertHeader = "x-forwarded-client-cert" - // xfccChainKey is the x-forwarded-client-cert key holding the URL-encoded - // PEM of the full presented chain, leaf first. - xfccChainKey = "chain" + agentgatewayClientCertificateAttribute = "source.certificate" + envoyClientCertificateMetadataNamespace = "dev.ate.egress.peer_certificate" + envoyClientCertificateChainKey = "chain" ) // deniedBody is the body of every policy denial. The reason goes to the log, @@ -321,6 +309,18 @@ func (h *Handler) validateActor(ctx context.Context, identity *substratex509.Act // on the request into a verified ActorIdentity, or an error describing why it // cannot be trusted. func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (*substratex509.ActorIdentity, error) { + if peer, present := md.DynamicMetadata[envoyClientCertificateMetadataNamespace]; present { + encoded := peer.GetFields()[envoyClientCertificateChainKey].GetStringValue() + chainPEM, err := url.PathUnescape(encoded) + if err != nil { + return nil, fmt.Errorf("decoding the client certificate chain: %w", err) + } + chain, err := parseCertificateChainPEM([]byte(chainPEM)) + if err != nil { + return nil, err + } + return h.verifyActorCertificate(chain) + } if certificate := md.Attribute(agentgatewayClientCertificateAttribute); certificate != "" { chain, err := parseCertificateChainPEM([]byte(certificate)) if err != nil { @@ -328,15 +328,7 @@ func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (*su } return h.verifyActorCertificate(chain) } - header := md.Header(forwardedClientCertHeader) - if header == "" { - return nil, fmt.Errorf("request carries no %s header", forwardedClientCertHeader) - } - chain, err := parseXFCCChain(header) - if err != nil { - return nil, err - } - return h.verifyActorCertificate(chain) + return nil, fmt.Errorf("request carries no trusted peer certificate") } // verifyActorCertificate checks that chain[0] is a live, non-CA, client-auth @@ -415,32 +407,6 @@ func (h *Handler) verifyActorCertificate(chain []*x509.Certificate) (*substratex return identity, nil } -// parseXFCCChain extracts the presented certificate chain, leaf first, from an -// x-forwarded-client-cert header value. -func parseXFCCChain(header string) ([]*x509.Certificate, error) { - // One element per proxy hop. SANITIZE_SET makes Envoy the only writer, so - // anything but exactly one element means either an unexpected proxy in front - // of the gateway or a listener that lost SANITIZE_SET — in both cases we no - // longer know which element describes our actual peer, so refuse to guess. - elements := splitXFCCUnquoted(header, ',') - if len(elements) != 1 { - return nil, fmt.Errorf("expected exactly one %s element, got %d", forwardedClientCertHeader, len(elements)) - } - encoded, ok := xfccValue(elements[0], xfccChainKey) - if !ok { - return nil, fmt.Errorf("%s carries no %q value", forwardedClientCertHeader, xfccChainKey) - } - // Envoy percent-encodes the PEM. PathUnescape, not QueryUnescape: base64 - // bodies contain '+', and query unescaping would decode it to a space and - // silently corrupt the DER. - chainPEM, err := url.PathUnescape(encoded) - if err != nil { - return nil, fmt.Errorf("decoding the client certificate chain: %w", err) - } - - return parseCertificateChainPEM([]byte(chainPEM)) -} - func parseCertificateChainPEM(chainPEM []byte) ([]*x509.Certificate, error) { var chain []*x509.Certificate rest := chainPEM @@ -465,82 +431,6 @@ func parseCertificateChainPEM(chainPEM []byte) ([]*x509.Certificate, error) { return chain, nil } -// xfccValue returns the value of key in one x-forwarded-client-cert element. -// Keys are matched case-insensitively; Envoy emits "Chain", but the header is -// consumed by enough different proxies that assuming its casing is not worth -// the failure mode. -func xfccValue(element, key string) (string, bool) { - for _, pair := range splitXFCCUnquoted(element, ';') { - k, v, found := strings.Cut(pair, "=") - if !found || !strings.EqualFold(strings.TrimSpace(k), key) { - continue - } - return unquoteXFCC(strings.TrimSpace(v)), true - } - return "", false -} - -// splitXFCCUnquoted splits on sep, ignoring separators inside a quoted value. -// x-forwarded-client-cert quotes any value containing its own delimiters, which -// the PEM ones always do. -func splitXFCCUnquoted(s string, sep rune) []string { - var parts []string - var current strings.Builder - quoted := false - escaped := false - for _, r := range s { - switch { - case escaped: - current.WriteRune(r) - escaped = false - case quoted && r == '\\': - current.WriteRune(r) - escaped = true - case r == '"': - quoted = !quoted - current.WriteRune(r) - case r == sep && !quoted: - parts = append(parts, current.String()) - current.Reset() - default: - current.WriteRune(r) - } - } - parts = append(parts, current.String()) - - trimmed := make([]string, 0, len(parts)) - for _, part := range parts { - if part = strings.TrimSpace(part); part != "" { - trimmed = append(trimmed, part) - } - } - return trimmed -} - -// unquoteXFCC strips the surrounding quotes from an x-forwarded-client-cert -// value and undoes the backslash escaping inside them. -func unquoteXFCC(value string) string { - if len(value) < 2 || !strings.HasPrefix(value, `"`) || !strings.HasSuffix(value, `"`) { - return value - } - inner := value[1 : len(value)-1] - var out strings.Builder - escaped := false - for _, r := range inner { - if escaped { - out.WriteRune(r) - escaped = false - continue - } - if r == '\\' { - escaped = true - continue - } - out.WriteRune(r) - } - return out.String() -} - // mapEgressIdentityError converts a GetActor failure into a client-facing // ext_proc denial. An unknown actor is treated as forbidden (the actor was // deleted out from under a still-valid certificate); transient control-plane diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index ab3f516cca..1b201b8723 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -25,7 +25,6 @@ import ( "encoding/json" "encoding/pem" "errors" - "fmt" "math/big" "net/url" "strings" @@ -85,6 +84,28 @@ func newTestCA(t *testing.T, commonName string) *testCA { return &testCA{cert: cert, key: key} } +func issueIntermediateCA(t *testing.T, parent *testCA, commonName string) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating intermediate key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(3), Subject: pkix.Name{CommonName: commonName}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign, BasicConstraintsValid: true, IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, parent.cert, &key.PublicKey, parent.key) + if err != nil { + t.Fatalf("creating intermediate certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing intermediate certificate: %v", err) + } + return &testCA{cert: cert, key: key} +} + func (ca *testCA) roots() *x509.CertPool { pool := x509.NewCertPool() pool.AddCert(ca.cert) @@ -186,23 +207,22 @@ func (ca *testCA) issueActorCertDER(t *testing.T, opts actorCertOptions) []byte return der } -// xfccHeader renders chain the way Envoy's SANITIZE_SET + -// set_current_client_cert_details{chain: true} does. -func xfccHeader(chain ...*x509.Certificate) string { +// encodedCertificateChain renders the URL-encoded PEM chain Envoy puts in +// trusted dynamic metadata. +func encodedCertificateChain(chain ...*x509.Certificate) string { der := make([][]byte, 0, len(chain)) for _, cert := range chain { der = append(der, cert.Raw) } - return xfccHeaderDER(der...) + return encodedCertificateChainDER(der...) } -func xfccHeaderDER(chain ...[]byte) string { +func encodedCertificateChainDER(chain ...[]byte) string { var buf strings.Builder for _, der := range chain { _ = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}) } - return fmt.Sprintf(`By=spiffe://cluster.local/ns/ate-system/sa/atenet-egress;Hash=abc123;Chain="%s"`, - url.PathEscape(buf.String())) + return url.PathEscape(buf.String()) } // egressHandler builds a Handler with an allow-everything policy, so the @@ -276,21 +296,22 @@ func runningActor() *ateapipb.Actor { // egressMetadata builds the CONNECT the egress listener hands to ext_proc, // with the peer certificate and the chain name of the CONNECT leg. -func egressMetadata(xfcc string) *extproc.RequestMetadata { +func egressMetadata(encoded string) *extproc.RequestMetadata { headers := []*corev3.HeaderValue{ {Key: ":method", RawValue: []byte("CONNECT")}, {Key: ":authority", RawValue: []byte("93.184.216.34:80")}, } - if xfcc != "" { - headers = append(headers, &corev3.HeaderValue{Key: forwardedClientCertHeader, RawValue: []byte(xfcc)}) - } - return extproc.NewRequestMetadata(headers, map[string]*structpb.Struct{ + md := extproc.NewRequestMetadata(headers, map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": { Fields: map[string]*structpb.Value{ extproc.FilterChainNameAttribute: structpb.NewStringValue(extproc.EgressFilterChainName), }, }, }) + if encoded != "" { + md.DynamicMetadata = map[string]*structpb.Struct{envoyClientCertificateMetadataNamespace: {Fields: map[string]*structpb.Value{envoyClientCertificateChainKey: structpb.NewStringValue(encoded)}}} + } + return md } func agentgatewayEgressMetadata(certificate string) *extproc.RequestMetadata { @@ -325,7 +346,7 @@ func TestHandleRequestHeadersAllowsVerifiedActor(t *testing.T) { leaf := ca.issueActorCert(t, actorCertOptions{}) h := egressHandler(ca.roots(), runningActor(), nil) - res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(encodedCertificateChain(leaf))) if err != nil { t.Fatalf("HandleRequestHeaders() error = %v, want nil", err) } @@ -346,6 +367,77 @@ func TestHandleRequestHeadersAllowsVerifiedActor(t *testing.T) { } } +func TestAuthenticateActorCertificateUsesTrustedMetadataOnly(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + h := egressHandler(ca.roots(), runningActor(), nil) + valid := encodedCertificateChain(leaf) + tests := []struct { + name string + edit func(*extproc.RequestMetadata) + }{ + {"missing namespace", func(md *extproc.RequestMetadata) { md.DynamicMetadata = map[string]*structpb.Struct{} }}, + {"nil namespace value", func(md *extproc.RequestMetadata) { + md.DynamicMetadata = map[string]*structpb.Struct{envoyClientCertificateMetadataNamespace: nil} + }}, + {"missing chain key", func(md *extproc.RequestMetadata) { + md.DynamicMetadata[envoyClientCertificateMetadataNamespace] = &structpb.Struct{} + }}, + {"foreign namespace", func(md *extproc.RequestMetadata) { + md.DynamicMetadata = map[string]*structpb.Struct{"foreign": md.DynamicMetadata[envoyClientCertificateMetadataNamespace]} + }}, + {"empty chain", func(md *extproc.RequestMetadata) { + md.DynamicMetadata[envoyClientCertificateMetadataNamespace].Fields[envoyClientCertificateChainKey] = structpb.NewStringValue("") + }}, + {"malformed escape", func(md *extproc.RequestMetadata) { + md.DynamicMetadata[envoyClientCertificateMetadataNamespace].Fields[envoyClientCertificateChainKey] = structpb.NewStringValue("%zz") + }}, + {"non-string chain", func(md *extproc.RequestMetadata) { + md.DynamicMetadata[envoyClientCertificateMetadataNamespace].Fields[envoyClientCertificateChainKey] = structpb.NewNumberValue(1) + }}, + {"invalid trusted metadata does not fall back", func(md *extproc.RequestMetadata) { + md.DynamicMetadata[envoyClientCertificateMetadataNamespace].Fields[envoyClientCertificateChainKey] = structpb.NewStringValue("bad") + md.Attributes["envoy.filters.http.ext_proc"].Fields[agentgatewayClientCertificateAttribute] = structpb.NewStringValue(string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leaf.Raw}))) + }}, + {"invalid trusted metadata does not fall back to header", func(md *extproc.RequestMetadata) { + md.DynamicMetadata[envoyClientCertificateMetadataNamespace].Fields[envoyClientCertificateChainKey] = structpb.NewStringValue("bad") + md.Headers["x-forwarded-client-cert"] = `Chain="` + encodedCertificateChain(leaf) + `"` + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + md := egressMetadata(valid) + tc.edit(md) + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, envoy_type.StatusCode_Forbidden) + }) + } +} + +func TestValidCertificateHeaderWithoutTrustedMetadataIsDenied(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + h := egressHandler(ca.roots(), runningActor(), nil) + md := egressMetadata("") + md.Headers["x-forwarded-client-cert"] = `Chain="` + encodedCertificateChain(leaf) + `"` + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, envoy_type.StatusCode_Forbidden) +} + +func TestTrustedMetadataWinsOverCertificateHeader(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + other := ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, ActorName: "other-actor", ActorUid: "8f14e45f-ceea-467a-9575-25a0d5d5e4b0", Purpose: substratex509.ActorIdentityPurposeAtunnel, + }}) + h := egressHandler(ca.roots(), runningActor(), nil) + md := egressMetadata(encodedCertificateChain(leaf)) + md.Headers["x-forwarded-client-cert"] = `Chain="` + encodedCertificateChain(other) + `"` + if _, err := h.HandleRequestHeaders(context.Background(), md); err != nil { + t.Fatalf("trusted metadata actor was not selected: %v", err) + } +} + // passthroughDestinationOf reads the address a CONNECT decision handed back // for the passthrough chain, or "" when it handed back none. func passthroughDestinationOf(res extproc.Result) string { @@ -377,7 +469,7 @@ func TestConnectLegDecidesAddressRules(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := New(&egressMockClient{actor: runningActor(), policy: tc.policy}, ca.roots(), 0, nil, "") - md := egressMetadata(xfccHeader(leaf)) + md := egressMetadata(encodedCertificateChain(leaf)) md.Host = tc.authority md.Headers[":authority"] = tc.authority res, err := h.HandleRequestHeaders(context.Background(), md) @@ -434,26 +526,26 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { otherCA := newTestCA(t, "some-other-ca") tests := []struct { - name string - xfcc func(t *testing.T) string - want envoy_type.StatusCode + name string + encodedChain func(t *testing.T) string + want envoy_type.StatusCode }{ { - name: "no client certificate at all", - xfcc: func(*testing.T) string { return "" }, - want: envoy_type.StatusCode_Forbidden, + name: "no client certificate at all", + encodedChain: func(*testing.T) string { return "" }, + want: envoy_type.StatusCode_Forbidden, }, { name: "signed by an unknown CA", - xfcc: func(t *testing.T) string { - return xfccHeader(otherCA.issueActorCert(t, actorCertOptions{})) + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(otherCA.issueActorCert(t, actorCertOptions{})) }, want: envoy_type.StatusCode_Forbidden, }, { name: "expired", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.NotBefore = time.Now().Add(-2 * time.Hour) c.NotAfter = time.Now().Add(-time.Hour) }})) @@ -462,8 +554,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "not yet valid", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.NotBefore = time.Now().Add(time.Hour) c.NotAfter = time.Now().Add(2 * time.Hour) }})) @@ -472,8 +564,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "no ClientAuth EKU", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} }})) }, @@ -484,8 +576,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { // VerifyOptions.KeyUsages; the explicit ClientAuth check is what // catches it. name: "empty EKU", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.ExtKeyUsage = nil }})) }, @@ -493,8 +585,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "is a CA certificate", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.IsCA = true c.KeyUsage |= x509.KeyUsageCertSign }})) @@ -503,8 +595,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "no ActorIdentity extension", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.ExtraExtensions = nil }})) }, @@ -518,8 +610,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { // handler denies rather than panics when the parse fails, and // substratex509 rejects a second copy if a parser ever allowed one. name: "two ActorIdentity extensions", - xfcc: func(t *testing.T) string { - return xfccHeaderDER(ca.issueActorCertDER(t, actorCertOptions{ + encodedChain: func(t *testing.T) string { + return encodedCertificateChainDER(ca.issueActorCertDER(t, actorCertOptions{ extraIdentity: &substratex509.ActorIdentity{ Atespace: testEgressAtespace, ActorName: "a-different-actor", @@ -532,8 +624,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "generic purpose", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ Atespace: testEgressAtespace, ActorName: testEgressActor, ActorUid: testEgressActorUID, @@ -544,8 +636,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "missing purpose", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ Atespace: testEgressAtespace, ActorName: testEgressActor, ActorUid: testEgressActorUID, @@ -555,8 +647,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "empty atespace", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ ActorName: testEgressActor, ActorUid: testEgressActorUID, Purpose: substratex509.ActorIdentityPurposeAtunnel, @@ -566,8 +658,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "empty actor name", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ Atespace: testEgressAtespace, ActorUid: testEgressActorUID, Purpose: substratex509.ActorIdentityPurposeAtunnel, @@ -577,8 +669,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "empty actor UID", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ Atespace: testEgressAtespace, ActorName: testEgressActor, Purpose: substratex509.ActorIdentityPurposeAtunnel, @@ -587,19 +679,16 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { want: envoy_type.StatusCode_Forbidden, }, { - // SANITIZE_SET makes Envoy the only writer of the header, so two - // elements means we can no longer tell which one is our peer. - name: "two XFCC elements", - xfcc: func(t *testing.T) string { - leaf := ca.issueActorCert(t, actorCertOptions{}) - return xfccHeader(leaf) + "," + xfccHeader(leaf) + name: "malformed encoded chain", + encodedChain: func(t *testing.T) string { + return "%zz" }, want: envoy_type.StatusCode_Forbidden, }, { - name: "XFCC without a Chain value", - xfcc: func(*testing.T) string { - return `By=spiffe://cluster.local/ns/ate-system/sa/atenet-egress;Hash=abc123` + name: "encoded chain without a certificate", + encodedChain: func(*testing.T) string { + return url.PathEscape("not a certificate") }, want: envoy_type.StatusCode_Forbidden, }, @@ -607,8 +696,8 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { // The CONNECT would authenticate as one actor and the requests // inside the tunnel be policed as another. name: "URI SAN names a different actor", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{ + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{ uriSAN: "spiffe://substrate-actor.local/atespace/" + testEgressAtespace + "/actor/other-actor", })) }, @@ -616,24 +705,31 @@ func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { }, { name: "no URI SAN at all", - xfcc: func(t *testing.T) string { - return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.URIs = nil }})) + encodedChain: func(t *testing.T) string { + return encodedCertificateChain(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { c.URIs = nil }})) }, want: envoy_type.StatusCode_Forbidden, }, { - name: "XFCC Chain that is not a certificate", - xfcc: func(*testing.T) string { + name: "encoded chain that is not a certificate", + encodedChain: func(*testing.T) string { return `Chain="` + url.PathEscape("-----BEGIN CERTIFICATE-----\nbm90LWEtY2VydA==\n-----END CERTIFICATE-----\n") + `"` }, want: envoy_type.StatusCode_Forbidden, }, + { + name: "certificate PEM with invalid DER", + encodedChain: func(*testing.T) string { + return encodedCertificateChainDER([]byte("not DER")) + }, + want: envoy_type.StatusCode_Forbidden, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := egressHandler(ca.roots(), runningActor(), nil) - _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(tc.xfcc(t))) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(tc.encodedChain(t))) wantStatus(t, err, tc.want) }) } @@ -701,7 +797,7 @@ func TestHandleRequestHeadersAuthorization(t *testing.T) { t.Run(tc.name, func(t *testing.T) { h := egressHandler(ca.roots(), tc.actor, tc.err) leaf := ca.issueActorCert(t, actorCertOptions{}) - _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(encodedCertificateChain(leaf))) wantStatus(t, err, tc.want) }) } @@ -714,7 +810,7 @@ func TestHandleRequestHeadersWithoutConfiguredCA(t *testing.T) { leaf := ca.issueActorCert(t, actorCertOptions{}) h := egressHandler(nil, runningActor(), nil) - _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(encodedCertificateChain(leaf))) wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable) } @@ -726,7 +822,7 @@ func TestHandleRequestHeadersRejectsNonAddressAuthority(t *testing.T) { h := egressHandler(ca.roots(), runningActor(), nil) for _, authority := range []string{"example.com:443", "93.184.216.34", ""} { - md := egressMetadata(xfccHeader(leaf)) + md := egressMetadata(encodedCertificateChain(leaf)) md.Host = authority md.Headers[":authority"] = authority _, err := h.HandleRequestHeaders(context.Background(), md) @@ -739,7 +835,7 @@ func TestHandleRequestHeadersRejectsNonConnect(t *testing.T) { leaf := ca.issueActorCert(t, actorCertOptions{}) h := egressHandler(ca.roots(), runningActor(), nil) - md := egressMetadata(xfccHeader(leaf)) + md := egressMetadata(encodedCertificateChain(leaf)) md.Method = "GET" md.Headers[":method"] = "GET" @@ -749,7 +845,7 @@ func TestHandleRequestHeadersRejectsNonConnect(t *testing.T) { // PEM bodies routinely contain '+'. Decoding the header as a query string would // turn those into spaces and corrupt the DER, so pin the round trip. -func TestParseXFCCChainPreservesPlusInPEM(t *testing.T) { +func TestEncodedCertificateChainPreservesPlusInPEM(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") // Serials differ per certificate, so mint until one encodes with a '+'. var leaf *x509.Certificate @@ -764,27 +860,42 @@ func TestParseXFCCChainPreservesPlusInPEM(t *testing.T) { t.Skip("no certificate with a '+' in its PEM body after 50 attempts") } - chain, err := parseXFCCChain(xfccHeader(leaf)) + decoded, err := url.PathUnescape(encodedCertificateChain(leaf)) + if err != nil { + t.Fatal(err) + } + chain, err := parseCertificateChainPEM([]byte(decoded)) if err != nil { - t.Fatalf("parseXFCCChain() error = %v", err) + t.Fatalf("parse encoded certificate chain: %v", err) } if len(chain) != 1 || !chain[0].Equal(leaf) { - t.Fatalf("parseXFCCChain() did not round-trip the certificate") + t.Fatalf("encoded certificate chain did not round-trip the certificate") } } -func TestParseXFCCChainIncludesIntermediates(t *testing.T) { - ca := newTestCA(t, "actor-identity-ca") - leaf := ca.issueActorCert(t, actorCertOptions{}) +func TestEncodedCertificateChainIncludesIntermediates(t *testing.T) { + root := newTestCA(t, "actor-identity-root") + intermediate := issueIntermediateCA(t, root, "actor-identity-intermediate") + leaf := intermediate.issueActorCert(t, actorCertOptions{}) - chain, err := parseXFCCChain(xfccHeader(leaf, ca.cert)) + decoded, err := url.PathUnescape(encodedCertificateChain(leaf, intermediate.cert)) if err != nil { - t.Fatalf("parseXFCCChain() error = %v", err) + t.Fatal(err) + } + chain, err := parseCertificateChainPEM([]byte(decoded)) + if err != nil { + t.Fatalf("parse encoded certificate chain: %v", err) } if len(chain) != 2 { - t.Fatalf("parseXFCCChain() returned %d certificates, want 2", len(chain)) + t.Fatalf("encoded certificate chain returned %d certificates, want 2", len(chain)) } if !chain[0].Equal(leaf) { - t.Error("parseXFCCChain() did not return the leaf first") + t.Error("encoded certificate chain did not return the leaf first") + } + h := egressHandler(root.roots(), runningActor(), nil) + if _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(encodedCertificateChain(leaf, intermediate.cert))); err != nil { + t.Fatalf("leaf plus intermediate was denied: %v", err) } + _, err = h.HandleRequestHeaders(context.Background(), egressMetadata(encodedCertificateChain(leaf))) + wantStatus(t, err, envoy_type.StatusCode_Forbidden) } diff --git a/cmd/atenet/internal/router/egress/request_test.go b/cmd/atenet/internal/router/egress/request_test.go index eb39c96a4a..eb2fca6fd7 100644 --- a/cmd/atenet/internal/router/egress/request_test.go +++ b/cmd/atenet/internal/router/egress/request_test.go @@ -295,7 +295,7 @@ func TestConnectLegRequiresAPolicy(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := New(tc.client, ca.roots(), DefaultPolicyCacheTTL, nil, "") - res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(encodedCertificateChain(leaf))) if tc.want == 0 { wantAllowed(t, res, err) if calls := tc.client.policyCalls.Load(); calls != 1 { diff --git a/cmd/atenet/internal/router/egresspolicymanifest_test.go b/cmd/atenet/internal/router/egresspolicymanifest_test.go index f9a134154b..caf314190d 100644 --- a/cmd/atenet/internal/router/egresspolicymanifest_test.go +++ b/cmd/atenet/internal/router/egresspolicymanifest_test.go @@ -31,12 +31,73 @@ import ( const ( extProcFilter = "envoy.filters.http.ext_proc" setFilterStateFilter = "envoy.filters.http.set_filter_state" + luaFilter = "envoy.filters.http.lua" extProcServerCluster = "ext_proc_server" passthroughCluster = "egress_tcp_passthrough" originalDstKey = "envoy.network.transport_socket.original_dst_address" dfpClusterType = "envoy.clusters.dynamic_forward_proxy" ) +const peerCertificateNamespace = "dev.ate.egress.peer_certificate" + +func TestEgressManifestsForwardPeerCertificateMetadata(t *testing.T) { + for _, path := range egressManifests { + t.Run(path, func(t *testing.T) { + tree := bootstrapTree(t, path) + outer := outerChain(t, tree) + ts := child(outer, "transport_socket") + tls := child(ts, "typed_config") + if required, _ := tls["require_client_certificate"].(bool); !required { + t.Error("outer TLS no longer requires an actor client certificate") + } + if _, ok := child(child(tls, "common_tls_context"), "validation_context")["trusted_ca"]; !ok { + t.Error("outer TLS has no actor identity trusted_ca") + } + h := hcm(outer) + if got := str(h, "forward_client_cert_details"); got != "SANITIZE" { + t.Errorf("forward_client_cert_details = %q, want SANITIZE", got) + } + if _, ok := h["set_current_client_cert_details"]; ok { + t.Error("outer HCM still generates certificate headers") + } + filters := list(h, "http_filters") + luaAt, extAt := filterIndex(filters, luaFilter), filterIndex(filters, extProcFilter) + if count := slices.IndexFunc(filters, func(f node) bool { return str(f, "name") == luaFilter }); count < 0 || slices.IndexFunc(filters[count+1:], func(f node) bool { return str(f, "name") == luaFilter }) >= 0 { + t.Errorf("outer HCM must have exactly one Lua producer") + } + if luaAt < 0 || extAt < 0 || luaAt >= extAt { + t.Fatalf("outer HCM Lua/ext_proc order = %d/%d", luaAt, extAt) + } + script := str(child(child(filters[luaAt], "typed_config"), "default_source_code"), "inline_string") + for _, want := range []string{"downstreamSslConnection", "urlEncodedPemEncodedPeerCertificateChain", peerCertificateNamespace, "chain", "chain = \"\""} { + if !strings.Contains(script, want) { + t.Errorf("Lua producer does not contain %q", want) + } + } + cfg := child(filters[extAt], "typed_config") + forward := strs(child(child(cfg, "metadata_options"), "forwarding_namespaces"), "untyped") + receive := strs(child(child(cfg, "metadata_options"), "receiving_namespaces"), "untyped") + if !slices.Equal(forward, []string{peerCertificateNamespace}) || !slices.Equal(receive, []string{extproc.EgressMetadataNamespace}) { + t.Errorf("metadata namespaces = forward %v receive %v", forward, receive) + } + if got := filterIndex(filters, luaFilter); got != 1 { + t.Errorf("Lua producer index = %d, want immediately after actor filter state", got) + } + if writers := filterStateWriters(tree, peerCertificateNamespace); len(writers) != 0 { + t.Errorf("certificate namespace appears in filter state writers: %d", len(writers)) + } + for _, lc := range allChains(tree) { + if str(lc.chain, "name") == extproc.EgressFilterChainName { + continue + } + if containsString(lc.chain, peerCertificateNamespace) { + t.Errorf("certificate namespace appears on inner chain %q", str(lc.chain, "name")) + } + } + }) + } +} + // requestLegs are the chains that decide per request and answer with a dial. var requestLegs = []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} @@ -151,6 +212,26 @@ func filterStateWriters(v any, key string) []node { return found } +func containsString(v any, want string) bool { + switch t := v.(type) { + case string: + return t == want + case map[string]any: + for _, value := range t { + if containsString(value, want) { + return true + } + } + case []any: + for _, value := range t { + if containsString(value, want) { + return true + } + } + } + return false +} + // outerChain returns the egress listener's CONNECT chain. func outerChain(t *testing.T, tree node) node { t.Helper() diff --git a/cmd/atenet/internal/router/extproc/extproc.go b/cmd/atenet/internal/router/extproc/extproc.go index 05f0f6fe48..fdd5c4fce7 100644 --- a/cmd/atenet/internal/router/extproc/extproc.go +++ b/cmd/atenet/internal/router/extproc/extproc.go @@ -129,6 +129,7 @@ func (s *Server) processRequestHeaders( ) *extprocv3.ProcessingResponse { start := time.Now() md := NewRequestMetadata(reqHeaders.GetHeaders().GetHeaders(), req.GetAttributes()) + md.DynamicMetadata = req.GetMetadataContext().GetFilterMetadata() // One atenet binary serves both directions, as two ext_proc handlers // selected here. They are deployed separately today — atenet-router fronts diff --git a/cmd/atenet/internal/router/extproc/extproc_test.go b/cmd/atenet/internal/router/extproc/extproc_test.go index 384b910ccd..457ee96286 100644 --- a/cmd/atenet/internal/router/extproc/extproc_test.go +++ b/cmd/atenet/internal/router/extproc/extproc_test.go @@ -16,26 +16,66 @@ package extproc import ( "context" + "reflect" "strings" "testing" + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/protobuf/types/known/structpb" ) // stubHandler records that it ran and returns an empty successful Result. type stubHandler struct { direction Direction called bool + metadata *RequestMetadata } func (h *stubHandler) Direction() Direction { return h.direction } -func (h *stubHandler) HandleRequestHeaders(context.Context, *RequestMetadata) (Result, error) { +func (h *stubHandler) HandleRequestHeaders(_ context.Context, md *RequestMetadata) (Result, error) { h.called = true + h.metadata = md return Result{Response: &extprocv3.HeadersResponse{Response: &extprocv3.CommonResponse{}}}, nil } +func TestProcessRequestHeadersForwardsDynamicMetadata(t *testing.T) { + h := &stubHandler{direction: DirectionEgress} + s := NewServer(50051, nil, Handlers{DirectionEgress: h}) + req := connectRequest("envoy.filters.http.ext_proc", EgressFilterChainName) + req.MetadataContext = &corev3.Metadata{FilterMetadata: map[string]*structpb.Struct{ + "dev.ate.egress.peer_certificate": {Fields: map[string]*structpb.Value{ + "chain": structpb.NewStringValue("encoded-peer-chain"), + }}, + }} + attrs := req.Attributes + s.processRequestHeaders(context.Background(), req, req.GetRequestHeaders()) + if h.metadata.DynamicMetadata["dev.ate.egress.peer_certificate"] != req.MetadataContext.FilterMetadata["dev.ate.egress.peer_certificate"] { + t.Fatal("dynamic metadata was copied instead of forwarded") + } + if !reflect.DeepEqual(h.metadata.Attributes, attrs) { + t.Fatalf("attributes changed: got %#v want %#v", h.metadata.Attributes, attrs) + } + if _, ok := h.metadata.Headers["x-forwarded-client-cert"]; ok { + t.Fatal("certificate header was copied into request metadata") + } + if got := h.metadata.DynamicMetadata["dev.ate.egress.peer_certificate"].GetFields()["chain"].GetStringValue(); got != "encoded-peer-chain" { + t.Errorf("chain metadata = %q, want encoded-peer-chain", got) + } + if got := h.metadata.Attribute(FilterChainNameAttribute); got != EgressFilterChainName { + t.Fatalf("filter chain attribute = %q, want %q", got, EgressFilterChainName) + } + h = &stubHandler{direction: DirectionEgress} + req = connectRequest("envoy.filters.http.ext_proc", EgressFilterChainName) + s = NewServer(50051, nil, Handlers{DirectionEgress: h}) + s.processRequestHeaders(context.Background(), req, req.GetRequestHeaders()) + if h.metadata.DynamicMetadata != nil { + t.Fatalf("nil metadata context became %#v", h.metadata.DynamicMetadata) + } +} + // The mux must pick the handler by the Envoy-asserted filter chain, and refuse // outright when this instance was not started to serve that direction (--mode). // Falling back to the other handler would run the request through the opposite diff --git a/cmd/atenet/internal/router/extproc/metadata.go b/cmd/atenet/internal/router/extproc/metadata.go index ffee0b035d..58eaf6db69 100644 --- a/cmd/atenet/internal/router/extproc/metadata.go +++ b/cmd/atenet/internal/router/extproc/metadata.go @@ -44,6 +44,8 @@ type RequestMetadata struct { // connect_terminate -> main_internal internal-listener hop that CONNECT // requests take. Attributes map[string]*structpb.Struct + // DynamicMetadata contains the dataplane's forwarded filter metadata. + DynamicMetadata map[string]*structpb.Struct } func NewRequestMetadata(headers []*corev3.HeaderValue, attributes map[string]*structpb.Struct) *RequestMetadata { diff --git a/internal/e2e/fixtures/testserver/egressprobe.go b/internal/e2e/fixtures/testserver/egressprobe.go index 3401a6c6a1..7f23e43374 100644 --- a/internal/e2e/fixtures/testserver/egressprobe.go +++ b/internal/e2e/fixtures/testserver/egressprobe.go @@ -15,6 +15,7 @@ package main import ( + "bufio" "context" "crypto/tls" "crypto/x509" @@ -23,7 +24,10 @@ import ( "errors" "fmt" "log" + "net" "net/http" + "net/url" + "os" "time" "github.com/spf13/cobra" @@ -132,6 +136,82 @@ func handshake(w http.ResponseWriter, r *http.Request, cfg probeConfig) { writeJSON(w, result) } +type connectResult struct { + Credential string `json:"credential"` + Stage string `json:"stage,omitempty"` + ConnectStatus int `json:"connect_status,omitempty"` + Error string `json:"error,omitempty"` +} + +func connectHandler(w http.ResponseWriter, r *http.Request, cfg probeConfig) { + destination := r.URL.Query().Get("destination") + credential := r.URL.Query().Get("credential-bundle") + if destination == "" || credential == "" { + http.Error(w, "missing destination or credential-bundle query parameter", http.StatusBadRequest) + return + } + ctx, cancel := context.WithTimeout(r.Context(), cfg.handshakeTimeout) + defer cancel() + status, stage, err := connect(ctx, destination, credential, r.URL.Query().Get("xfcc"), cfg) + result := connectResult{Credential: credential, Stage: stage, ConnectStatus: status} + if err != nil { + result.Error = err.Error() + } + writeJSON(w, result) +} + +func connect(ctx context.Context, destination, credentialBundle, xfcc string, cfg probeConfig) (int, string, error) { + cert, err := credbundle.ClientLoader(credentialBundle)(nil) + if err != nil { + return 0, stageClient, fmt.Errorf("loading credential bundle: %w", err) + } + rootsPEM, err := os.ReadFile(cfg.trustBundlePath) + if err != nil { + return 0, stageClient, fmt.Errorf("reading gateway trust bundle: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(rootsPEM) { + return 0, stageClient, fmt.Errorf("gateway trust bundle contains no certificates") + } + dialer := net.Dialer{} + conn, err := dialer.DialContext(ctx, "tcp", cfg.gatewayAddress) + if err != nil { + return 0, stageTunnel, fmt.Errorf("dialing gateway: %w", err) + } + defer conn.Close() + if deadline, ok := ctx.Deadline(); ok { + if err := conn.SetDeadline(deadline); err != nil { + return 0, stageTunnel, fmt.Errorf("setting gateway deadline: %w", err) + } + } + tlsConn := tls.Client(conn, &tls.Config{ServerName: serverName(cfg.gatewayAddress), RootCAs: roots, MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{*cert}}) + if err := tlsConn.HandshakeContext(ctx); err != nil { + return 0, stageGatewayTLS, fmt.Errorf("gateway TLS handshake: %w", err) + } + defer tlsConn.Close() + req := &http.Request{Method: http.MethodConnect, URL: &url.URL{Host: destination}, Host: destination, Header: make(http.Header)} + if xfcc != "" { + req.Header.Set("X-Forwarded-Client-Cert", xfcc) + } + if err := req.Write(tlsConn); err != nil { + return 0, stageTunnel, fmt.Errorf("writing CONNECT request: %w", err) + } + resp, err := http.ReadResponse(bufio.NewReader(tlsConn), req) + if err != nil { + return 0, stageTunnel, fmt.Errorf("reading CONNECT response: %w", err) + } + status := resp.StatusCode + // Close the transport before closing a successful CONNECT body: net/http + // represents that body with the live tunnel and Close may otherwise wait for + // EOF from the gateway. + _ = tlsConn.Close() + _ = resp.Body.Close() + if status >= 200 && status < 300 { + return status, "", nil + } + return status, stageConnect, nil +} + func encodeChain(chain []*x509.Certificate) string { var out []byte for _, cert := range chain { @@ -234,6 +314,9 @@ func newEgressProbeCmd() *cobra.Command { mux.HandleFunc("/handshake", func(w http.ResponseWriter, r *http.Request) { handshake(w, r, cfg) }) + mux.HandleFunc("/connect", func(w http.ResponseWriter, r *http.Request) { + connectHandler(w, r, cfg) + }) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) diff --git a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl index 97b78a7137..ea63c24fd0 100644 --- a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl @@ -64,6 +64,12 @@ spec: volumeMounts: - name: "actor-identity-unknown" mountPath: "/run/actor-identity-unknown" + - name: "actor-identity-live" + mountPath: "/run/actor-identity-live" + - name: "actor-identity-live-chain" + mountPath: "/run/actor-identity-live-chain" + - name: "actor-identity-wrong-uid" + mountPath: "/run/actor-identity-wrong-uid" - name: "podidentity" mountPath: "/run/podidentity.podcert.ate.dev" - name: "servicedns-ca" @@ -72,6 +78,18 @@ spec: - name: "actor-identity-unknown" secret: secretName: egressprobe-unknown-actor + - name: "actor-identity-live" + secret: + secretName: egressprobe-live-actor + optional: true + - name: "actor-identity-live-chain" + secret: + secretName: egressprobe-live-actor-chain + optional: true + - name: "actor-identity-wrong-uid" + secret: + secretName: egressprobe-wrong-uid + optional: true - name: "podidentity" projected: sources: diff --git a/internal/e2e/fixtures/testserver/egressprobe_test.go b/internal/e2e/fixtures/testserver/egressprobe_test.go new file mode 100644 index 0000000000..7b10f50880 --- /dev/null +++ b/internal/e2e/fixtures/testserver/egressprobe_test.go @@ -0,0 +1,239 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "net/http" + "os" + "testing" + "time" +) + +func TestEgressProbeConnect(t *testing.T) { + caKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test CA"}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + ca, err = x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + serverTLS, _ := testTLSCredential(t, ca, caKey, false) + clientTLS, clientCert := testTLSCredential(t, ca, caKey, true) + rootsPath := writePEMFile(t, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})) + bundlePath := writeBundleFile(t, clientTLS, clientCert) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + tlsListener := tls.NewListener(listener, &tls.Config{Certificates: []tls.Certificate{serverTLS}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: certPool(ca)}) + defer tlsListener.Close() + received := make(chan *connectRequest, 4) + heldReady, heldRelease := make(chan struct{}), make(chan struct{}) + stallReady, stallRelease := make(chan struct{}), make(chan struct{}) + go func() { + for { + conn, err := tlsListener.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + tlsConn := conn.(*tls.Conn) + if err := tlsConn.Handshake(); err != nil { + return + } + req, err := http.ReadRequest(bufio.NewReader(tlsConn)) + if err != nil { + return + } + received <- &connectRequest{method: req.Method, host: req.Host, xfcc: req.Header.Get("X-Forwarded-Client-Cert"), serial: tlsConn.ConnectionState().PeerCertificates[0].SerialNumber.String()} + status := 200 + if req.Header.Get("X-Forwarded-Client-Cert") != "" { + status = 403 + } + if req.Host == "192.0.2.45:443" { + close(stallReady) + <-stallRelease + return + } + if req.Host == "192.0.2.46:443" { + fmt.Fprintf(tlsConn, "HTTP/1.1 200\r\n\r\n") + close(heldReady) + <-heldRelease + return + } + fmt.Fprintf(tlsConn, "HTTP/1.1 %d\r\nContent-Length: 0\r\n\r\n", status) + }() + } + }() + cfg := probeConfig{gatewayAddress: listener.Addr().String(), trustBundlePath: rootsPath, handshakeTimeout: 5 * time.Second} + for _, tc := range []struct { + name, xfcc string + wantStatus int + wantStage string + }{{"success", "", 200, ""}, {"forbidden", `Chain="hostile"`, 403, stageConnect}} { + t.Run(tc.name, func(t *testing.T) { + status, stage, err := connect(t.Context(), "192.0.2.44:443", bundlePath, tc.xfcc, cfg) + if err != nil || stage != tc.wantStage || status != tc.wantStatus { + t.Fatalf("connect() = (%d, %q, %v), want (%d, %q, nil)", status, stage, err, tc.wantStatus, tc.wantStage) + } + select { + case got := <-received: + if got.method != "CONNECT" || got.host != "192.0.2.44:443" || got.xfcc != tc.xfcc { + t.Fatalf("request = %+v", got) + } + if got.serial != clientCert.SerialNumber.String() { + t.Fatalf("client serial = %s, want %s", got.serial, clientCert.SerialNumber) + } + case <-time.After(time.Second): + t.Fatal("server did not receive CONNECT") + } + }) + } + t.Run("stalled response is bounded", func(t *testing.T) { + defer close(stallRelease) + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + result := make(chan struct { + status int + stage string + err error + }, 1) + go func() { + status, stage, err := connect(ctx, "192.0.2.45:443", bundlePath, "", cfg) + result <- struct { + status int + stage string + err error + }{status, stage, err} + }() + select { + case <-stallReady: + case <-time.After(time.Second): + t.Fatal("server did not receive stalled CONNECT") + } + var out struct { + status int + stage string + err error + } + select { + case out = <-result: + case <-time.After(3 * time.Second): + t.Fatal("stalled CONNECT did not respect deadline") + } + status, stage, err := out.status, out.stage, out.err + var netErr net.Error + if err == nil || !errors.As(err, &netErr) || !netErr.Timeout() || status != 0 || stage != stageTunnel { + t.Fatalf("connect() = (%d, %q, %v), want bounded tunnel failure", status, stage, err) + } + }) + t.Run("held-open success returns before release", func(t *testing.T) { + defer close(heldRelease) + result := make(chan struct { + status int + stage string + err error + }, 1) + go func() { + status, stage, err := connect(t.Context(), "192.0.2.46:443", bundlePath, "", cfg) + result <- struct { + status int + stage string + err error + }{status, stage, err} + }() + select { + case <-heldReady: + case <-time.After(time.Second): + t.Fatal("server did not receive held CONNECT") + } + select { + case out := <-result: + if out.err != nil || out.status != 200 || out.stage != "" { + t.Fatalf("held CONNECT = %+v", out) + } + case <-time.After(time.Second): + t.Fatal("CONNECT waited for held tunnel release") + } + }) +} + +type connectRequest struct{ method, host, xfcc, serial string } + +func testTLSCredential(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, client bool) (tls.Certificate, *x509.Certificate) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + ext := x509.ExtKeyUsageServerAuth + if client { + ext = x509.ExtKeyUsageClientAuth + } + cert := &x509.Certificate{SerialNumber: newSerial(), Subject: pkix.Name{CommonName: "fixture"}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{ext}} + if !client { + cert.DNSNames = []string{"localhost"} + cert.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + } + der, err := x509.CreateCertificate(rand.Reader, cert, ca, &key.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + parsed, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, parsed +} + +func certPool(ca *x509.Certificate) *x509.CertPool { p := x509.NewCertPool(); p.AddCert(ca); return p } +func writePEMFile(t *testing.T, data []byte) string { + t.Helper() + p := t.TempDir() + "/trust.pem" + if err := os.WriteFile(p, data, 0600); err != nil { + t.Fatal(err) + } + return p +} +func writeBundleFile(t *testing.T, cert tls.Certificate, parsed *x509.Certificate) string { + t.Helper() + keyDER, _ := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) + data := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + data = append(data, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: parsed.Raw})...) + p := t.TempDir() + "/bundle.pem" + if err := os.WriteFile(p, data, 0600); err != nil { + t.Fatal(err) + } + return p +} + +func newSerial() *big.Int { return big.NewInt(time.Now().UnixNano()) } diff --git a/internal/e2e/suites/egressauthz/actoridentity_test.go b/internal/e2e/suites/egressauthz/actoridentity_test.go index ffe82e5e2a..1da27006c0 100644 --- a/internal/e2e/suites/egressauthz/actoridentity_test.go +++ b/internal/e2e/suites/egressauthz/actoridentity_test.go @@ -22,6 +22,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "math/big" "net/url" "path" "testing" @@ -132,6 +133,28 @@ func mintActorCredential(t *testing.T, ca *localca.CA, identity *substratex509.A return bundle } +// mintActorCredentialWithIntermediate proves the probe sends a complete chain; +// the gateway trusts only the actor root, so omitting this intermediate must fail. +func mintActorCredentialWithIntermediate(t *testing.T, ca *localca.CA, identity *substratex509.ActorIdentity) []byte { + t.Helper() + intermediateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + intermediateTemplate := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "test actor intermediate"}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(actorCertificateLifetime), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature} + intermediateDER, err := x509.CreateCertificate(rand.Reader, intermediateTemplate, ca.RootCertificate, &intermediateKey.PublicKey, ca.SigningKey) + if err != nil { + t.Fatal(err) + } + intermediate, err := x509.ParseCertificate(intermediateDER) + if err != nil { + t.Fatal(err) + } + bundle := mintActorCredential(t, &localca.CA{RootCertificate: intermediate, SigningKey: intermediateKey}, identity) + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: intermediateDER})...) + return bundle +} + // writeCredentialSecret puts a minted bundle where the probe pod can mount it. func writeCredentialSecret(t *testing.T, ctx context.Context, ns, name string, bundle []byte) { t.Helper() diff --git a/internal/e2e/suites/egressauthz/egressauthz_test.go b/internal/e2e/suites/egressauthz/egressauthz_test.go index d470fee53e..befdc72fad 100644 --- a/internal/e2e/suites/egressauthz/egressauthz_test.go +++ b/internal/e2e/suites/egressauthz/egressauthz_test.go @@ -15,7 +15,7 @@ // Package egressauthz e2e-tests the egress gateway's front door: the two ways // it refuses a caller that is not a running actor. // -// Both tests are negative, and that is the whole of the package on purpose. +// The suite covers front-door denial, live certificate transport, and XFCC spoof resistance. // - TestGatewayRefusesANonActorWorkload needs a credential no test process // can mint. The probe's podidentity certificate is issued by kubelet from // a real signer, so presenting it proves the gateway's downstream @@ -32,7 +32,6 @@ package egressauthz import ( "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -40,7 +39,6 @@ import ( "os" "path/filepath" "strings" - "sync" "testing" "time" @@ -63,7 +61,7 @@ const ( func TestGatewayRefusesANonActorWorkload(t *testing.T) { ctx := context.Background() - probe := sharedProbe(t, ctx) + probe := startProbe(t, ctx) const sni = "podidentity.example.com" result := probe.handshakeAs(t, ctx, sni, podIdentityCredentialPath) @@ -87,7 +85,7 @@ func TestGatewayRefusesANonActorWorkload(t *testing.T) { func TestGatewayRefusesAnUnknownActor(t *testing.T) { ctx := context.Background() - probe := sharedProbe(t, ctx) + probe := startProbe(t, ctx) const sni = "unknown.example.com" result := probe.handshakeAs(t, ctx, sni, unknownActorCredentialPath) @@ -115,36 +113,39 @@ type probeClient struct { http *http.Client } -var ( - probeOnce sync.Once - probeVal *probeClient - probeErr error -) - -// sharedProbe returns the one probe pod the whole suite uses. -func sharedProbe(t *testing.T, ctx context.Context) *probeClient { +func (c *probeClient) connectAs(t *testing.T, ctx context.Context, destination, credential, xfcc string) connectResult { t.Helper() - probeOnce.Do(func() { - // startProbe reports failures through t, which unwinds this goroutine - // without returning. Leave something behind so the tests that run - // afterwards fail pointing at the first one instead of dereferencing - // nil. - defer func() { - if probeVal == nil && probeErr == nil { - probeErr = errors.New("setup did not complete; see the failure reported by the first test that needed the probe") - } - }() - probeVal = startProbe(t, ctx) - }) - if probeErr != nil { - t.Fatalf("starting the shared egress probe: %v", probeErr) + endpoint := c.baseURL + "/connect?destination=" + url.QueryEscape(destination) + "&credential-bundle=" + url.QueryEscape(credential) + "&xfcc=" + url.QueryEscape(xfcc) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + t.Fatal(err) + } + resp, err := c.http.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out connectResult + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) } - return probeVal + return out +} + +type connectResult struct { + Credential string `json:"credential"` + Stage string `json:"stage"` + ConnectStatus int `json:"connect_status"` + Error string `json:"error"` } // startProbe creates the probe's namespace, mints its credentials there, builds // and deploys the probe, waits for it to be ready, and returns a client for it. func startProbe(t *testing.T, ctx context.Context) *probeClient { + return startProbeWithProvision(t, ctx, nil) +} + +func startProbeWithProvision(t *testing.T, ctx context.Context, provision func(string)) *probeClient { t.Helper() if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) @@ -152,6 +153,9 @@ func startProbe(t *testing.T, ctx context.Context) *probeClient { ns := e2e.CreateNamespace(t).Name provisionProbeCredentials(t, ctx, ns) + if provision != nil { + provision(ns) + } root, err := e2e.FindRepoRoot() if err != nil { t.Fatalf("FindRepoRoot: %v", err) @@ -187,7 +191,7 @@ func startProbe(t *testing.T, ctx context.Context) *probeClient { if err != nil { t.Fatalf("port-forwarding %s/%s: %v", ns, probeName, err) } - e2e.RegisterSuiteCleanup(stop) + t.Cleanup(stop) return &probeClient{ baseURL: fmt.Sprintf("http://127.0.0.1:%d", localPort), diff --git a/internal/e2e/suites/egressauthz/identity_transport_test.go b/internal/e2e/suites/egressauthz/identity_transport_test.go new file mode 100644 index 0000000000..2369e5b507 --- /dev/null +++ b/internal/e2e/suites/egressauthz/identity_transport_test.go @@ -0,0 +1,183 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egressauthz + +import ( + "context" + "encoding/pem" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGatewayCertificateMetadataTransport(t *testing.T) { + ctx := context.Background() + clients := e2e.GetClients() + origin := e2e.DeployServerPod(t, ctx, e2e.ServerPod{Name: "egress-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 8080}) + var liveUID string + var liveName = "live-" + strings.ToLower(strings.ReplaceAll(t.Name(), "/", "-")) + probe := startProbeWithProvision(t, ctx, func(ns string) { + at := e2e.CreateSubstrateCounterTemplate(ctx, t, clients, ns, e2e.SubstrateTemplateOptions{Atespace: ns, Name: "counter", PoolName: "counter", PoolReplicas: 1, Labels: map[string]string{"egressauthz": ns}}) + ref := &ateapipb.ObjectRef{Atespace: ns, Name: liveName} + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: ns, Name: liveName}, ActorTemplate: e2e.TemplateRef(at)}}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}) + }) + bits := 128 + if ip := net.ParseIP(origin.ClusterIP); ip != nil && ip.To4() != nil { + bits = 32 + } + e2e.EnsureEgressPolicy(t, ctx, clients, ref, e2e.EgressAllowCIDRs(origin.ClusterIP+"/"+strconv.Itoa(bits))) + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + got, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err == nil && got.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_RUNNING { + liveUID = got.GetMetadata().GetUid() + break + } + time.Sleep(time.Second) + } + if liveUID == "" { + t.Fatal("actor did not reach RUNNING") + } + identity := &substratex509.ActorIdentity{Atespace: ns, ActorName: liveName, ActorUid: liveUID, Purpose: substratex509.ActorIdentityPurposeAtunnel} + ca := actorIdentityCA(t, ctx) + writeCredentialSecret(t, ctx, ns, "egressprobe-live-actor", mintActorCredential(t, ca, identity)) + writeCredentialSecret(t, ctx, ns, "egressprobe-live-actor-chain", mintActorCredentialWithIntermediate(t, ca, identity)) + }) + destination := origin.Address() + result := probe.connectAs(t, ctx, destination, "/run/actor-identity-live/credential-bundle.pem", "") + if result.Stage != "" || result.ConnectStatus != http.StatusOK { + t.Fatalf("direct-root credential: got stage %q status %d: %s", result.Stage, result.ConnectStatus, result.Error) + } + t.Run("leaf-plus-intermediate (Envoy scope)", func(t *testing.T) { + if os.Getenv(e2e.AtenetDataplaneEnv) == "agentgateway" { + t.Skip("chain transport proof is scoped to the Envoy egress implementation") + } + result := probe.connectAs(t, ctx, destination, "/run/actor-identity-live-chain/credential-bundle.pem", "") + if os.Getenv("E2E_EGRESS_CERTIFICATE_LEAF_ONLY") == "1" { + if result.Stage != stageConnect || result.ConnectStatus != http.StatusForbidden { + t.Fatalf("leaf-only chain credential: got stage %q status %d: %s", result.Stage, result.ConnectStatus, result.Error) + } + return + } + if result.Stage != "" || result.ConnectStatus != http.StatusOK { + t.Fatalf("chain credential: got stage %q status %d: %s", result.Stage, result.ConnectStatus, result.Error) + } + }) +} + +func TestGatewayIgnoresSpoofedXFCC(t *testing.T) { + ctx := context.Background() + clients := e2e.GetClients() + origin := e2e.DeployServerPod(t, ctx, e2e.ServerPod{Name: "spoof-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 8080}) + var uid, name, livePEM string + var unknownPEM string + probe := startProbeWithProvision(t, ctx, func(ns string) { + name = "spoof-live-" + strings.ToLower(strings.ReplaceAll(t.Name(), "/", "-")) + at := e2e.CreateSubstrateCounterTemplate(ctx, t, clients, ns, e2e.SubstrateTemplateOptions{Atespace: ns, Name: "counter", PoolName: "counter", PoolReplicas: 1, Labels: map[string]string{"egressauthz": ns}}) + ref := &ateapipb.ObjectRef{Atespace: ns, Name: name} + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: ns, Name: name}, ActorTemplate: e2e.TemplateRef(at)}}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}) + }) + bits := 128 + if ip := net.ParseIP(origin.ClusterIP); ip != nil && ip.To4() != nil { + bits = 32 + } + e2e.EnsureEgressPolicy(t, ctx, clients, ref, e2e.EgressAllowCIDRs(origin.ClusterIP+"/"+strconv.Itoa(bits))) + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatal(err) + } + for deadline := time.Now().Add(2 * time.Minute); time.Now().Before(deadline); { + got, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err == nil && got.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_RUNNING { + uid = got.GetMetadata().GetUid() + break + } + time.Sleep(time.Second) + } + if uid == "" { + t.Fatal("actor did not reach RUNNING") + } + ca := actorIdentityCA(t, ctx) + live := &substratex509.ActorIdentity{Atespace: ns, ActorName: name, ActorUid: uid, Purpose: substratex509.ActorIdentityPurposeAtunnel} + wrong := *live + wrong.ActorUid = "00000000-0000-0000-0000-000000000000" + liveBundle := mintActorCredential(t, ca, live) + livePEM = certificatePEM(liveBundle) + writeCredentialSecret(t, ctx, ns, "egressprobe-live-actor", liveBundle) + writeCredentialSecret(t, ctx, ns, "egressprobe-wrong-uid", mintActorCredential(t, ca, &wrong)) + secret, err := clients.K8s.CoreV1().Secrets(ns).Get(ctx, unknownActorCredentialSecret, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + unknownPEM = certificatePEM(secret.Data[credentialBundleKey]) + if unknownPEM == "" { + t.Fatal("unknown certificate PEM is empty") + } + }) + if livePEM == "" { + t.Fatal("live certificate PEM is empty") + } + valid := `Chain="` + url.PathEscape(livePEM) + `"` + if result := probe.connectAs(t, ctx, origin.Address(), "/run/actor-identity-live/credential-bundle.pem", ""); result.ConnectStatus != http.StatusOK { + t.Fatalf("live actor got stage %q status %d: %s", result.Stage, result.ConnectStatus, result.Error) + } + if result := probe.connectAs(t, ctx, origin.Address(), "/run/actor-identity-live/credential-bundle.pem", "malformed"); result.ConnectStatus != http.StatusOK { + t.Fatalf("malformed XFCC changed live authorization: stage %q status %d: %s", result.Stage, result.ConnectStatus, result.Error) + } + if result := probe.connectAs(t, ctx, origin.Address(), "/run/actor-identity-live/credential-bundle.pem", `Chain="`+url.PathEscape(unknownPEM)+`"`); result.ConnectStatus != http.StatusOK { + t.Fatalf("live actor with unknown valid XFCC got stage %q status %d: %s", result.Stage, result.ConnectStatus, result.Error) + } + for _, credential := range []string{"/run/actor-identity-unknown/credential-bundle.pem", "/run/actor-identity-wrong-uid/credential-bundle.pem"} { + result := probe.connectAs(t, ctx, origin.Address(), credential, valid) + if result.Stage != stageConnect || result.ConnectStatus != http.StatusForbidden { + t.Fatalf("spoofed identity with %s got stage %q status %d: %s", credential, result.Stage, result.ConnectStatus, result.Error) + } + } +} + +func certificatePEM(bundle []byte) string { + for rest := bundle; ; { + block, tail := pem.Decode(rest) + if block == nil { + return "" + } + rest = tail + if block.Type == "CERTIFICATE" { + return string(pem.EncodeToMemory(block)) + } + } +} diff --git a/manifests/ate-install/atenet-egress-with-sdsmint.yaml b/manifests/ate-install/atenet-egress-with-sdsmint.yaml index 0f67d1be30..ef671545a4 100644 --- a/manifests/ate-install/atenet-egress-with-sdsmint.yaml +++ b/manifests/ate-install/atenet-egress-with-sdsmint.yaml @@ -102,10 +102,7 @@ data: codec_type: HTTP1 upgrade_configs: - upgrade_type: CONNECT - # TODO(liorlieberman): Can we make this cleaner? - forward_client_cert_details: SANITIZE_SET - set_current_client_cert_details: - chain: true + forward_client_cert_details: SANITIZE # Emit the access log as soon as the CONNECT tunnel is established # (atunnel keeps the tunnel open, so the default log-on-close would # not fire during the demo). @@ -172,6 +169,19 @@ data: text_format_source: inline_string: "%DOWNSTREAM_PEER_URI_SAN%" omit_empty_values: true + - name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + default_source_code: + inline_string: | + function envoy_on_request(handle) + local ssl = handle:streamInfo():downstreamSslConnection() + local chain = "" + if ssl ~= nil then + chain = ssl:urlEncodedPemEncodedPeerCertificateChain() + end + handle:streamInfo():dynamicMetadata():set("dev.ate.egress.peer_certificate", "chain", chain) + end # The CONNECT leg. The sidecar authenticates the actor from its # certificate and decides the address rules against the CONNECT # authority (see cmd/atenet/internal/router/egress). It answers with @@ -214,6 +224,9 @@ data: # Envoy silently drops metadata from a namespace not listed # here, and the filter state below would never be set. metadata_options: + forwarding_namespaces: + untyped: + - dev.ate.egress.peer_certificate receiving_namespaces: untyped: - dev.ate.egress diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 16de9a4a5b..89c2461f84 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -104,10 +104,7 @@ data: codec_type: HTTP1 upgrade_configs: - upgrade_type: CONNECT - # TODO(liorlieberman): Can we make this cleaner? - forward_client_cert_details: SANITIZE_SET - set_current_client_cert_details: - chain: true + forward_client_cert_details: SANITIZE # Emit the access log as soon as the CONNECT tunnel is established # (atunnel keeps the tunnel open, so the default log-on-close would # not fire during the demo). @@ -166,6 +163,19 @@ data: text_format_source: inline_string: "%DOWNSTREAM_PEER_URI_SAN%" omit_empty_values: true + - name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + default_source_code: + inline_string: | + function envoy_on_request(handle) + local ssl = handle:streamInfo():downstreamSslConnection() + local chain = "" + if ssl ~= nil then + chain = ssl:urlEncodedPemEncodedPeerCertificateChain() + end + handle:streamInfo():dynamicMetadata():set("dev.ate.egress.peer_certificate", "chain", chain) + end # The CONNECT leg. The sidecar authenticates the actor from its # certificate and decides the address rules against the CONNECT # authority (see cmd/atenet/internal/router/egress). It answers with @@ -208,6 +218,9 @@ data: # Envoy silently drops metadata from a namespace not listed # here, and the filter state below would never be set. metadata_options: + forwarding_namespaces: + untyped: + - dev.ate.egress.peer_certificate receiving_namespaces: untyped: - dev.ate.egress