diff --git a/Scripts/sf-client-eku-test.csx b/Scripts/sf-client-eku-test.csx new file mode 100644 index 00000000..ad0c4332 --- /dev/null +++ b/Scripts/sf-client-eku-test.csx @@ -0,0 +1,321 @@ +#!/usr/bin/env dotnet-script +// ============================================================================ +// sf-client-eku-test.csx +// +// C# script to test client certificate authentication behavior with Service +// Fabric clusters when using certificates with only Server Authentication EKU +// (missing Client Authentication EKU 1.3.6.1.5.5.7.3.2). +// +// IMPORTANT: Each test runs in a SEPARATE PROCESS to prevent Windows SChannel +// TLS session caching from contaminating results. SChannel caches successful +// TLS sessions, so a working connection (e.g. SslStreamCertificateContext) can +// cause a subsequent HttpClientHandler test to succeed when it should fail. +// +// Prerequisites: +// - .NET 8+ SDK installed +// - dotnet-script tool: dotnet tool install -g dotnet-script +// - Client certificate installed in CurrentUser\My certificate store +// +// Usage: +// dotnet script sf-client-eku-test.csx -- +// +// Example: +// dotnet script sf-client-eku-test.csx -- mycluster.eastus.cloudapp.azure.com ABC123DEF456 +// +// Run a single test only (used internally for process isolation): +// dotnet script sf-client-eku-test.csx -- --test 1 +// +// See also: +// Security/certificate-client-authentication-eku-removal-impact.md +// ============================================================================ + +using System.Diagnostics; +using System.Net.Http; +using System.Net.Security; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; + +// --- parse arguments --- +if (Args.Count < 2) +{ + Console.WriteLine("Usage: dotnet script sf-client-eku-test.csx -- "); + Console.WriteLine("Example: dotnet script sf-client-eku-test.csx -- mycluster.eastus.cloudapp.azure.com ABC123DEF456"); + return; +} + +string clusterFqdn = Args[0]; +string thumbprint = Args[1].ToUpperInvariant(); +string baseUrl = $"https://{clusterFqdn}:19080"; +string testPath = "/$/GetClusterHealth?api-version=9.1&timeout=10"; + +// check if running a single test (child process mode) +int singleTest = 0; +if (Args.Count >= 4 && Args[2] == "--test" && int.TryParse(Args[3], out int t)) +{ + singleTest = t; +} + +// --- find certificate --- +X509Certificate2 cert = null; +using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser)) +{ + store.Open(OpenFlags.ReadOnly); + var matches = store.Certificates.Find(X509FindType.FindByThumbprint, thumbprint, false); + if (matches.Count == 0) + { + Console.WriteLine($"ERROR: Certificate with thumbprint {thumbprint} not found in CurrentUser\\My"); + return; + } + cert = matches[0]; +} + +// server cert validation callback (accept self-signed cluster certs for testing) +bool ServerCertValidation(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) +{ + if (errors == SslPolicyErrors.None) return true; + // Accept self-signed certs: the cluster's server cert may differ from the client cert being tested. + // For testing purposes, accept any cert where the only error is untrusted root or name mismatch. + if (errors == SslPolicyErrors.RemoteCertificateChainErrors || errors == SslPolicyErrors.RemoteCertificateNameMismatch + || errors == (SslPolicyErrors.RemoteCertificateChainErrors | SslPolicyErrors.RemoteCertificateNameMismatch)) + { + return true; + } + return false; +} + +// ============================================================================ +// ORCHESTRATOR MODE: spawn each test in its own process +// ============================================================================ +if (singleTest == 0) +{ + Console.WriteLine($"Cluster: {baseUrl}"); + Console.WriteLine($"Thumbprint: {thumbprint}"); + Console.WriteLine(); + Console.WriteLine($"Subject: {cert.Subject}"); + Console.WriteLine($"HasPrivKey: {cert.HasPrivateKey}"); + Console.WriteLine($"NotAfter: {cert.NotAfter}"); + + // display EKU info + bool hasClientEku = false; + foreach (var ext in cert.Extensions) + { + if (ext is X509EnhancedKeyUsageExtension ekuExt) + { + Console.WriteLine("EKU:"); + foreach (var oid in ekuExt.EnhancedKeyUsages) + { + string label = oid.Value switch + { + "1.3.6.1.5.5.7.3.1" => "Server Authentication", + "1.3.6.1.5.5.7.3.2" => "Client Authentication", + _ => oid.FriendlyName + }; + Console.WriteLine($" - {label} ({oid.Value})"); + if (oid.Value == "1.3.6.1.5.5.7.3.2") hasClientEku = true; + } + } + } + + if (!hasClientEku) + { + Console.WriteLine(); + Console.WriteLine("NOTE: Certificate does NOT have Client Authentication EKU."); + Console.WriteLine(" Tests below will show which .NET approaches still work."); + } + else + { + Console.WriteLine(); + Console.WriteLine("NOTE: Certificate HAS Client Authentication EKU. All tests should pass."); + } + + Console.WriteLine(); + Console.WriteLine("Each test runs in a SEPARATE PROCESS to prevent SChannel TLS session"); + Console.WriteLine("cache from contaminating results between tests."); + Console.WriteLine(); + Console.WriteLine(new string('=', 70)); + + // get the path to this script + string scriptPath = Path.GetFullPath("sf-client-eku-test.csx"); + // if that doesn't exist, try to find it via the first arg in the process command line + if (!File.Exists(scriptPath)) + { + string cmdLine = Environment.CommandLine; + // look for .csx in the command line + int csxIdx = cmdLine.IndexOf(".csx", StringComparison.OrdinalIgnoreCase); + if (csxIdx >= 0) + { + int start = cmdLine.LastIndexOf('"', csxIdx); + if (start < 0) start = cmdLine.LastIndexOf(' ', csxIdx); + scriptPath = cmdLine.Substring(start + 1, csxIdx + 4 - start - 1).Trim('"'); + } + } + + // run each test in its own process + int[] testResults = new int[3]; // 0 = unknown, 1 = PASS, 2 = FAIL + for (int testNum = 1; testNum <= 3; testNum++) + { + var psi = new ProcessStartInfo + { + FileName = "dotnet", + Arguments = $"script \"{scriptPath}\" -- {clusterFqdn} {thumbprint} --test {testNum}", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi); + string output = await proc.StandardOutput.ReadToEndAsync(); + string errors = await proc.StandardError.ReadToEndAsync(); + await proc.WaitForExitAsync(); + + Console.Write(output); + if (!string.IsNullOrEmpty(errors)) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.Write(errors); + Console.ResetColor(); + } + + if (output.Contains("RESULT: PASS")) testResults[testNum - 1] = 1; + else testResults[testNum - 1] = 2; + } + + Console.WriteLine(new string('=', 70)); + Console.WriteLine(); + Console.WriteLine("Summary (each test ran in isolated process - no TLS session cache sharing):"); + Console.WriteLine($" Test 1 (HttpClientHandler.ClientCertificates): {(testResults[0] == 1 ? "PASS" : "FAIL")}"); + Console.WriteLine($" Test 2 (SocketsHttpHandler + Callback, .NET 8+): {(testResults[1] == 1 ? "PASS" : "FAIL")}"); + Console.WriteLine($" Test 3 (SslStreamCertificateContext, .NET 8+): {(testResults[2] == 1 ? "PASS" : "FAIL")}"); + Console.WriteLine(); + Console.WriteLine("If your certificate has only Server Authentication EKU:"); + Console.WriteLine(" - Test 1 should FAIL (SChannel silently drops the certificate)"); + Console.WriteLine(" - Tests 2 and 3 should PASS on .NET 8+ (bypasses SChannel EKU filtering)"); + Console.WriteLine(); + Console.WriteLine("For more information, see:"); + Console.WriteLine(" Security/certificate-client-authentication-eku-removal-impact.md"); + + return; +} + +// ============================================================================ +// CHILD PROCESS MODE: run a single test +// ============================================================================ + +if (singleTest == 1) +{ + // --- Test 1: HttpClientHandler.ClientCertificates --- + Console.WriteLine(); + Console.WriteLine($"TEST 1: HttpClientHandler.ClientCertificates (PID {Environment.ProcessId})"); + Console.WriteLine(" Expected with server-only EKU: FAIL (SChannel drops cert)"); + Console.WriteLine(); + try + { + var handler = new HttpClientHandler(); + handler.ClientCertificates.Add(cert); + handler.ServerCertificateCustomValidationCallback = (msg, c, ch, e) => ServerCertValidation(msg, c, ch, e); + + using var client = new HttpClient(handler); + client.Timeout = TimeSpan.FromSeconds(15); + var response = await client.GetAsync($"{baseUrl}{testPath}"); + Console.WriteLine($" HTTP: {(int)response.StatusCode} {response.StatusCode}"); + if (response.IsSuccessStatusCode) + { + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine($" Body: {body.Substring(0, Math.Min(200, body.Length))}..."); + Console.WriteLine(" RESULT: PASS"); + } + else + { + Console.WriteLine(" RESULT: FAIL (server rejected or cert not sent)"); + } + } + catch (Exception ex) + { + Console.WriteLine($" ERROR: {ex.GetType().Name}: {ex.Message}"); + Console.WriteLine(" RESULT: FAIL"); + } + Console.WriteLine(); +} +else if (singleTest == 2) +{ + // --- Test 2: SocketsHttpHandler + LocalCertificateSelectionCallback --- + Console.WriteLine(); + Console.WriteLine($"TEST 2: SocketsHttpHandler + LocalCertificateSelectionCallback (PID {Environment.ProcessId})"); + Console.WriteLine(" Expected with server-only EKU: PASS on .NET 8+ (SocketsHttpHandler bypasses SChannel filtering)"); + Console.WriteLine(); + try + { + var handler = new SocketsHttpHandler(); + handler.SslOptions = new SslClientAuthenticationOptions + { + ClientCertificates = new X509CertificateCollection { cert }, + LocalCertificateSelectionCallback = (sender, host, certs, remoteCert, issuers) => + { + Console.WriteLine($" Callback invoked. Certs available: {certs.Count}"); + return certs.Count > 0 ? certs[0] : null; + }, + RemoteCertificateValidationCallback = (sender, c, ch, e) => ServerCertValidation(sender, c, ch, e) + }; + + using var client = new HttpClient(handler); + client.Timeout = TimeSpan.FromSeconds(15); + var response = await client.GetAsync($"{baseUrl}{testPath}"); + Console.WriteLine($" HTTP: {(int)response.StatusCode} {response.StatusCode}"); + if (response.IsSuccessStatusCode) + { + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine($" Body: {body.Substring(0, Math.Min(200, body.Length))}..."); + Console.WriteLine(" RESULT: PASS"); + } + else + { + Console.WriteLine(" RESULT: FAIL (server rejected or cert not sent)"); + } + } + catch (Exception ex) + { + Console.WriteLine($" ERROR: {ex.GetType().Name}: {ex.Message}"); + Console.WriteLine(" RESULT: FAIL"); + } + Console.WriteLine(); +} +else if (singleTest == 3) +{ + // --- Test 3: SslStreamCertificateContext (.NET 8+ workaround) --- + Console.WriteLine(); + Console.WriteLine($"TEST 3: SslStreamCertificateContext (ClientCertificateContext) - .NET 8+ (PID {Environment.ProcessId})"); + Console.WriteLine(" Expected with server-only EKU: PASS (bypasses SChannel filtering)"); + Console.WriteLine(); + try + { + var certContext = SslStreamCertificateContext.Create(cert, additionalCertificates: null, offline: true); + var handler = new SocketsHttpHandler(); + handler.SslOptions = new SslClientAuthenticationOptions + { + ClientCertificateContext = certContext, + RemoteCertificateValidationCallback = (sender, c, ch, e) => ServerCertValidation(sender, c, ch, e) + }; + + using var client = new HttpClient(handler); + client.Timeout = TimeSpan.FromSeconds(15); + var response = await client.GetAsync($"{baseUrl}{testPath}"); + Console.WriteLine($" HTTP: {(int)response.StatusCode} {response.StatusCode}"); + if (response.IsSuccessStatusCode) + { + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine($" Body: {body.Substring(0, Math.Min(200, body.Length))}..."); + Console.WriteLine(" RESULT: PASS"); + } + else + { + Console.WriteLine($" RESULT: FAIL ({(int)response.StatusCode})"); + } + } + catch (Exception ex) + { + Console.WriteLine($" ERROR: {ex.GetType().Name}: {ex.Message}"); + Console.WriteLine(" RESULT: FAIL"); + } + Console.WriteLine(); +} diff --git a/Security/certificate-client-authentication-eku-removal-impact.md b/Security/certificate-client-authentication-eku-removal-impact.md index 4a5c521e..2e00ffcc 100644 --- a/Security/certificate-client-authentication-eku-removal-impact.md +++ b/Security/certificate-client-authentication-eku-removal-impact.md @@ -2,25 +2,44 @@ ## Problem Description -Public certificate authorities (Microsoft, DigiCert) are removing the Client Authentication EKU (OID: 1.3.6.1.5.5.7.3.2) from public TLS certificates. While Service Fabric's server-side validation doesn't require this EKU, **browsers will not display certificates without it** in certificate selection dialogs, blocking access to Service Fabric Explorer web UI. +Public certificate authorities (Microsoft, DigiCert) are removing the Client Authentication EKU (OID: 1.3.6.1.5.5.7.3.2) from public TLS certificates. Service Fabric's server-side validation does not check or require this EKU (see [SF Server-Side EKU Behavior](#sf-server-side-eku-behavior)). The issue is entirely **client-side**: Windows SChannel refuses to present certificates without Client Auth EKU during the TLS handshake, so the certificate never reaches the server. This affects browsers, .NET `HttpClientHandler`, and any client that relies on SChannel for client certificate selection. **Primary Impact:** - **Browser-based SFX access**: Users cannot select client certificates in browsers -- **Clusters with REST clients connecting to http gateway port 19080**: Client certificate authentication to Service Fabric backends may fail +- **Clusters with REST clients connecting to http gateway port 19080**: Client certificate authentication fails when client uses Windows SChannel (e.g., .NET `HttpClientHandler`, APIM) - **Clusters with Managed Identity Token Service**: Managed Identity authentication failures (HTTP 403) -**What Still Works:** +**What Still Works (TCP-based connections - port 19000):** -- PowerShell, SDK, REST APIs, CLI tools -- Service Fabric Explorer standalone application +- Service Fabric SDK (`Connect-ServiceFabricCluster`) - uses TCP transport, not HTTP +- Service Fabric Explorer desktop application - Node-to-node communication - Core cluster operations +- .NET `SocketsHttpHandler` + `LocalCertificateSelectionCallback` workaround (.NET 8+) +- .NET `SslStreamCertificateContext` workaround (.NET 8+) +- PowerShell 7.4+ (pwsh) `Invoke-WebRequest` / `Invoke-RestMethod` on .NET 8+ (uses `SocketsHttpHandler` internally) +- Node.js HTTPS clients (uses OpenSSL, not SChannel) +- Python `urllib` / `requests` (uses OpenSSL, not SChannel) +- curl Windows (SChannel build, cert store access) + +**What Is Impacted (HTTP-based connections - port 19080):** + +- Browser-based SFX access (certificate not shown in selection dialog) +- .NET `HttpClientHandler.ClientCertificates` (Windows SChannel filters cert) +- PowerShell 5.1 `Invoke-WebRequest` / `Invoke-RestMethod` (uses .NET Framework HttpClientHandler) +- PowerShell 7 (pwsh) `Invoke-RestMethod` / `Invoke-WebRequest` with `-Certificate` parameter on **.NET 7 and earlier** (uses `HttpClientHandler.ClientCertificates` which goes through SChannel EKU filtering) + +> [!NOTE] +> **PowerShell 7 on .NET 8+:** PowerShell 7.4+ running on .NET 8+ internally uses `SocketsHttpHandler` for `Invoke-WebRequest` and `Invoke-RestMethod`, which bypasses SChannel EKU filtering. Server-only EKU certificates **work** in PS 7.4+ on .NET 8+. See [Client Compatibility Matrix](#client-compatibility-matrix) for verified results. +- `Connect-SFCluster` (SF HTTP PowerShell module) +- Azure API Management (APIM) Service Fabric backend (uses [`Microsoft.ServiceFabric.Client.Http`](https://github.com/microsoft/service-fabric-client-dotnet) with `HttpClientHandler.ClientCertificates`) +- MITS (Managed Identity Token Service) ### Timeline - **Microsoft certificate services**: September 15 - November 9, 2025 -- **DigiCert**: Started October 1, 2025; ends May 1, 2026 +- **DigiCert**: Started October 1, 2025; ends March 1, 2027 (extended per Google Chrome Root Program update) - **Impact**: Begins immediately upon certificate renewal ### Why This Is Happening @@ -29,8 +48,8 @@ Public CAs are standardizing TLS certificates to include only Server Authenticat **Industry Context:** -- **[Chrome Root Program policy (Section 3.2.2)](https://www.chromium.org/Home/chromium-security/root-ca-policy/#322-pki-hierarchies-included-in-the-chrome-root-store)** requires Certificate Authorities to use dedicated TLS root hierarchies -- Chrome restricts TLS certificates to Server Authentication EKU only starting June 15, 2026 +- **[Chrome Root Program policy](https://googlechrome.github.io/chromerootprogram/)** requires Certificate Authorities to use dedicated TLS root hierarchies +- Chrome restricts TLS certificates to Server Authentication EKU only starting March 15, 2026 - DigiCert and other public CAs are implementing this change: [DigiCert announcement](https://knowledge.digicert.com/alerts/sunsetting-client-authentication-eku-from-digicert-public-tls-certificates) - **CA/Browser Forum** baseline requirements align with Chrome's policy for public trust - Private PKI and self-signed certificates remain free to issue certificates with both EKUs for internal mutual TLS scenarios @@ -58,11 +77,12 @@ This bidirectional authentication provides stronger security for sensitive opera **Why Client Authentication EKU is Required:** - MITS presents the cluster certificate AS a **client certificate** when querying http gateway (REST) -- Service Fabric management endpoint validates the certificate must have Client Authentication EKU (1.3.6.1.5.5.7.3.2) for mTLS -- Without this EKU, the server rejects the client certificate, resulting in HTTP 403 +- Windows SChannel (the TLS provider used by MITS's .NET HTTP client) checks for Client Authentication EKU (1.3.6.1.5.5.7.3.2) during client certificate selection +- Without this EKU, SChannel silently drops the certificate during the TLS handshake -- the certificate is never sent to the server +- The server receives no client certificate and returns HTTP 403 "Client certificate required" > [!NOTE] -> This is a unique scenario where the **cluster certificate** (normally used for server authentication) is also used as a **client certificate** for mTLS. This dual purpose requires both Server Authentication and Client Authentication EKUs. +> Service Fabric's server-side validation does NOT check EKU on incoming client certificates (see [SF Server-Side EKU Behavior](#sf-server-side-eku-behavior)). The filtering happens entirely on the **client side** in Windows SChannel. This is a unique scenario where the **cluster certificate** (normally used for server authentication) is also used as a **client certificate** for mTLS. This dual purpose requires both Server Authentication and Client Authentication EKUs. **Error:** @@ -89,10 +109,21 @@ This is a security measure: the browser ensures only certificates explicitly aut **Platform-Specific Behavior:** -- **Edge/Chrome (Windows):** Uses Windows Schannel - strictly filters by Client Authentication EKU -- **Firefox:** Uses NSS (Network Security Services) - may have different filtering behavior but typically requires Client Authentication EKU -- **.NET/HttpClient:** May reject certificates missing Client Authentication EKU for client certificate authentication -- **PowerShell/Azure CLI:** Work correctly - use certificate store directly, not browser filtering +- **Edge/Chrome (Windows):** Uses Windows SChannel - strictly filters by Client Authentication EKU +- **Firefox:** Uses NSS (Network Security Services) - filters by Client Authentication EKU +- **.NET HttpClient (Windows):** Uses SChannel for TLS. SChannel silently drops client certificates missing Client Authentication EKU during the TLS handshake. This affects `HttpClientHandler.ClientCertificates` and PowerShell 5.1 `Invoke-WebRequest`/`Invoke-RestMethod`. +- **.NET 8+ SocketsHttpHandler (default in PS 7.4+):** PowerShell 7.4+ on .NET 8+ uses `SocketsHttpHandler` internally for `Invoke-WebRequest` and `Invoke-RestMethod`, which performs TLS via managed code instead of SChannel. This **bypasses SChannel EKU filtering**, so server-only EKU certificates work. Verified with PS 7.5.4 on .NET 9.0.10. +- **.NET SocketsHttpHandler + callback (.NET 8+):** `SocketsHttpHandler` with `LocalCertificateSelectionCallback` bypasses SChannel EKU filtering on .NET 8+. +- **.NET SslStreamCertificateContext (.NET 8+):** Bypasses SChannel EKU filtering. See [Client Compatibility Matrix](#client-compatibility-matrix) for details. +- **Service Fabric SDK (`Connect-ServiceFabricCluster`):** Uses TCP transport (port 19000), not HTTP. Works with server-only EKU certificates. +- **PowerShell 5.1 `Invoke-RestMethod` / `Invoke-WebRequest`:** Uses .NET Framework HttpClient with SChannel. SChannel drops the cert. **FAILS.** +- **PowerShell 7.4+ (pwsh) `Invoke-RestMethod` / `Invoke-WebRequest` on .NET 8+:** Uses `SocketsHttpHandler` internally, which bypasses SChannel EKU filtering via managed TLS. **WORKS.** Verified: PS 7.5.4 on .NET 9.0.10 — both `Invoke-WebRequest` and `Invoke-RestMethod` with `-Certificate` parameter successfully authenticate with a server-only EKU certificate. In the same process, `HttpClientHandler.ClientCertificates` (SChannel path) returns 403 while `Invoke-RestMethod` returns 200, proving different code paths. +- **PowerShell 7 on .NET 7 and earlier:** Uses `HttpClientHandler.ClientCertificates` (SChannel path). **FAILS** — same behavior as PS 5.1. +- **Node.js:** Uses OpenSSL, not Windows SChannel. Works with server-only EKU certificates. +- **Python (`urllib`, `requests`):** Uses OpenSSL (`ssl` module), not Windows SChannel. Works with server-only EKU certificates. +- **curl (Windows/SChannel build):** Sends client certificates from the Windows certificate store without EKU filtering. Works with server-only EKU certificates. + +See [Client Compatibility Matrix](#client-compatibility-matrix) for a complete list of verified client behaviors. **Result:** The browser certificate selection dialog appears empty or does not show your cluster certificate, preventing SFX web access even though the certificate exists in your certificate store. @@ -100,6 +131,88 @@ This is a security measure: the browser ensures only certificates explicitly aut **Note:** Node-to-node communication and core cluster operations are unaffected. +## Client Compatibility Matrix + +The following table shows which clients work with certificates that have **only** Server Authentication EKU (no Client Authentication EKU). Results verified against a live Service Fabric cluster configured with a server-only EKU certificate. + +### Verified Results + +| Client / Method | Port | Transport | Works? | Notes | +|---|---|---|---|---| +| `Connect-ServiceFabricCluster` (SF SDK) | 19000 | TCP | **Yes** | Uses `System.Fabric.FabricClient` over TCP, bypasses SChannel HTTP filtering | +| Service Fabric Explorer (Desktop App) | 19080 | HTTP | **Yes** | Native app, does not filter certificates by EKU | +| Node.js HTTPS client | 19080 | HTTP | **Yes** | Uses OpenSSL (bundled), not Windows SChannel | +| Python (`urllib`, `requests`) | 19080 | HTTP | **Yes** | Uses OpenSSL (`ssl` module), not Windows SChannel | +| curl (Windows/SChannel build) | 19080 | HTTP | **Yes** | Sends cert from Windows cert store without EKU filtering | +| .NET `SslStreamCertificateContext` (.NET 8+) | 19080 | HTTP | **Yes** | Bypasses SChannel EKU filtering via `ClientCertificateContext` property | +| .NET `SocketsHttpHandler` + callback (.NET 8+) | 19080 | HTTP | **Yes** | `SocketsHttpHandler` on .NET 8+ bypasses SChannel EKU filtering when using `LocalCertificateSelectionCallback` | +| PowerShell 5.1 `Invoke-WebRequest` | 19080 | HTTP | **No** | Uses .NET Framework HttpClient; SChannel drops cert | +| PowerShell 5.1 `Invoke-RestMethod` | 19080 | HTTP | **No** | Same as `Invoke-WebRequest` | +| PowerShell 7.4+ (pwsh) `Invoke-RestMethod` (.NET 8+) | 19080 | HTTP | **Yes** | Uses `SocketsHttpHandler` internally on .NET 8+, bypasses SChannel EKU filtering. Verified: PS 7.5.4 / .NET 9.0.10 | +| PowerShell 7.4+ (pwsh) `Invoke-WebRequest` (.NET 8+) | 19080 | HTTP | **Yes** | Same as `Invoke-RestMethod` — both use `SocketsHttpHandler` on .NET 8+ | +| PowerShell 7 (pwsh) on .NET 7 or earlier | 19080 | HTTP | **No** | `-Certificate` param uses `HttpClientHandler.ClientCertificates`; SChannel drops cert | +| .NET `HttpClientHandler.ClientCertificates` | 19080 | HTTP | **No** | SChannel silently removes cert without Client Auth EKU during TLS handshake | +| `Connect-SFCluster` (SF HTTP PowerShell module) | 19080 | HTTP | **No** | Uses .NET HttpClient, same SChannel filtering | +| Azure API Management (APIM) SF backend | 19080 | HTTP | **No** | Uses [`Microsoft.ServiceFabric.Client.Http`](https://github.com/microsoft/service-fabric-client-dotnet) which adds cert via `HttpClientHandler.ClientCertificates` (SChannel filters it) | +| Browsers (Edge, Chrome) | 19080 | HTTP | **No** | SChannel filters certificate selection dialog | +| Firefox | 19080 | HTTP | **No** | NSS filters by Client Authentication EKU | +| MITS (Managed Identity Token Service) | 19080 | HTTP | **No** | Requires Client Auth EKU for mTLS | + +### Key Finding: TCP vs HTTP + +- **TCP connections (port 19000):** The Service Fabric SDK uses `System.Fabric.FabricClient` which communicates over TCP. The TCP transport layer does **not** filter client certificates by EKU, so server-only EKU certificates work. +- **HTTP connections (port 19080):** HTTP clients on Windows that use **SChannel** (the Windows TLS implementation) for the TLS handshake enforce EKU filtering: when presenting a client certificate, SChannel checks for Client Authentication EKU (1.3.6.1.5.5.7.3.2) and silently drops certificates that lack it. However, clients that use **OpenSSL** (Node.js, Python), .NET 8+ `SocketsHttpHandler` with `SslStreamCertificateContext`, or **PowerShell 7.4+ on .NET 8+** (which uses `SocketsHttpHandler` internally) bypass this filtering entirely. + +> [!WARNING] +> **SChannel TLS Session Caching:** Windows SChannel caches successful TLS sessions at the OS level (in lsass.exe). If a client that bypasses SChannel EKU filtering (e.g., PS 7.4+, Node.js) successfully connects first, subsequent connections from SChannel-based clients (e.g., PS 5.1) may succeed by reusing the cached TLS session. Always test in **process-isolated** environments and be aware that cache contamination can produce false positives. See [Scripts/sf-client-eku-test.csx](../Scripts/sf-client-eku-test.csx) which uses per-test process isolation for this reason. + +### SF Server-Side EKU Behavior + +Service Fabric's server-side certificate validation does **NOT** check or enforce Client Authentication EKU on incoming client certificates. The EKU filtering that causes failures is entirely a **client-side Windows SChannel behavior** - SChannel refuses to present certificates without Client Auth EKU during the TLS handshake, so the SF server never even receives the certificate. + +**HTTP Gateway (port 19080):** + +The HTTP gateway validates incoming client certificates by checking: + +- Certificate thumbprint matching against configured admin/readonly client certificates +- Common name (CN) and issuer matching against configured X509 names +- Certificate chain validity + +**No EKU check exists in this code path.** The gateway accepts any certificate that matches the configured thumbprint or CN/issuer, regardless of which EKUs it contains. + +**TLS Transport Layer (port 19000 and internal):** + +The TLS transport layer examines EKU only for **routing** - to determine whether to validate through the client auth or server auth code path: + +1. If the cert has Client Auth EKU: validates as client authentication +2. If the cert does **not** have Client Auth EKU: skips the client auth path, falls through to server auth validation +3. Server auth validation explicitly does NOT require Server Auth EKU for backward compatibility + +In both cases, the cert is **accepted** if it matches the configured thumbprint or CN/issuer. A cert with only Server Auth EKU (or no EKU at all) still passes validation on the server side. + +**SF Certificate Generation:** + +When SF generates certificates internally, it adds **both** Server Auth and Client Auth EKU - confirming that the expected configuration is to have both EKUs present. + +**Bottom line:** The problem is not that SF rejects server-only EKU certs. SF accepts them. The problem is that Windows SChannel on the **client side** never sends them during the TLS handshake, so they never reach SF for validation. + +### .NET Workaround: SslStreamCertificateContext (.NET 8+) + +On .NET 8 and later, the `SslStreamCertificateContext.Create()` API bypasses SChannel's EKU filtering by providing the certificate directly to the TLS layer: + +```csharp +var handler = new SocketsHttpHandler(); +handler.SslOptions = new SslClientAuthenticationOptions +{ + ClientCertificateContext = SslStreamCertificateContext.Create(cert, additionalCerts: null, offline: true) +}; +var client = new HttpClient(handler); +``` + +This works because `ClientCertificateContext` provides the certificate context directly to the TLS layer, bypassing the SChannel certificate selection that normally filters by EKU. + +See [Scripts/sf-client-eku-test.csx](../Scripts/sf-client-eku-test.csx) for a complete test script customers can run to verify client certificate behavior with their own cluster. + ## Verify Certificate EKU 1. **Download certificate from Azure Key Vault** (CER format) @@ -253,7 +366,7 @@ If browser-based SFX cannot be used due to Client Authentication EKU limitations ### Use Private PKI (Recommended) -Public CAs will stop issuing certificates with Client Authentication EKU after May 2026. Use a private PKI that can issue certificates with both EKUs for internal clusters. +Public CAs will stop issuing certificates with Client Authentication EKU (DigiCert by March 2027, others on varying timelines). Use a private PKI that can issue certificates with both EKUs for internal clusters. > [!IMPORTANT] > **Certificate Configuration Requirements:**