From 724a40a6366404da8652a3089cc00f04977e5c5a Mon Sep 17 00:00:00 2001 From: maxlisongsong Date: Thu, 16 Jul 2026 13:07:02 +0800 Subject: [PATCH 001/107] [fix][broker] Fix ownership-generation races in OwnershipCache removeOwnership and lock-expiry cleanup --- .../channel/ServiceUnitStateChannelImpl.java | 23 +- .../broker/namespace/NamespaceService.java | 20 +- .../pulsar/broker/namespace/OwnedBundle.java | 59 ++++- .../broker/namespace/OwnershipCache.java | 187 ++++++++++++- .../pulsar/broker/service/BrokerService.java | 144 ++++++---- .../broker/namespace/OwnershipCacheTest.java | 247 ++++++++++++++++++ .../broker/service/BrokerServiceTest.java | 79 ++++++ 7 files changed, 678 insertions(+), 81 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 3f5bd2569faf9..5834fa4c61d78 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -87,6 +87,7 @@ import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Schema; @@ -1071,12 +1072,28 @@ private CompletableFuture closeServiceUnit(String serviceUnit, boolean long startTime = System.nanoTime(); MutableInt unloadedTopics = new MutableInt(); NamespaceBundle bundle = LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit); + // Capture the topic futures for this bundle once, before unloading starts, and reuse the same snapshot + // both to close the topics below and to clean up afterward, so the cleanup is identity-safe: a stale + // cleanup call must only remove what this snapshot observed, never a newer ownership generation's topic + // installed for the same bundle in the meantime. + // + // A straggler topic loaded into BrokerService's topic cache for this bundle *after* the snapshot is + // taken is deliberately left alone: neither closed below nor evicted by the cleanup. The alternative — + // force-evicting whatever is in the cache at cleanup time regardless of identity — could yank a topic + // that was never closed, letting a second, independent instance for the same name be created on the + // next load. A straggler surviving the unload does not need this unload to fix it: new connections + // converge through the normal lookup path once ownership has actually moved, and BookKeeper's ledger + // fencing (see ManagedLedgerImpl#addEntryFailedDueToConcurrentlyModified) is what prevents it from + // silently double-writing if a new owner's ManagedLedger instance does start writing the same topic. + Map>> topicFutures = + pulsar.getBrokerService().getTopicFuturesInBundle(bundle); return pulsar.getBrokerService().unloadServiceUnit( bundle, disconnectClients, true, pulsar.getConfig().getNamespaceBundleUnloadingTimeoutMs(), - TimeUnit.MILLISECONDS) + TimeUnit.MILLISECONDS, + topicFutures) .thenApply(numUnloadedTopics -> { unloadedTopics.setValue(numUnloadedTopics); return numUnloadedTopics; @@ -1084,7 +1101,7 @@ private CompletableFuture closeServiceUnit(String serviceUnit, boolean .whenComplete((__, ex) -> { if (disconnectClients) { // clean up topics that failed to unload from the broker ownership cache - pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle); + pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle, topicFutures); } pulsar.getNamespaceService().onNamespaceBundleUnload(bundle); double unloadBundleTime = TimeUnit.NANOSECONDS @@ -1094,7 +1111,7 @@ private CompletableFuture closeServiceUnit(String serviceUnit, boolean .exception(ex) .log("Failed to close topics under bundle"); if (!disconnectClients) { - pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle); + pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle, topicFutures); } } else { log.info().attr("bundle", bundle).attr("topicCount", unloadedTopics) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 91632c1ee4cd7..1c0313f065494 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1052,12 +1052,24 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, // success updateNamespaceBundles // disable old bundle in memory getOwnershipCache().updateBundleState(bundle, false) - .thenRun(() -> { + .thenCompose(__ -> { // update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); loadManager.get().setLoadReportForceUpdateFlag(); - // release old bundle from ownership cache - pulsar.getNamespaceService().getOwnershipCache().removeOwnership(bundle); + // Release old bundle from ownership cache. Compose on the returned future instead of + // discarding it, so a delayed or failed release is observed here rather than letting + // completionFuture complete while the release may still be in flight; a release + // failure is logged and does not fail the split, which has already succeeded. + return pulsar.getNamespaceService().getOwnershipCache().removeOwnership(bundle) + .exceptionally(ex1 -> { + log.warn() + .attr("bundle", bundle.toString()) + .exception(ex1) + .log("Failed to release ownership of the old bundle after split"); + return null; + }); + }) + .thenRun(() -> { completionFuture.complete(null); if (unload) { // Unload new split bundles, in background. This will not @@ -1069,7 +1081,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, .exceptionally(e -> { String msg1 = format( "failed to disable bundle %s under namespace [%s] with error %s", - bundle.getNamespaceObject().toString(), bundle, ex.getMessage()); + bundle.getNamespaceObject().toString(), bundle, e.getMessage()); log.warn().exception(e).log(msg1); completionFuture.completeExceptionally(new ServiceUnitNotReadyException(msg1)); return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java index 71fd8efa1aec9..c1580108459f6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.namespace; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -27,8 +29,10 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.coordination.ResourceLock; @EqualsAndHashCode @ToString @@ -36,6 +40,15 @@ public class OwnedBundle { private final NamespaceBundle bundle; + /** + * The resource lock acquired for this ownership. Ties this instance to a single ownership generation: a + * bundle can be re-acquired (new lock, new {@link OwnedBundle}) after this instance's lock expired, and + * cleanup done through this instance must never touch the newer generation. + */ + @ToString.Exclude + @EqualsAndHashCode.Exclude + private final ResourceLock resourceLock; + /** * {@link #nsLock} is used to protect read/write access to {@link #isActive} flag and the corresponding code section * based on {@link #isActive} flag. @@ -55,9 +68,8 @@ public class OwnedBundle { * @param suName */ public OwnedBundle(NamespaceBundle suName) { - this.bundle = suName; - IS_ACTIVE_UPDATER.set(this, TRUE); - }; + this(suName, true); + } /** * Constructor to allow set initial active flag. @@ -67,9 +79,20 @@ public OwnedBundle(NamespaceBundle suName) { */ public OwnedBundle(NamespaceBundle suName, boolean active) { this.bundle = suName; + this.resourceLock = null; IS_ACTIVE_UPDATER.set(this, active ? TRUE : FALSE); } + OwnedBundle(NamespaceBundle suName, ResourceLock resourceLock) { + this.bundle = suName; + this.resourceLock = resourceLock; + IS_ACTIVE_UPDATER.set(this, TRUE); + } + + ResourceLock getResourceLock() { + return resourceLock; + } + /** * Access to the namespace name. * @@ -130,11 +153,27 @@ public CompletableFuture handleUnloadRequest(PulsarService pulsar, long ti AtomicInteger unloadedTopics = new AtomicInteger(); log.info().attr("ownership", this.bundle).log("Disabling ownership"); + // Capture the topic futures for this bundle once, before unloading starts, and reuse the same snapshot + // both to close the topics below and to clean up afterward: if this generation's lock has already expired + // and the bundle is re-acquired while the close futures are still running, the cleanup step must only + // touch what this snapshot observed, never a newer generation's topic. + // + // A straggler topic loaded into BrokerService's topic cache for this bundle *after* the snapshot is + // taken is deliberately left alone: neither closed below nor evicted by the cleanup. The alternative — + // force-evicting whatever is in the cache at cleanup time regardless of identity — could yank a topic + // that was never closed, letting a second, independent instance for the same name be created on the + // next load. A straggler surviving the unload does not need this unload to fix it: new connections + // converge through the normal lookup path once ownership has actually moved, and BookKeeper's ledger + // fencing (see ManagedLedgerImpl#addEntryFailedDueToConcurrentlyModified) is what prevents it from + // silently double-writing if a new owner's ManagedLedger instance does start writing the same topic. + Map>> topicFutures = + pulsar.getBrokerService().getTopicFuturesInBundle(bundle); + // close topics forcefully - return pulsar.getNamespaceService().getOwnershipCache() - .updateBundleState(this.bundle, false) - .thenCompose(v -> pulsar.getBrokerService().unloadServiceUnit( - bundle, true, closeWithoutWaitingClientDisconnect, timeout, timeoutUnit)) + // isActive was already flipped to false above; looking the bundle up in the ownership cache here could + // deactivate a newer OwnedBundle that re-acquired the bundle after this instance's lock expired. + return pulsar.getBrokerService().unloadServiceUnit( + bundle, true, closeWithoutWaitingClientDisconnect, timeout, timeoutUnit, topicFutures) .handle((numUnloadedTopics, ex) -> { if (ex != null) { // ignore topic-close failure to unload bundle @@ -146,12 +185,12 @@ public CompletableFuture handleUnloadRequest(PulsarService pulsar, long ti unloadedTopics.set(numUnloadedTopics); } // clean up topics that failed to unload from the broker ownership cache - pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle); + pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle, topicFutures); return null; }) .thenCompose(v -> { - // delete ownership node on zk - return pulsar.getNamespaceService().getOwnershipCache().removeOwnership(bundle); + // delete ownership node on zk, but only for this instance's ownership generation + return pulsar.getNamespaceService().getOwnershipCache().removeOwnership(this); }).whenComplete((ignored, ex) -> { double unloadBundleTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - unloadBundleStartTime)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index bc37d1f990620..5bfe975d1a887 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -33,6 +33,9 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import lombok.CustomLog; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -73,6 +76,22 @@ public class OwnershipCache { private final Map> locallyAcquiredLocks; + /** + * Serializes, per bundle, this cache's own acquire ({@link #tryAcquiringOwnership(NamespaceBundle)}) against + * both of its release paths: the generation-agnostic {@link #removeOwnership(NamespaceBundle)} and the + * generation-aware {@link #removeOwnership(OwnedBundle)} — the one every normal + * {@link OwnedBundle#handleUnloadRequest} unload actually goes through. All three touch + * {@link #locallyAcquiredLocks} without otherwise coordinating with each other: an acquire installs its lock + * asynchronously, well after the cache entry that triggered it becomes visible, while a release removes + * whatever lock is currently registered for the bundle (blindly for the {@code NamespaceBundle}-keyed + * overload, or only if it still matches a specific generation for the {@code OwnedBundle}-keyed one). + * Without this barrier, a release can run in the gap between those two points and either release a lock a + * concurrent acquire just installed (reporting success to that acquire's caller while the underlying ZK + * lock is gone) or run before the install and be silently bypassed by it (reporting ownership fully + * released while a fresh acquisition lands moments later). + */ + private final Map> bundleOperationBarriers = new ConcurrentHashMap<>(); + /** * The loading cache of locally owned NamespaceBundle objects. */ @@ -92,14 +111,53 @@ public CompletableFuture asyncLoad(NamespaceBundle namespaceBundle, return lockManager.acquireLock(ServiceUnitUtils.path(namespaceBundle), selfOwnerInfo) .thenApply(rl -> { locallyAcquiredLocks.put(namespaceBundle, rl); + OwnedBundle ownedBundle = new OwnedBundle(namespaceBundle, rl); + // Set by the expiry listener below once it runs, whether that happens synchronously as + // part of registering it on the next line (the lock was already expired at that point) + // or later, asynchronously on whatever thread completes the future. thenRun() only + // guarantees inline, synchronous execution for the former case: per the CompletableFuture + // class javadoc, a non-async dependent action "may be performed by the thread that + // completes the current CompletableFuture, or by any other caller of a completion + // method" — there is no guarantee that a *concurrent* completion is reflected here by + // the time thenRun() returns. So this flag alone cannot be trusted to catch every case + // where the lock died right around registration; the isDone() recheck below closes that + // gap by reading the future's own state directly instead of relying on the listener + // having finished running. + AtomicBoolean expiredBeforePublication = new AtomicBoolean(false); rl.getLockExpiredFuture() .thenRun(() -> { log.info().attr("path", rl.getPath()).log("Resource lock has expired"); - namespaceService.unloadNamespaceBundle(namespaceBundle); - invalidateLocalOwnerCache(namespaceBundle); + expiredBeforePublication.set(true); + locallyAcquiredLocks.remove(namespaceBundle, rl); + namespaceService.unloadNamespaceBundle(namespaceBundle) + .exceptionally(ex -> { + log.debug() + .attr("bundle", namespaceBundle) + .exception(ex) + .log("Failed to unload namespace bundle after its" + + " resource lock expired"); + return null; + }); + invalidateLocalOwnerCache(namespaceBundle, ownedBundle); namespaceService.onNamespaceBundleUnload(namespaceBundle); }); - return new OwnedBundle(namespaceBundle); + if (expiredBeforePublication.get() || rl.getLockExpiredFuture().isDone()) { + // Expiry won the race: never publish an OwnedBundle whose lock is already gone. Let + // this load fail instead; Caffeine removes a failed load from the cache automatically, + // so no separate cache invalidation is needed here, and the caller of + // tryAcquiringOwnership observes a failure instead of a fleeting, already-invalid + // success. The isDone() check is a defensive addition alongside the flag: it reads + // the future's state directly, so it also catches the case where the lock expired + // concurrently with registration above but the listener callback (and therefore the + // flag) hasn't finished running yet. Even without it, a lock that expires around here + // and is missed by both checks still converges correctly once the listener does run + // (see invalidateLocalOwnerCache(NamespaceBundle, OwnedBundle)) — this just narrows + // that window rather than being the sole safeguard against it. + throw new IllegalStateException( + "Lock for bundle " + namespaceBundle + + " expired before ownership could be published"); + } + return ownedBundle; }); } } @@ -209,13 +267,14 @@ public CompletableFuture tryAcquiringOwnership(Namespace log.info().attr("bundle", bundle).log("Trying to acquire ownership"); // Doing a get() on the ownedBundlesCache will trigger an async metadata write to acquire the lock over the - // service unit - return ownedBundlesCache.get(bundle) + // service unit. Serialized against both removeOwnership(NamespaceBundle) and removeOwnership(OwnedBundle) + // for the same bundle: see bundleOperationBarriers. + return serialize(bundle, () -> ownedBundlesCache.get(bundle) .thenApply(namespaceBundle -> { - log.info().attr("bundle", namespaceBundle).log("Successfully acquired ownership"); - namespaceService.onNamespaceBundleOwned(bundle); - return selfOwnerInfo; - }); + log.info().attr("bundle", namespaceBundle).log("Successfully acquired ownership"); + namespaceService.onNamespaceBundleOwned(bundle); + return selfOwnerInfo; + })); } /** @@ -223,13 +282,86 @@ public CompletableFuture tryAcquiringOwnership(Namespace * */ public CompletableFuture removeOwnership(NamespaceBundle bundle) { - ResourceLock lock = locallyAcquiredLocks.remove(bundle); + // Serialized against tryAcquiringOwnership(NamespaceBundle) for the same bundle: see + // bundleOperationBarriers. This also subsumes waiting for a concurrently in-flight acquire to settle + // before this blind, generation-agnostic release runs. + return serialize(bundle, () -> { + ResourceLock lock = locallyAcquiredLocks.remove(bundle); + if (lock == null) { + // We don't own the specified bundle anymore + return CompletableFuture.completedFuture(null); + } + + return lock.release(); + }); + } + + /** + * Runs {@code operation} only after any previously queued {@link #tryAcquiringOwnership(NamespaceBundle)}, + * {@link #removeOwnership(NamespaceBundle)}, or {@link #removeOwnership(OwnedBundle)} call for the same + * bundle has settled, and queues subsequent calls for that bundle behind this one in turn. The barrier entry + * for a bundle is removed once no further operation is queued behind it, so {@link #bundleOperationBarriers} + * does not grow unboundedly. + * + *

The {@code compute} call below only captures the preceding barrier and installs the new one; it does not + * chain {@code operation} itself. {@code ConcurrentHashMap.compute} runs its remapping function while holding + * the map's per-bucket lock, and a same-thread reentrant call into {@code compute} for the same key from + * inside that function is a usage the map's contract leaves undefined. If {@code operation} (or the preceding + * barrier) were chained inline here, an already-completed source — a fast-failing metadata call, or the + * completed futures test doubles commonly return — would make {@code thenCompose} run {@code operation} + * synchronously right there, still under that lock; were {@code operation} to then reach {@code serialize()} + * again for the same bundle, that would be exactly such a reentrant call. Building the chain after + * {@code compute} returns means even a fully synchronous, self-reentrant {@code operation} only ever produces + * ordinary, non-nested {@code compute} calls. + */ + private CompletableFuture serialize(NamespaceBundle bundle, Supplier> operation) { + CompletableFuture result = new CompletableFuture<>(); + CompletableFuture opDone = new CompletableFuture<>(); + AtomicReference> precedingOpRef = new AtomicReference<>(); + bundleOperationBarriers.compute(bundle, (k, previous) -> { + precedingOpRef.set(previous != null ? previous : CompletableFuture.completedFuture(null)); + return opDone; + }); + precedingOpRef.get().handle((r, e) -> null) + .thenCompose(ignore -> operation.get()) + .whenComplete((r, e) -> { + if (e != null) { + result.completeExceptionally(e); + } else { + result.complete(r); + } + opDone.complete(null); + }); + opDone.whenComplete((r, e) -> bundleOperationBarriers.remove(bundle, opDone)); + return result; + } + + /** + * Method to remove the ownership that was acquired for the given {@link OwnedBundle} instance only. + * + *

If the bundle has since been re-acquired (the given instance's lock expired and a newer + * {@link OwnedBundle} with a newer lock owns the bundle now), the newer ownership is left untouched. + * + *

Serialized against {@link #tryAcquiringOwnership(NamespaceBundle)} for the same bundle: see + * {@link #bundleOperationBarriers}. This is the release path every normal {@link OwnedBundle#handleUnloadRequest} + * unload actually goes through, so without this barrier a concurrent acquire could still observe and + * report success for the generation this call is in the middle of releasing. + */ + public CompletableFuture removeOwnership(OwnedBundle ownedBundle) { + ResourceLock lock = ownedBundle.getResourceLock(); if (lock == null) { - // We don't own the specified bundle anymore - return CompletableFuture.completedFuture(null); + // The instance is not bound to a lock (not created by this cache): fall back to removing whatever + // ownership currently exists for the bundle, which is itself serialized already. + return removeOwnership(ownedBundle.getNamespaceBundle()); } - - return lock.release(); + return serialize(ownedBundle.getNamespaceBundle(), () -> { + if (!locallyAcquiredLocks.remove(ownedBundle.getNamespaceBundle(), lock)) { + // This ownership generation was already released or has expired; a newer acquisition may own the + // bundle now and must not be disturbed. + return CompletableFuture.completedFuture(null); + } + return lock.release(); + }); } /** @@ -343,6 +475,33 @@ public void invalidateLocalOwnerCache(NamespaceBundle namespaceBundle) { this.ownedBundlesCache.synchronous().invalidate(namespaceBundle); } + /** + * Invalidate the local owner cache entry once it holds the given {@link OwnedBundle} instance, so that a + * stale lock-expiry callback cannot drop an entry installed by a newer acquisition. + * + *

The callback that calls this can run before the cache's own load future for this bundle has completed: + * the lock-expiry listener is registered inside the cache loader's {@code thenApply}, and if the lock was + * already expired at that point the listener fires inline, synchronously, before the loader returns and the + * future is published as done. (If the lock instead expires concurrently with registration, {@code thenRun} + * gives no such synchronous guarantee — the listener may run later, on whichever thread completes the + * future — but that only changes when this method is called relative to publication, not whether it is + * called; the handling below covers both.) Waiting on the future via {@code whenComplete} instead of + * requiring {@code isDone()} up front handles both cases: if the future is already done the callback runs + * immediately, and if not, the removal is deferred until the loader publishes it, so the newly-published + * {@link OwnedBundle} — whose lock has already expired — is not left claiming active ownership forever. + */ + private void invalidateLocalOwnerCache(NamespaceBundle namespaceBundle, OwnedBundle expectedOwnedBundle) { + CompletableFuture future = ownedBundlesCache.getIfPresent(namespaceBundle); + if (future == null) { + return; + } + future.whenComplete((ownedBundle, ex) -> { + if (ex == null && ownedBundle == expectedOwnedBundle) { + ownedBundlesCache.asMap().remove(namespaceBundle, future); + } + }); + } + @VisibleForTesting public Map> getLocallyAcquiredLocks() { return locallyAcquiredLocks; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 1ccd1478daf9e..f3f6e2e3910d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2619,8 +2619,24 @@ public CompletableFuture checkTopicNsOwnership(final String topic) { public CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, boolean disconnectClients, boolean closeWithoutWaitingClientDisconnect, long timeout, TimeUnit unit) { + return unloadServiceUnit(serviceUnit, disconnectClients, closeWithoutWaitingClientDisconnect, timeout, unit, + getTopicFuturesInBundle(serviceUnit)); + } + + /** + * Same as {@link #unloadServiceUnit(NamespaceBundle, boolean, boolean, long, TimeUnit)}, but takes the topic + * futures to unload as given, instead of scanning {@link #topics} again. Callers that also need to run + * {@link #cleanUnloadedTopicFromCache(NamespaceBundle, Map)} afterward should capture the snapshot once via + * {@link #getTopicFuturesInBundle(NamespaceBundle)} and pass the very same map to both calls: reusing one + * snapshot guarantees the cleanup step can neither miss a topic this call targeted, nor evict one it never + * observed (for example one installed by a newer ownership generation while this call was still running). + */ + public CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, + boolean disconnectClients, + boolean closeWithoutWaitingClientDisconnect, long timeout, TimeUnit unit, + Map>> topicFutures) { CompletableFuture future = unloadServiceUnit( - serviceUnit, disconnectClients, closeWithoutWaitingClientDisconnect); + serviceUnit, disconnectClients, closeWithoutWaitingClientDisconnect, topicFutures); ScheduledFuture taskTimeout = executor().schedule(() -> { if (!future.isDone()) { log.warn().attr("serviceUnit", serviceUnit).log("Unloading of has timed out"); @@ -2644,50 +2660,50 @@ public CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, */ private CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, boolean disconnectClients, - boolean closeWithoutWaitingClientDisconnect) { + boolean closeWithoutWaitingClientDisconnect, + Map>> + topicFutures) { List> closeFutures = new ArrayList<>(); - topics.forEach((name, topicFuture) -> { + topicFutures.forEach((name, topicFuture) -> { TopicName topicName = TopicName.get(name); - if (serviceUnit.includes(topicName)) { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) - && ExtensibleLoadManagerImpl.isInternalTopic(topicName.toString())) { - if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) { - log.info() - .attr("topic", topicName) - .log("Skip unloading ExtensibleLoadManager internal topics. Such internal topic " - + "should be closed when shutting down the broker."); - } - return; + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) + && ExtensibleLoadManagerImpl.isInternalTopic(topicName.toString())) { + if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) { + log.info() + .attr("topic", topicName) + .log("Skip unloading ExtensibleLoadManager internal topics. Such internal topic " + + "should be closed when shutting down the broker."); } + return; + } - // Topic needs to be unloaded - log.info().attr("topic", topicName).log("Unloading topic"); - if (topicFuture.isCompletedExceptionally()) { - try { - topicFuture.get(); - } catch (InterruptedException | ExecutionException ex) { - if (ex.getCause() instanceof ServiceUnitNotReadyException) { - // Topic was already unloaded - log.debug().attr("topic", topicName).log("Topic was already unloaded"); - return; - } else { - log.warn().attr("topic", topicName).exception(ex).log("Got exception when closing topic"); - } + // Topic needs to be unloaded + log.info().attr("topic", topicName).log("Unloading topic"); + if (topicFuture.isCompletedExceptionally()) { + try { + topicFuture.get(); + } catch (InterruptedException | ExecutionException ex) { + if (ex.getCause() instanceof ServiceUnitNotReadyException) { + // Topic was already unloaded + log.debug().attr("topic", topicName).log("Topic was already unloaded"); + return; + } else { + log.warn().attr("topic", topicName).exception(ex).log("Got exception when closing topic"); } } - closeFutures.add(topicFuture - .thenCompose(t -> t.isPresent() ? t.get().close( - disconnectClients, closeWithoutWaitingClientDisconnect) - : CompletableFuture.completedFuture(null)) - .exceptionally(e -> { - if (e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException - && e.getMessage().contains("Please redo the lookup")) { - log.warn().attr("topic", topicName).log("Topic ownership check failed. Skipping it"); - return null; - } - throw FutureUtil.wrapToCompletionException(e); - })); } + closeFutures.add(topicFuture + .thenCompose(t -> t.isPresent() ? t.get().close( + disconnectClients, closeWithoutWaitingClientDisconnect) + : CompletableFuture.completedFuture(null)) + .exceptionally(e -> { + if (e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException + && e.getMessage().contains("Please redo the lookup")) { + log.warn().attr("topic", topicName).log("Topic ownership check failed. Skipping it"); + return null; + } + throw FutureUtil.wrapToCompletionException(e); + })); }); if (getPulsar().getConfig().isTransactionCoordinatorEnabled() @@ -2705,17 +2721,39 @@ private CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit return FutureUtil.waitForAll(closeFutures).thenApply(v -> closeFutures.size()); } - public void cleanUnloadedTopicFromCache(NamespaceBundle serviceUnit) { - for (String topic : topics.keySet()) { - TopicName topicName = TopicName.get(topic); - if (serviceUnit.includes(topicName) && getTopicReference(topic).isPresent()) { + /** + * Captures the topic futures currently cached for the given bundle. Call this before starting an unload so + * that a later {@link #cleanUnloadedTopicFromCache(NamespaceBundle, Map)} call can remove only the exact + * futures this unload observed, and never a newer ownership generation's entry installed afterward for the + * same bundle. + */ + public Map>> getTopicFuturesInBundle(NamespaceBundle serviceUnit) { + Map>> topicFutures = new HashMap<>(); + topics.forEach((name, topicFuture) -> { + if (serviceUnit.includes(TopicName.get(name))) { + topicFutures.put(name, topicFuture); + } + }); + return topicFutures; + } + + /** + * Cleans up topics that failed to unload from the broker's topic cache. Only removes a topic if its + * currently-cached future is exactly the one captured in {@code topicFutures} (see + * {@link #getTopicFuturesInBundle(NamespaceBundle)}), so a stale call for an old ownership generation can + * never evict a newer generation's topic. + */ + public void cleanUnloadedTopicFromCache(NamespaceBundle serviceUnit, + Map>> topicFutures) { + topicFutures.forEach((topic, topicFuture) -> { + if (getTopicReference(topic).isPresent()) { log.info() .attr("value", serviceUnit.toString()) .attr("topic", topic) .log("Clean unloaded topic from cache."); - pulsar.getBrokerService().removeTopicFromCache(topicName.toString(), serviceUnit, null); + removeTopicFromCache(topic, serviceUnit, topicFuture); } - } + }); } public AuthorizationService getAuthorizationService() { @@ -2739,6 +2777,18 @@ public CompletableFuture removeTopicFromCache(AbstractTopic topic) { private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, CompletableFuture> createTopicFuture) { + // Gate everything below on the identity-guarded removal itself: if the currently-cached future no + // longer matches createTopicFuture (a newer ownership generation already replaced it), none of the + // bookkeeping or events below may run either, or a stale cleanup call would still strip a still-live + // newer generation's multiLayerTopicsMap/replication-metrics/compactor-stats/segment-load bookkeeping + // and fire spurious UNLOAD events for a topic that was never actually removed. + boolean removed = createTopicFuture == null + ? topics.remove(topic) != null + : topics.remove(topic, createTopicFuture); + if (!removed) { + return; + } + String bundleName = namespaceBundle.toString(); String namespaceName = TopicName.get(topic).getNamespaceObject().toString(); @@ -2767,12 +2817,6 @@ private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, } } - if (createTopicFuture == null) { - topics.remove(topic); - } else { - topics.remove(topic, createTopicFuture); - } - Compactor compactor = pulsar.getNullableCompactor(); if (compactor != null) { compactor.getStats().removeTopic(topic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java index 43e816aaa3a4b..5c6e98b071f01 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java @@ -27,12 +27,16 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNotSame; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Range; import com.google.common.hash.Hashing; import java.util.EnumSet; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -50,6 +54,8 @@ import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.coordination.CoordinationService; +import org.apache.pulsar.metadata.api.coordination.LockManager; +import org.apache.pulsar.metadata.api.coordination.ResourceLock; import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl; @@ -97,9 +103,14 @@ public void setup() throws Exception { bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32()); nsService = mock(NamespaceService.class); + doReturn(CompletableFuture.completedFuture(null)).when(nsService) + .unloadNamespaceBundle(any(NamespaceBundle.class)); brokerService = mock(BrokerService.class); doReturn(CompletableFuture.completedFuture(1)).when(brokerService) .unloadServiceUnit(any(), anyBoolean(), anyBoolean(), anyLong(), any()); + doReturn(CompletableFuture.completedFuture(1)).when(brokerService) + .unloadServiceUnit(any(), anyBoolean(), anyBoolean(), anyLong(), any(), any()); + doReturn(Map.of()).when(brokerService).getTopicFuturesInBundle(any()); doReturn(config).when(pulsar).getConfiguration(); doReturn(nsService).when(pulsar).getNamespaceService(); @@ -407,4 +418,240 @@ public void testReestablishOwnership() throws Exception { assertNotNull(cache.getOwnedBundle(testFullBundle)); } + @Test + public void testStaleUnloadDoesNotReleaseReacquiredOwnership() throws Exception { + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); + doReturn(cache).when(nsService).getOwnershipCache(); + NamespaceBundle bundle = new NamespaceBundle(NamespaceName.get("pulsar/ns-stale-unload"), + Range.closedOpen(0L, (long) Integer.MAX_VALUE), + bundleFactory); + + cache.tryAcquiringOwnership(bundle).get(); + OwnedBundle staleOwnedBundle = cache.getOwnedBundle(bundle); + assertNotNull(staleOwnedBundle); + + // Simulate the lock-expiry self-heal path: the expiry callback invalidated the local cache while the + // unload it triggered is still in flight, and a concurrent lookup re-acquires the bundle in between. + cache.invalidateLocalOwnerCache(bundle); + cache.tryAcquiringOwnership(bundle).get(); + OwnedBundle reacquiredOwnedBundle = cache.getOwnedBundle(bundle); + assertNotNull(reacquiredOwnedBundle); + assertNotSame(reacquiredOwnedBundle, staleOwnedBundle); + ResourceLock reacquiredLock = cache.getLocallyAcquiredLocks().get(bundle); + assertNotNull(reacquiredLock); + + // The stale unload chain now runs its remaining steps against the old OwnedBundle instance. + staleOwnedBundle.handleUnloadRequest(pulsar, 5, TimeUnit.SECONDS).join(); + + // The re-acquired ownership must survive the stale unload untouched. + assertSame(cache.getLocallyAcquiredLocks().get(bundle), reacquiredLock); + assertTrue(store.exists(ServiceUnitUtils.path(bundle)).join()); + assertTrue(reacquiredOwnedBundle.isActive()); + assertTrue(cache.checkOwnershipAsync(bundle).get()); + } + + @Test + public void testTryAcquiringOwnershipWaitsForInFlightOwnedBundleRelease() throws Exception { + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); + NamespaceBundle bundle = new NamespaceBundle(NamespaceName.get("pulsar/ns-ownedbundle-release-inflight"), + Range.closedOpen(0L, (long) Integer.MAX_VALUE), + bundleFactory); + + cache.tryAcquiringOwnership(bundle).get(); + OwnedBundle gen1 = cache.getOwnedBundle(bundle); + assertNotNull(gen1); + + // Mirror what OwnedBundle.handleUnloadRequest does at the very end of a normal unload: release via the + // generation-aware, OwnedBundle-keyed overload. Do NOT wait for it to finish before racing an acquire + // against it below. + CompletableFuture removeFuture = cache.removeOwnership(gen1); + + // A concurrent lookup racing the in-flight release must be queued behind it by the barrier, not see the + // stale, concurrently-releasing generation. + NamespaceEphemeralData reacquired = cache.tryAcquiringOwnership(bundle).get(); + OwnedBundle afterReacquire = cache.getOwnedBundle(bundle); + + removeFuture.get(10, TimeUnit.SECONDS); + + assertNotSame(afterReacquire, gen1, + "tryAcquiringOwnership returned the stale, concurrently-releasing generation instead of a fresh one"); + assertTrue(afterReacquire.isActive(), "reacquired OwnedBundle should be active"); + assertSame(cache.getLocallyAcquiredLocks().get(bundle), afterReacquire.getResourceLock()); + } + + @Test + public void testRemoveOwnershipWithAcquisitionInFlight() throws Exception { + // Gate the lock acquisition so the test can invoke removeOwnership while the acquisition is in flight + LockManager realLockManager = + coordinationService.getLockManager(NamespaceEphemeralData.class); + CompletableFuture gate = new CompletableFuture<>(); + LockManager gatedLockManager = new LockManager() { + @Override + public CompletableFuture> readLock(String path) { + return realLockManager.readLock(path); + } + + @Override + public CompletableFuture> acquireLock(String path, + NamespaceEphemeralData value) { + return gate.thenCompose(__ -> realLockManager.acquireLock(path, value)); + } + + @Override + public CompletableFuture> listLocks(String path) { + return realLockManager.listLocks(path); + } + + @Override + public CompletableFuture asyncClose() { + return realLockManager.asyncClose(); + } + + @Override + public void close() throws Exception { + realLockManager.close(); + } + }; + CoordinationService gatedCoordinationService = mock(CoordinationService.class); + doReturn(gatedLockManager).when(gatedCoordinationService).getLockManager(NamespaceEphemeralData.class); + doReturn(gatedCoordinationService).when(pulsar).getCoordinationService(); + + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); + NamespaceBundle bundle = new NamespaceBundle(NamespaceName.get("pulsar/ns-inflight-remove"), + Range.closedOpen(0L, (long) Integer.MAX_VALUE), + bundleFactory); + + CompletableFuture acquireFuture = cache.tryAcquiringOwnership(bundle); + assertFalse(acquireFuture.isDone()); + + CompletableFuture removeFuture = cache.removeOwnership(bundle); + gate.complete(null); + acquireFuture.join(); + removeFuture.get(10, TimeUnit.SECONDS); + + // After removeOwnership reported success and the in-flight acquisition settled, the broker must not + // silently retain (zombie) ownership. + Awaitility.await().untilAsserted(() -> { + assertTrue(cache.getLocallyAcquiredLocks().isEmpty()); + assertTrue(cache.getOwnedBundles().isEmpty()); + assertFalse(store.exists(ServiceUnitUtils.path(bundle)).join()); + }); + } + + @Test + public void testExpiredLockIsRemovedFromLocallyAcquiredLocks() throws Exception { + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); + NamespaceBundle bundle = new NamespaceBundle(NamespaceName.get("pulsar/ns-expired-lock"), + Range.closedOpen(0L, (long) Integer.MAX_VALUE), + bundleFactory); + + cache.tryAcquiringOwnership(bundle).get(); + ResourceLock lock = cache.getLocallyAcquiredLocks().get(bundle); + assertNotNull(lock); + + // The lock dies without going through removeOwnership, like on a metadata session expiry + lock.release().join(); + + Awaitility.await().untilAsserted(() -> { + assertTrue(cache.getOwnedBundles().isEmpty()); + assertTrue(cache.getLocallyAcquiredLocks().isEmpty()); + }); + } + + @Test + public void testExpiryBeforePublicationDoesNotLeaveActiveZombieOwnership() throws Exception { + // A ResourceLock whose expiry future is *already* completed by the time the loader attaches its + // lock-expiry listener: this deterministically forces the listener to run synchronously, inside the + // cache loader's thenApply, before the loader returns and the cache's own future for this bundle is + // published as done. + ResourceLock alreadyExpiredLock = new ResourceLock<>() { + @Override + public String getPath() { + return "/dummy"; + } + + @Override + public NamespaceEphemeralData getValue() { + return null; + } + + @Override + public CompletableFuture updateValue(NamespaceEphemeralData newValue) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture release() { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture getLockExpiredFuture() { + return CompletableFuture.completedFuture(null); + } + }; + // Gate the lock acquisition itself so it settles only after tryAcquiringOwnership(bundle) has already + // returned to the caller: this ensures the cache has genuinely registered its (not yet done) future for + // the bundle before the loader's thenApply — and the already-expired lock's listener inside it — runs, + // matching how a real ZK acquisition callback fires on a different thread than the initial get() call. + CompletableFuture gate = new CompletableFuture<>(); + LockManager raceyLockManager = new LockManager<>() { + @Override + public CompletableFuture> readLock(String path) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> acquireLock(String path, + NamespaceEphemeralData value) { + return gate.thenApply(ignore -> alreadyExpiredLock); + } + + @Override + public CompletableFuture> listLocks(String path) { + return CompletableFuture.completedFuture(List.of()); + } + + @Override + public CompletableFuture asyncClose() { + return CompletableFuture.completedFuture(null); + } + + @Override + public void close() { + } + }; + CoordinationService raceyCoordinationService = mock(CoordinationService.class); + doReturn(raceyLockManager).when(raceyCoordinationService).getLockManager(NamespaceEphemeralData.class); + doReturn(raceyCoordinationService).when(pulsar).getCoordinationService(); + + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); + NamespaceBundle bundle = new NamespaceBundle(NamespaceName.get("pulsar/ns-expiry-before-publication"), + Range.closedOpen(0L, (long) Integer.MAX_VALUE), + bundleFactory); + + CompletableFuture acquireFuture = cache.tryAcquiringOwnership(bundle); + assertFalse(acquireFuture.isDone()); + + gate.complete(null); + + // Expiry won the race: the acquisition itself must fail instead of publishing an OwnedBundle whose lock + // is already gone. + try { + acquireFuture.get(10, TimeUnit.SECONDS); + fail("acquisition should fail when the lock expired before ownership could be published"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof IllegalStateException); + } + + // A failed load must not leave any trace behind: no cache entry claiming ownership, and no lock + // bookkeeping. + Awaitility.await().untilAsserted(() -> { + assertNull(cache.getOwnedBundle(bundle), + "cache still claims active ownership for a bundle whose lock expired before publication"); + assertTrue(cache.getLocallyAcquiredLocks().isEmpty()); + assertFalse(cache.checkOwnershipAsync(bundle).get()); + }); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 4a6e329c72f81..65797b5d800a3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -31,6 +31,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNotSame; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -2051,6 +2052,84 @@ public void testGetTopicWhenTopicPoliciesFail() throws Exception { assertFalse(MockTopicPoliciesService.FAILED_TOPICS.contains(topicName)); } + @Test + public void testCleanUnloadedTopicFromCacheIsGenerationSafe() throws Exception { + final String topicName = "persistent://prop/ns-abc/staleCleanupTest-" + UUID.randomUUID(); + admin.topics().createNonPartitionedTopic(topicName); + + BrokerService brokerService = pulsar.getBrokerService(); + NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(TopicName.get(topicName)); + + // Generation 1: load the topic and capture its future the way a real unload snapshot would. + Topic staleGenerationTopic = brokerService.getTopic(topicName, true).get().orElseThrow(); + CompletableFuture> staleGenerationFuture = + CompletableFuture.completedFuture(Optional.of(staleGenerationTopic)); + + // Simulate re-acquisition: the topic is closed and reloaded, installing a *new* generation's future/topic + // under the same name, while this broker still owns and serves the bundle. + admin.topics().unload(topicName); + Topic newGenerationTopic = brokerService.getTopic(topicName, true).get().orElseThrow(); + assertNotSame(newGenerationTopic, staleGenerationTopic); + + // A stale cleanup call for the old generation's unload arrives late. It must only ever act on the exact + // future it captured at unload start, never on a newer generation's entry for the same topic name. + brokerService.cleanUnloadedTopicFromCache(bundle, Map.of(topicName, staleGenerationFuture)); + + assertTrue(brokerService.getTopicReference(topicName).isPresent(), + "stale cleanup wrongly evicted the newer generation's topic from the cache"); + } + + @Test + public void testCleanUnloadedTopicFromCacheIsGenerationSafeForBookkeeping() throws Exception { + final String topicName = "persistent://prop/ns-abc/staleCleanupBookkeepingTest-" + UUID.randomUUID(); + admin.topics().createNonPartitionedTopic(topicName); + + BrokerService brokerService = pulsar.getBrokerService(); + NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(TopicName.get(topicName)); + + // Generation 1: load the topic and capture its future the way a real unload snapshot would. + Topic staleGenerationTopic = brokerService.getTopic(topicName, true).get().orElseThrow(); + CompletableFuture> staleGenerationFuture = + CompletableFuture.completedFuture(Optional.of(staleGenerationTopic)); + + // Simulate re-acquisition: the topic is closed and reloaded, installing a *new* generation's future/topic + // under the same name, while this broker still owns and serves the bundle. + admin.topics().unload(topicName); + brokerService.getTopic(topicName, true).get().orElseThrow(); + // addTopicToStatsMaps() runs asynchronously off the topic-load future, so wait for it to land. + Awaitility.await().untilAsserted(() -> + assertTrue(brokerService.getTopicStats(bundle).containsKey(topicName), + "the newer generation should be tracked in the per-bundle stats index after reload")); + + // A stale cleanup call for the old generation's unload arrives late. The topics-map removal is a + // guarded no-op (proven by testCleanUnloadedTopicFromCacheIsGenerationSafe above), but the surrounding + // bookkeeping around it must be gated on that same guard too, not run unconditionally. + brokerService.cleanUnloadedTopicFromCache(bundle, Map.of(topicName, staleGenerationFuture)); + + assertTrue(brokerService.getTopicStats(bundle).containsKey(topicName), + "stale cleanup wrongly stripped the still-live newer generation's topic from the per-bundle " + + "stats index (multiLayerTopicsMap), even though the topics-map removal itself was " + + "correctly skipped"); + } + + @Test + public void testCleanUnloadedTopicFromCacheRemovesMatchingSnapshot() throws Exception { + final String topicName = "persistent://prop/ns-abc/staleCleanupMatchTest-" + UUID.randomUUID(); + admin.topics().createNonPartitionedTopic(topicName); + + BrokerService brokerService = pulsar.getBrokerService(); + NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(TopicName.get(topicName)); + brokerService.getTopic(topicName, true).get(); + + // A snapshot that matches exactly what is currently cached must still be cleaned up: this is the + // legitimate backstop case (a topic whose close() failed to remove itself from the cache). + Map>> currentSnapshot = brokerService.getTopicFuturesInBundle(bundle); + brokerService.cleanUnloadedTopicFromCache(bundle, currentSnapshot); + + assertFalse(brokerService.getTopicReference(topicName).isPresent(), + "cleanup should still remove a topic future that matches what was captured"); + } + static class MockTopicPoliciesService extends TopicPoliciesService.TopicPoliciesServiceDisabled { static final Set FAILED_TOPICS = ConcurrentHashMap.newKeySet(); From bb4768d4902693d78c22d91e2621ecfb926e5e18 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 3 Aug 2026 18:40:54 +0300 Subject: [PATCH 002/107] [feat][pip] PIP-478: Asynchronous v5 client auth plugin interfaces and TLS material provider plugin interface (#25890) --- pip/pip-478.md | 1353 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1353 insertions(+) create mode 100644 pip/pip-478.md diff --git a/pip/pip-478.md b/pip/pip-478.md new file mode 100644 index 0000000000000..4032bdc1c8afc --- /dev/null +++ b/pip/pip-478.md @@ -0,0 +1,1353 @@ +# PIP-478: Asynchronous v5 client authentication SPI, Pulsar TLS factory SPI, and Pulsar HTTP client SPI + +> **This PIP at a glance.** It introduces three SPIs — an async, capability-segregated **v5 authentication SPI** (replacing PIP-466's synchronous `Authentication` stub), a purpose-driven **`PulsarTlsFactory` SPI**, and a framework-managed **`PulsarHttpClient` SPI** — and, as a deliberate **breaking change**, **removes PIP-337's `PulsarSslFactory` entirely**. It also makes Pulsar 5.0 TLS **secure by default** (hostname verification on, SAN-only). Reviewer's map: [Motivation](#motivation) (five concrete v4 problems) → [High Level Design](#high-level-design) (the shapes and the reasoning) → [Detailed Design](#detailed-design) (full type listings) → [PIP-337 removal impact](#pip-337-removal-impact) (disposition of every removed config key / method) → [Resolved design decisions](#resolved-design-decisions) (options weighed). Security- and compatibility-focused reviewers should also read [Security Considerations](#security-considerations) and [Backward & Forward Compatibility](#backward--forward-compatibility). + +# Background knowledge + +Apache Pulsar's Java client lets applications plug in custom authentication through `org.apache.pulsar.client.api.Authentication` and `org.apache.pulsar.client.api.AuthenticationDataProvider`. The same pair has been in place since the first 2.x releases and is the basis of every client-side auth mechanism Pulsar ships (token, mTLS, KeyStore TLS, Basic, OAuth2, Athenz, SASL). + +**PIP-97** (*Asynchronous Authentication Provider*, accepted for 2.10/2.12) made the **broker-side** authentication framework asynchronous so that providers can do non-blocking I/O during authentication — for example, the OAuth2 / OIDC verifier needs to call out to an identity provider. PIP-97 deliberately scoped itself to the broker side and explicitly deferred the client side: deprecated methods on `Authentication` were left behind for later cleanup. Three years on, that cleanup never happened, and the deprecated methods (`getAuthData()` no-arg, `configure(Map)`) are still part of the v4 surface. + +**PIP-337** (*SSL Factory Plugin*, Pulsar 3.x) introduced a pluggable `PulsarSslFactory` so operators can replace Pulsar's default file-based TLS material loading with custom logic (e.g., using a Key Management System (KMS) API). The factory takes a `PulsarSslConfiguration` value object that includes — among the TLS file paths — a direct field `AuthenticationDataProvider authData`. This forced the SSL layer in `pulsar-common` to depend on the v4 `AuthenticationDataProvider` interface, even though the SSL factory ought to be a transport concern independent of who supplies the credentials. The PIP-337 classes live in `org.apache.pulsar.common.util`, alongside dozens of unrelated utility helpers, which is the wrong package for what is now a public SPI. + +**PIP-466** (*New Java Client API V5*, accepted for Pulsar 5.0) created two new modules — `pulsar-client-api-v5` and `pulsar-client-v5` — to host a modernised Java client API. PIP-466 is explicitly *additive*: the existing v4 modules (`pulsar-client-api`, `pulsar-client`) remain unchanged. PIP-466 sketched a sync `Authentication` stub in `pulsar-client-api-v5` and a basic adapter in `pulsar-client-v5`; this PIP replaces both with the async, capability-segregated SPI and the bridges described below. PIP-466 punted on auth design and explicitly invited a follow-up proposal to fill it in. This PIP is that follow-up. + +## How Pulsar client authentication works + +Pulsar clients authenticate to the broker over **two transports**, each with their own protocol semantics. A plugin typically supports both, because applications commonly produce / consume on the binary protocol AND administer the cluster through the HTTP REST API with the same credentials. + +### Transport A — Pulsar binary protocol + +The Pulsar binary protocol is a length-prefixed Protocol Buffers framing used for produce, consume, lookup, and admin RPCs over TCP (typically port 6650; 6651 for TLS). Authentication is conveyed in dedicated `Command*` messages: + +1. Client opens a TCP (or TLS) connection. +2. Client sends `org.apache.pulsar.common.api.proto.CommandConnect` carrying: + - `auth_method_name` — stable string identifying the plugin (e.g., `"token"`, `"basic"`, `"sasl"`). + - `auth_data` — opaque byte payload (the credential bytes — a JWT token, a SASL initial frame, etc.). +3. Broker either: + - **Succeeds immediately** (single-pass authentication): broker sends `CommandConnected`. The connection is ready. + - **Sends a challenge** (multi-round authentication): broker sends `CommandAuthChallenge` carrying a `challenge` payload. Client replies with `CommandAuthResponse` carrying response bytes. This repeats until the broker accepts (then sends `CommandConnected`) or rejects. + - **Fails**: broker sends `ServerError` with `AuthenticationError`. The connection is closed. + +The binary protocol also reuses `CommandAuthChallenge` for **broker-pushed credential refresh**: a magic payload `AuthData.REFRESH_AUTH_DATA_BYTES = "PulsarAuthRefresh"` signals "your short-lived credential is about to expire (or has already expired) — produce a fresh one and respond with `CommandAuthResponse`." This is how OAuth2 / Athenz / other short-lived-credential schemes stay valid across long-lived connections. + +### Transport B — HTTP/HTTPS REST API (admin client, HTTP topic lookup) + +The Pulsar admin client and HTTP-mode topic lookup speak HTTP/1.1 (typically port 8080; 8443 for TLS). Authentication rides in standard HTTP headers: + +1. Client constructs an HTTP request. +2. Client attaches authentication headers. For single-pass credentials, typically `Authorization: ` — e.g., JWT auth sends `Authorization: Bearer `; Basic auth sends `Authorization: Basic `; Athenz sends a custom `Athenz-Role-Auth: ` header. +3. Server either: + - **Succeeds**: handles the request normally (2xx response). + - **Sends a challenge** (multi-round): replies with `401 Unauthorized` carrying the challenge in Pulsar-specific SASL headers — `SASL-Token` (the base64 token bytes), `State`, and `SASL-Server-ID` (to correlate the multi-round session) — **not** the standard `WWW-Authenticate` header. The client resubmits the request with updated `SASL-Token` / `State` / `SASL-Server-ID` headers. This repeats until the server accepts, at which point it responds `200 OK` carrying the final `SaslAuthRoleToken`. SASL (Kerberos) over HTTP follows this pattern, driven server-side by `AuthenticationProviderSasl` and client-side by `AuthenticationSasl`. + - **Fails**: the request is rejected (`401`/`403`) and not retried. + +Today this multi-round HTTP loop is **not** driven by a generic framework component — the SASL plugin implements it itself. `AuthenticationSasl.authenticationStage(...)` recursively resubmits the request on each `401` until the server returns `200`, and both the admin client and the HTTP-lookup client simply call the plugin's `authenticationStage(...)` / `newRequestHeader(...)` hooks. Two quirks follow from this: the SASL stage always re-issues the exchange as a `GET` to the original URI (even when the original request was a `POST`), and the admin client and HTTP-lookup client reach the same plugin through two different HTTP client APIs — JAX-RS (Jersey) for the admin client, raw AsyncHttpClient for HTTP lookup — even though both run over the same AsyncHttpClient transport (Jersey is wired through Pulsar's own `AsyncHttpConnector`). The v5 design relocates this loop into a framework-side **driver** — one shared loop implementation behind a thin adapter per client API — leaving the plugin to compute only each round's response; see [the capability-segregated SPI](#capability-segregated-authentication-spi). + +### Authentication styles + +Independent of transport, every Pulsar authentication method follows one of two styles: + +- **Single-pass credential exchange.** The client presents a credential (a token, a username/password pair, a signed assertion) and the broker accepts or rejects in a single round trip. Long-lived credentials need no refresh at all — a static JWT (`Token`) or a `Basic` username/password is presented as-is on every connection. Short-lived credentials *do* expire, and refreshing them is an **internal concern of the authentication plugin**: the plugin produces a fresh credential when next asked (and may proactively renew in the background, or react to the broker's refresh sentinel — see Transport A). The framework does not manage refresh; it simply re-invokes the plugin's async credential method. Examples: `Token`/JWT and `Basic` (long-lived, no refresh); OAuth2 (short-lived access token the plugin renews against the IdP) and Athenz (short-lived role tokens the plugin renews against ZTS). + +- **Multi-round challenge/response.** The client and broker exchange multiple frames before the broker grants access. The exchanged bytes are opaque to the framework; only the implementation knows how to interpret a challenge and compose a response. Example: SASL (Kerberos). + +### A note on mTLS + +Mutual TLS (mTLS) is sometimes called an "authentication method", but the TLS material and handshake are a **transport-layer concern** rather than a credential-carrying authentication plugin (in v5 the mTLS case is handled by the built-in `TlsAuthentication` plugin — see below). The client's certificate and private key are configured at the `PulsarClient` builder level and attached to both transports' `SSLContext`s. The broker authenticates the client by inspecting the certificate chain presented during the TLS handshake (its server-side `AuthenticationProviderTls` reads `SSLContext.getSession()`). No `auth_data` payload is exchanged at the Pulsar protocol layer. + +The v4 client conflated this by exposing `AuthenticationTls` as an `Authentication` plugin whose main purpose was to point the SSL layer at certificate files. **In v5, the TLS key and certificate for mTLS are configured directly at the `PulsarClient` builder**, not in an authentication plugin. mTLS authentication is then represented by the built-in `TlsAuthentication` plugin, which carries no TLS material — it only makes the binary protocol set `auth_method_name=tls` on `CommandConnect` (the broker reads the client certificate from the TLS handshake). The compatibility layer maps legacy v4 `AuthenticationTls` configuration onto this builder-level TLS material **plus** the built-in `TlsAuthentication` plugin, and the same implementation backs both the v4 and v5 client APIs in Pulsar 5.0. + +### Concrete examples + +| Method | Style | Binary `auth_data` | HTTP encoding | Notes | +|---|---|---|---|-------------------------------------------------------------------------------------| +| Token / JWT | single-pass | `utf8(jwt)` | `Authorization: Bearer ` | Most common deployment | +| Basic | single-pass | `utf8(user:pass)` | `Authorization: Basic ` | | +| OAuth2 | single-pass | `utf8(access_token)` | `Authorization: Bearer ` | Token-exchange step uses HTTP to the IdP | +| Athenz | single-pass | `utf8(role_token)` | custom `Athenz-Role-Auth: ` header | ZTS exchange uses Athenz SDK | +| SASL | multi-round | `sasl_initial_bytes`, then `CommandAuthResponse` rounds | custom `SASL-Token` / `State` / `SASL-Server-ID` headers, multi-round via repeated `401` (no `WWW-Authenticate`) | | +| mTLS | transport-layer | empty (`auth_method_name=tls`) | n/a (cert presented at TLS handshake) | TLS material configured at the builder; the built-in `TlsAuthentication` plugin sets `auth_method_name=tls` on the binary protocol | + +### Key concepts used in this proposal + +- *Capability segregation* — an interface describes one cross-cutting concern; an implementation that needs to expose several concerns implements several interfaces. The opposite is the *kitchen-sink* shape, where a single interface declares every concern and implementations leave most methods returning `null`. + +- *Async authentication* — completing authentication-related work via `CompletableFuture` so the calling thread (typically a Netty I/O thread on the connection path) is not blocked while the auth provider performs other I/O (e.g., calling a remote token endpoint). + +- *Single-pass vs challenge-response* — the two authentication styles described above. Reflected in the v5 interfaces. + +- *Configuration vs initialization* — in v5 these are separate concerns. **Configuration** is supplying the plugin its `authParams` (the path to the token file, the OAuth2 issuer URL, etc.) — static, plugin-specific data, independent of when/where the plugin runs. **Initialization** is when the plugin is given runtime services (a `PulsarHttpClient` factory, a scheduler, an OpenTelemetry handle) and may do I/O to prepare itself. The v4 interface conflated these with `configure(Map)` + `start()`; v5 splits them cleanly so the same plugin instance can be constructed and configured at one time and initialized later when a `PulsarClient` is built. + +# Motivation + +The v4 client authentication surface has five concrete, observable problems that are not addressed by any in-flight work. + +### 1. The Netty event loop blocks under token refresh. + +`ClientCnx.handleAuthChallenge()` is invoked when the broker pushes `CommandAuthChallenge`. The broker also uses this command, with payload `AuthData.REFRESH_AUTH_DATA_BYTES`, to signal "your short-lived credential is about to expire or has expired — fetch a new one and reply." `ClientCnx` services this push synchronously: it calls `authentication.getAuthData(remoteHostName)` and then `authenticationDataProvider.authenticate(challenge)` on whatever thread delivered the channel-read event — namely, the Netty I/O loop. For `AuthenticationToken` and `AuthenticationTls` this is fast, but for `AuthenticationOAuth2` it can stall the loop while a token endpoint is hit, and for `AuthenticationAthenz` while ZTS is queried. A single slow refresh therefore halts every I/O multiplexed onto that loop — producers, consumers, lookups — across every connection. + +### 2. `AuthenticationDataProvider` is a kitchen sink. + +A single interface declares 13 methods covering three unrelated concerns: TLS material (cert chain, private key, file paths, keystore params, truststore stream — 7 methods), HTTP request authentication (auth-type, headers — 3 methods), and binary-protocol authentication (command data, plus SASL `authenticate(AuthData)` for challenge-response — 3 methods). Every built-in implementation returns `null` or `false` from the methods it does not care about. Implementers cannot tell at compile time which transports an `Authentication` actually supports — the answer is "whichever combination of getters happen to return non-null at runtime." There is no compile-time guarantee that an impl claiming `hasDataForTls()` will return non-null from `getTlsCertificates()`, and no clean way to write a composite implementation (say, mTLS *and* OAuth) without re-hashing this implicit contract. + +### 3. Authentication plugins such as the plugin for OAuth2 cannot share resources with the rest of the runtime. + +`AuthenticationOAuth2` needs HTTP to talk to its identity provider. Today the OAuth2 implementation creates its own private `DefaultAsyncHttpClient` (built in `FlowBase` and shared with `TokenClient` and `DefaultMetadataResolver`) because the API surface gives it nowhere to ask "the client" for an HTTP client, and `FlowBase` even constructs a private `DefaultPulsarSslFactory` to configure it. Apache Pulsar issue [#24795](https://github.com/apache/pulsar/issues/24795) is exactly this complaint — operators want to control the HTTP client (proxies, retry policy, observability) without forking — and PR [#24944](https://github.com/apache/pulsar/pull/24944) is the workaround that consolidated what were previously *three separate* clients (one each in `FlowBase`, `TokenClient`, and `DefaultMetadataResolver`) down to the single shared one. The root cause is that the `Authentication` SPI gives implementations no hook to acquire shared client services. Beyond a plain HTTP client, plugins also need to share the Netty DNS-resolution configuration and DNS cache. + +### 4. The PIP-337 SSL Factory interface is a kitchen sink that exposes implementation details and prevents a clean separation of concerns. + +`PulsarSslConfiguration` — the value object that `PulsarSslFactory` implementations consume — carries a field `AuthenticationDataProvider authData`, which forces the SSL layer in `pulsar-common` to depend on the entire v4 auth SPI even though it only needs TLS material. This coupling exists because v4 `AuthenticationTls` could override the client's TLS configuration, and that override was wired through `PulsarSslConfiguration` instead of a clean integration point. + +The interface itself is also a kitchen sink: `initialize(PulsarSslConfiguration)`, `needsUpdate()`, `update()`, `createInternalSslContext()`, `getInternalSslContext()`, and `getInternalNettySslContext()` expose implementation details — the order in which a default file-based implementation rebuilds its state on rotation — as public methods. A custom plugin should not have to deal with any of this; it should answer with a fully configured TLS object for the use case it supports (for example "TLS for the Pulsar binary protocol client", "TLS for the admin HTTP client", or "TLS for OAuth2 token-endpoint calls") and deliver a rebuilt object through a reload callback when its material changes — keeping rebuild orchestration internal to the plugin instead of spread across every consumer. The SPI is also synchronous, so loading TLS material can block the Netty I/O loop — the same hazard as Motivation #1. Finally, PIP-337's API already traffics in Netty types (`getInternalNettySslContext()`), yet the consequences for custom factories used with the **shaded** Pulsar client — where Netty is relocated — are undocumented and unresolved; this PIP documents the packaging requirement explicitly. + +The decoupling matters operationally, not just structurally. Many organizations operate under security policies that forbid storing TLS private keys in files at all — keys must stay inside an HSM or KMS, or be delivered in-memory by a workload-identity system (e.g. SPIFFE/SPIRE). A TLS SPI that asks the plugin only for a working TLS object — and confines rotation to a reload callback — is what makes such integrations practical. PIP-337 nominally allows a custom factory, but its surface forces the implementer to re-own context construction and rotation and couples them to the v4 auth SPI, so in practice those deployments fork the SSL layer instead. Replacing PIP-337 with an SPI decoupled from the implementation closes this gap. + +The same configurability is what makes **FIPS-compliant TLS transport** practical. Deployments under FedRAMP, DoD, PCI, or healthcare regimes must terminate TLS through a **FIPS-140-validated cryptographic provider** (e.g. BC-FIPS) running on a non-native engine — never through a non-validated native library. Pulsar's TLS configuration must therefore let an operator (a) select the TLS **engine** (JDK, not native OpenSSL/BoringSSL) per component, (b) pin the **JSSE (SSLContext) provider** that builds the TLS context on that engine, and (c) pin the **JCA (crypto) provider** that parses the key material and manufactures the `PrivateKey`/`X509Certificate` objects handed to that context. Neither provider axis implies the other, and a deployment that pins only one is FIPS-*shaped* rather than FIPS-compliant. This PIP's TLS SPI delivers all three: the engine is already selectable and wired through every server component and the client; `TlsPolicy.jsseProvider` and `TlsPolicy.jcaProvider` are new (see [Goals](#in-scope)). The provider pair to configure, why both axes are required, and how a name is resolved are in the [Detailed Design](#redesigned-pip-337-ssl-provider-pulsartlsfactory). The broader FIPS-compliance concerns beyond the TLS transport — approved algorithms in message encryption and authentication, FIPS distribution packaging, and a fail-fast FIPS-mode validation switch — are a separate effort (see Out of Scope); this PIP covers the TLS-transport slice. + +FIPS 140-3 also governs how a **Sensitive Security Parameter** (here, a TLS private key) may cross the cryptographic module's boundary, and the requirement differs by assurance level — which is why both the file-based and the custom-factory tier matter. A **Level 1** software module such as BC-FIPS permits plaintext key entry through its API from within the same operational environment, so Pulsar parsing a PEM private key and handing it to the provider is compliant: the file-based `TlsPolicy`, with both provider axes pinned, serves that case directly. Higher assurance levels require encrypted key entry or a trusted channel: a **Level 3** deployment keeps keys inside a separate hardware module (a PKCS#11 HSM) whose keys never leave its boundary, so the private key is never handed to Pulsar at all — it is referenced by handle. That is the HSM / never-in-files case above, served by a custom `PulsarTlsFactory` that builds its TLS context against the HSM. The decoupled SPI serves both levels without special-casing either. + +### 5. Challenge-response authentication is a maintenance burden because its handling is scattered across the stack. + +Multi-round (challenge/response) authentication has no single integration point, so its logic is duplicated and entangled with unrelated concerns. On the binary protocol, `ClientCnx.handleAuthChallenge()` interleaves the initial connect, broker-pushed credential refresh, and SASL challenge rounds on one synchronous path. On HTTP there is no generic multi-round driver at all: the SASL plugin implements the `401`→resubmit→`200` loop itself (`AuthenticationSasl.authenticationStage(...)`), and the admin (JAX-RS) client and the HTTP-lookup client each drive it separately, so the loop is effectively re-implemented per HTTP client API — even though both APIs run over the same AsyncHttpClient transport. The SASL plugin even takes over request construction, re-issuing the exchange as a `GET` to the original URI regardless of the original method. The result is costly to maintain and hard to extend: supporting another challenge-response scheme (such as HTTP digest) would mean changing the plugin, both HTTP client integrations, and the binary connect path instead of implementing one well-defined capability. + +# Goals + +## In Scope + +This PIP introduces new API and implementation across the existing `pulsar-client-api-v5` and `pulsar-client-v5` modules created by PIP-466, plus two new small, dependency-light SPI modules — `pulsar-tls-factory-api` (the TLS factory SPI) and `pulsar-http-client-api` (the HTTP client SPI, which depends on the former) — so the sibling broker-side PIP can depend on them without importing a client artifact (see [Resolved design decisions](#resolved-design-decisions)): + +1. **Asynchronous, capability-segregated v5 authentication SPI** in `org.apache.pulsar.client.api.v5.auth` +2. **Two configuration paths** matching the existing v4 idioms: + - **Programmatic**: user constructs an `Authentication` instance (typically via a constructor or builder) and passes it to `PulsarClient.builder().authentication(myAuth)`. `configure(...)` is NOT called by the framework in this path (the instance is assumed already configured by the caller). + - **String-based**: user supplies `authPluginClassName` + `authParams` (JSON map or existing `key:val,key:val` String format). The framework reflectively instantiates the class via its no-arg constructor, calls `configure(parsedAuthParams)` once, then `initializeAsync(ctx)` once. Matches the existing `AuthenticationUtil.create(...)` semantics. + +3. **`PulsarHttpClient` SPI** in `org.apache.pulsar.http`, with **framework-managed lifecycle**. The framework owns Netty event loops, timers, DNS caches, and TLS material refresh integration; plugins describe what kind of HTTP client they need via a `PulsarHttpClientConfig` (timeouts, a `TlsPurpose` selecting dedicated TLS material) and obtain an instance via `AuthenticationInitContext.httpClientFactory().newHttpClient(config)`. The framework MAY issue multiple `PulsarHttpClient` instances per `PulsarClient` for different uses — OAuth2's mTLS exchange to the IdP and a third-party plugin's own HTTP endpoint may each target a different trust domain and so need a different TLS configuration, but they share the underlying event loop / timer / DNS resources. The implementation is framework-owned and backed by AsyncHttpClient; the HTTP *backend* is deliberately not pluggable (see the Detailed Design). + +4. **Redesigned PIP-337 SSL provider** — purpose-driven, no kitchen-sink config. The existing `PulsarSslFactory` / `PulsarSslConfiguration` surface is completely removed and replaced in Pulsar 5.0 and in v5 client's TLS configuration and integration points. This change also impacts the broker side. + +5. **v5 client builder TLS configuration**, including mTLS, at the client configuration level (mTLS may also be used for TLS auth). This subsumes the experimental PIP-466-era `org.apache.pulsar.client.api.v5.config.TlsPolicy` sketch in `pulsar-client-api-v5` (see the [Detailed Design](#detailed-design)). The TLS configuration covers the settings a Pulsar deployment needs to configure TLS transport: cert/key/trust material (PEM or keystore), ciphers and protocols, hostname verification, the TLS **engine** (JDK vs native OpenSSL), and — goals of this PIP — **any JSSE (SSLContext) provider** via `TlsPolicy.jsseProvider` plus **any JCA (crypto) provider** via `TlsPolicy.jcaProvider`. Together with the engine selection already wired through every server component (broker binary/web, proxy, websocket, functions-worker) and the client, those two axes make a **FIPS-compliant TLS transport configurable** — a validated provider on the JDK engine, with the key material parsed and held inside the validated module, and no reliance on a non-validated native engine (BoringSSL); the pair to set and why both are needed are in the [Detailed Design](#redesigned-pip-337-ssl-provider-pulsartlsfactory). The same configurability is available to custom `PulsarTlsFactory` implementations, which may source their material and provider however they choose. + +6. **Compatibility bridge** in `pulsar-client-v5`, in both directions: + - `LegacyV4AuthenticationAdapter` — wraps a v4 `Authentication` instance as a v5 `Authentication` declaring the right capability for its style. v4 calls are always off-loaded to a separate, dedicated executor (`ctx.blockingExecutor()`); the Netty event loop never runs the v4 plugin's I/O. + - `V5ToV4AuthenticationAdapter` — exposes a v5 `Authentication` through the v4 `Authentication` interface that `ClientCnx` already drives, so the v5 SPI can back the v4 client API and built-in v4 shims. + +7. **Pulsar 5.0 client's internal implementation migration to use the v5 Authentication SPI and TLS SPI**. This applies also to the v4 client API usage in Pulsar 5.0 client since it provides both v4 and v5 client APIs. + +## Out of Scope + +- **Broker-side authentication.** Pulsar's broker-side `AuthenticationProvider` / `AuthenticationState` interfaces have their own set of design problems (PIP-97 only covered the async basics). They are out of scope; a sibling PIP will address them with the same design principles. + +- **Removal of the v4 `Authentication` interface or its deprecated methods.** The v4 surface — including the deprecated `getAuthData()` no-arg and `configure(Map)` methods — is retained indefinitely for source compatibility. Their bodies are re-implemented on top of the v5 SPI as part of in-scope item #7, but the method signatures stay. + +- **Non-Java client SDKs.** Each non-Java SDK (Python, Go, C++, Node.js) follows its own auth model and will be addressed by per-SDK PIPs. + +- **Off-loading the proxy's own broker-client credential I/O (known gap).** Motivation #1's event-loop-safety guarantee is delivered for `PulsarClient` / `PulsarAdmin` (and the broker's outbound clients, which are genuine clients that bind the framework's `ClientAuthenticationServices`). The **proxy's** connection to the broker is not a `PulsarClientImpl`: its lookup path uses a bare `ConnectionPool` and its data path a hand-rolled Netty `DirectProxyHandler`, and neither binds the client auth services (bounded blocking executor, framework HTTP client factory). A proxy configured with a blocking-credential broker-client plugin (e.g. OAuth2) therefore still runs that credential fetch inline on the proxy's Netty loop — **the same behavior as v4, not a regression introduced by this PIP**. Closing it requires reordering proxy startup (the broker-client `Authentication` is created and started before the proxy's event loop, DNS resolver, and TLS factory exist) plus an async rework of `DirectProxyHandler`'s inline `getAuthData()`; it is deferred to a follow-up rather than bundled into this change. + +- **The broader FIPS-compliance mode.** This PIP covers the **TLS-transport** requirements for FIPS: a configurable TLS engine (JDK, not native BoringSSL) wired through every component plus the two configurable provider axes (Motivation #4, Goal #5). It does **not** define a full FIPS-mode profile: FIPS-approved algorithms in message encryption (key-wrap) and authentication (password hashing, token signing), a FIPS distribution/packaging variant (shipping `bc-fips` and excluding non-validated `bcprov` / `netty-tcnative-boringssl` / Conscrypt), and a fail-fast `fipsMode` validation switch are a **separate effort** — Pulsar-wide in scope and independent of this SPI. Concretely, the shipped `pulsar-server` distribution today bundles the **non-FIPS** BouncyCastle provider (`bcprov-jdk18on` / `bcpkix-jdk18on`) and explicitly excludes `bc-fips`, and it ships no `bctls-fips` (the jar registering `BCJSSE`) at all; because the two BouncyCastle families declare the same `org.bouncycastle.*` classes under different signers they cannot coexist on one classpath, so until that packaging effort lands a FIPS deployment assembles the provider classpath itself. The in-tree `pulsar-client-test-bcfips` module assembles a classpath that way — excluding the non-FIPS BouncyCastle jars in favour of `bc-fips` — but it covers the crypto side only: it ships no `bctls-fips` and sets neither provider key, so it is not an end-to-end FIPS TLS test. This PIP deliberately provides only the TLS-transport configurability those deployments require, so the two efforts compose without one blocking the other. + +# High Level Design + +This proposal centers on a small core `Authentication` interface plus four narrow, opt-in **capability interfaces**, a framework-managed `PulsarHttpClient` SPI, and a purpose-driven `PulsarTlsFactory` SPI that replaces PIP-337. This section describes the shape and the reasoning; the full type listings live in the [Detailed Design](#detailed-design). + +## Design principles + +Four principles run through everything below; they are also the yardsticks the [Resolved design decisions](#resolved-design-decisions) at the end of this document are argued against. + +1. **Simple things stay simple.** The overwhelmingly common deployment — TLS material in PEM or keystore files, a single trust domain, one of the built-in auth methods — keeps file-path-level ergonomics and never sees a provider, purpose, or capability type. The new SPIs are opt-in layers *underneath* that configuration, not a new prerequisite for it. (See the [three usage tiers](#the-pulsartlsfactory-spi-pip-337-replacement) of the TLS surface.) +2. **Plugins exist for what files cannot cover.** Many organizations have security policies that TLS private keys must never be stored in files — keys live in an HSM/KMS or are delivered in-memory by a workload-identity system. Because the PIP-337 replacement asks a factory only for ready-built TLS objects — material sourcing and key handling stay entirely inside the factory, and key material never crosses a Pulsar API — those deployments become first-class instead of requiring a fork of the SSL layer. +3. **One concern per interface, asynchronous by construction.** A plugin declares exactly the transports and styles it supports as capability interfaces (the 2×2 matrix below); every credential-producing method returns a `CompletableFuture`, so the Netty event loop is never blocked. +4. **The framework owns shared runtime resources.** HTTP clients, event loops, DNS caches, and schedulers are framework-managed and handed to plugins through contexts; a plugin never builds parallel infrastructure. + +## Capability-segregated authentication SPI + +Every plugin implements the core `Authentication` interface, which carries only lifecycle: a `configure(Map)` hook (called for the string-based path only), an `initializeAsync(AuthenticationInitContext)` hook that may do I/O, a `capability(Class)` lookup for delegating wrappers, and `close()`. + +The actual credential work lives on **four capability interfaces** — one per cell of a 2×2 matrix whose axes are the two transports and the two authentication styles described in the [Background](#how-pulsar-client-authentication-works). This matrix *is* the conceptual model: an authentication plugin is the core lifecycle interface plus whichever cells it supports, and nothing else. + +| | Pulsar binary protocol | HTTP / REST | +|------------------------|----------------------------------|------------------------------------------------| +| **Single-pass** | `BinaryAuthDataProvider` | `HttpAuthHeadersProvider` | +| **Challenge/response** | `BinaryAuthChallengeHandler` | `HttpAuthChallengeHandler` (SASL style) | + +- `BinaryAuthDataProvider` — single-pass credential for the binary protocol. +- `HttpAuthHeadersProvider` — single-pass credential for HTTP. +- `BinaryAuthChallengeHandler` — multi-round challenge/response for the binary protocol. +- `HttpAuthChallengeHandler` — multi-round challenge/response for HTTP in the **SASL style** (repeated `401` carrying Pulsar's custom SASL headers, as used by the existing SASL-over-HTTP mechanism). + +Placing the built-ins on the matrix: Token, Basic, OAuth2, and Athenz occupy the **single-pass row** (both cells); SASL occupies **all four cells** (an initial frame plus challenge rounds, on each transport); the built-in `TlsAuthentication` plugin occupies only the binary single-pass cell, with empty credential bytes (see below). The four names deliberately read directly off the matrix — the transport prefix names the row's transport, and the suffix names the style (`…Provider` = single-pass, `…Handler` = challenge/response); see [Resolved design decisions](#resolved-design-decisions). + +These four are the integration points today's protocols need; the model is open-ended. Additional capability interfaces can be introduced later if a future mechanism needs to plug into the binary or HTTP authentication flow in a way these four don't capture, without disturbing existing plugins. HTTP multi-round auth is the first place this is expected: the SASL style above is custom to Pulsar, whereas standard `WWW-Authenticate`-based schemes (e.g. HTTP digest) follow a different exchange. The design therefore anticipates a **second, standard-style** HTTP challenge-response interface alongside `HttpAuthChallengeHandler`, with the framework's HTTP auth **driver** (below) selecting the handling by which interface a plugin implements. Only the SASL-style interface ships in this PIP — a deliberate decision; see [Resolved design decisions](#resolved-design-decisions). + +**HTTP auth drivers.** Unlike the binary protocol — where `ClientCnx` is the single place that runs the `CommandAuthChallenge`/`CommandAuthResponse` loop — HTTP is reached through two client APIs: JAX-RS (Jersey) for the admin client and raw AsyncHttpClient for HTTP topic lookup, both running over the same AsyncHttpClient transport (see [Transport B](#transport-b--httphttps-rest-api-admin-client-http-topic-lookup)). In v4 the multi-round loop lives inside the SASL plugin itself, which also re-issues the exchange as a `GET` to the original URI. v5 instead puts the loop in a framework-side **driver** — a single `401`→resubmit→`200` state machine behind a thin request/response adapter per client API — that calls the plugin only to compute each round's response. For backward compatibility the SASL driver preserves the existing behaviour (custom headers, the `GET`-to-original-URI takeover); a future standard-style driver will handle `WWW-Authenticate`/digest. The driver is internal to the framework and is not part of the plugin SPI. + +For convenience, one **composite interface** bundles the overwhelmingly common combination so an implementation can declare a single interface: + +- `SinglePassAuthentication extends Authentication, BinaryAuthDataProvider, HttpAuthHeadersProvider` — a single-pass credential served over both transports. + +The composite is purely a convenience; an implementation is free to implement the individual capability interfaces directly and combine only the ones it needs. (A four-capability `ChallengeResponseAuthentication` composite was considered and left out — its only plausible implementor is the built-in SASL plugin, which simply lists its interfaces.) For the typical plugin author — a custom single-pass credential served over both transports — the entire task is: implement `SinglePassAuthentication` and return the credential from its two async methods. The full matrix only becomes visible when a mechanism genuinely spans it (today, only SASL does). + +**Built-in `TlsAuthentication` plugin (mTLS).** mTLS is not a capability that other plugins mix in — it is a small, self-contained `Authentication` implementation. The built-in `TlsAuthentication` class implements `Authentication` and `BinaryAuthDataProvider`, reporting `authMethodName() == "tls"` (fixed) with empty `auth_data` so the binary protocol authenticates via the TLS handshake. It carries **no** TLS material — certificates and keys are configured at the `PulsarClient` builder (see [A note on mTLS](#a-note-on-mtls)) — and exists only so a client that wants mTLS auth can select it and have the binary protocol send `auth_method_name=tls` on `CommandConnect`. + +These interfaces replace v4's kitchen-sink `AuthenticationDataProvider`: capabilities make a plugin's supported transports and styles visible at compile time, and composing concerns (e.g., binary + HTTP single-pass) is a matter of implementing two interfaces rather than re-hashing an implicit contract. The core `Authentication` acts as an explicit **capability factory**: the framework discovers what a plugin supports *only* through `capability(Class)` — never via `instanceof` on the plugin instance — so an implementation is free to serve a capability from a separate internal class (or forward a wrapped delegate's) instead of implementing it on the plugin type itself. The default implementation returns `this` when the plugin implements the requested interface directly, so the common case needs no override; there is no registry. Making the lookup the single mechanism removes a standing trap: if discovery were split between `instanceof` and the lookup, a wrapper forwarding its delegate's capabilities would silently break every `instanceof` call site. All capability methods are asynchronous (`CompletableFuture`), so credential acquisition never blocks the Netty event loop. + +**Threading contract (normative).** Because the SPI is asynchronous, the framework MAY invoke a native v5 plugin's capability methods (`getAuthDataAsync`, `getHttpHeadersAsync`, `respondToChallengeAsync`, `respondToHttpChallengeAsync`) **directly on the Netty event loop**. An implementation therefore **MUST return promptly and MUST NOT block the calling thread**: any blocking or long-running work (a token-endpoint round-trip, a ZTS exchange, disk or network I/O) MUST be off-loaded onto the executors the framework supplies through `AuthenticationInitContext` — `blockingExecutor()` for potentially-blocking calls, `scheduler()` for delayed or periodic work — and the returned future completed from there. This is the native-plugin form of Motivation #1's guarantee. The framework enforces it *for wrapped v4 plugins* by off-loading every v4 call to `blockingExecutor()` (see the [compatibility bridge](#compatibility-bridge-for-v4-plugins)); a **native** v5 plugin owns its own off-loading, since the framework does not interpose the blocking executor in front of it. + +Cross-round runtime state (a SASL conversation, a multi-stage HTTP handshake) lives on the per-call context's **state slot**, not on method parameters (see [per-call contexts](#authenticationcallcontext--httpauthcallcontext)). + +## Two configuration paths + +Both v4 idioms are preserved (see In-Scope item #2): the **programmatic** path, where the user constructs and configures the `Authentication` instance and `configure(...)` is not called by the framework; and the **string-based** path, where `authPluginClassName` + `authParams` drive reflective construction followed by one `configure(...)` and one `initializeAsync(...)`. + +## Initialization and per-call contexts + +`AuthenticationInitContext` is passed once to `initializeAsync(...)` and exposes the framework's shared runtime services: a `PulsarHttpClientFactory`, a `ScheduledExecutorService` for scheduled work, a separate `Executor` for potentially-blocking work, a `Clock`, an `OpenTelemetry` handle, and the client instance id. The framework owns and closes these shared services; the plugin may retain references for its lifetime and releases its own resources in `close()`. (A plugin's configured params are delivered through `Authentication.configure(Map)` on the string-based path, so the init context carries no separate copy.) + +`AuthenticationCallContext` (binary) and `HttpAuthCallContext` (HTTP) are cheap per-call objects that carry transport-specific routing (broker host, or request URI) and expose a per-exchange **state slot** (`getStateObject`/`setStateObject`, keyed by class) so an implementation can retain conversation state across challenge-response rounds. The slot's lifetime equals one authentication exchange; concurrent authentications to different brokers each get their own context, so in-flight handshakes don't collide. (This general state slot subsumes what a transport-specific `previousResponseHeaders` parameter would otherwise carry.) + +## The `PulsarHttpClient` SPI + +Auth plugins that need HTTP (OAuth2's token endpoint and well-known metadata; a JWKS endpoint) obtain a client from the framework rather than constructing their own — fixing the v4 problem where `AuthenticationOAuth2` spins up its own private `DefaultAsyncHttpClient` (and a private `DefaultPulsarSslFactory` to configure it) instead of sharing the runtime's (Motivation #3). Athenz is the exception: its ZTS role-token exchange runs on the Athenz SDK's own HTTP transport, which the SDK owns and this PIP does not reroute through the framework (consistent with the Athenz row in [Concrete examples](#concrete-examples)). The **framework manages HTTP client lifecycle** (Netty event loop, timer, DNS cache, TLS material refresh) for the plugins it does serve; such a plugin describes what it needs via a `PulsarHttpClientConfig` and receives an instance from `AuthenticationInitContext.httpClientFactory()`. + +The framework may hand out **multiple** `PulsarHttpClient` instances per `PulsarClient`: OAuth2's mTLS exchange to the IdP and a third-party plugin's own HTTP endpoint can each target a different trust domain and so need a different TLS configuration, while sharing the underlying event-loop / timer / DNS resources. This need is not hypothetical. When the application terminates mTLS itself — rather than delegating the secure channel to a service mesh (e.g. Istio) — each outbound trust domain (the Pulsar cluster, the identity provider, a JWKS endpoint) presents its own client certificate and trust anchors. Workload-identity frameworks make this concrete: under [SPIFFE](https://spiffe.io/)/SPIRE a workload receives a distinct X.509-SVID and trust bundle per trust domain it talks to, so a single process legitimately holds several non-interchangeable key materials at once. Binding TLS configuration to a *purpose* rather than to one process-wide default is what lets the framework route each `PulsarHttpClient` to the right material. + +The `PulsarHttpClient` model — a small request/response SPI with framework-owned resources and purpose-driven TLS — is deliberately not auth-specific. The same model can later serve other HTTP needs inside Pulsar (for example JWKS fetching in the broker-side OIDC provider, which today embeds its own HTTP client) under the sibling broker-side PIP; to enable that reuse the SPI lives in its own small, dependency-light `pulsar-http-client-api` module (see [Resolved design decisions](#resolved-design-decisions)). + +## The `PulsarTlsFactory` SPI (PIP-337 replacement) + +PIP-337's `PulsarSslFactory` / `PulsarSslConfiguration` are removed entirely (Motivation #4) and replaced by a deliberately small, purpose-driven **instance factory**. A **`TlsPurpose`** is a simple named key that identifies *why* TLS is requested and in what role: a role (client or server) and a well-known name (`CLIENT_DEFAULT`, `CLIENT_OAUTH2`, `BROKER_CLIENT`; `BROKER`, `PROXY`, `WEB`). When a purpose has no material configured, resolution is **terminal**: a client purpose resolves to the system default (OS trust store, no client certificate), a server purpose is a configuration error. Client-side consumers may additionally pass the **destination endpoint** (host and port) as a per-request hint, so a factory that serves per-destination material (multi-cluster deployments, per-target workload identities) can specialize; factories are free to ignore it. The factory answers `createInstance(purpose, instanceClass)` with a **fully configured TLS object** of the requested class — `io.netty.handler.ssl.SslContext`, Jetty's `SslContextFactory.Server`, or `javax.net.ssl.SSLContext` (the fallback baseline) — or `Optional.empty()` when it does not support that combination, in which case the framework synthesizes the richer object from the JDK `SSLContext` the factory does provide. A factory therefore need only implement whichever classes it can build directly; as long as it supplies at least the `SSLContext` for a purpose, the framework can derive the Netty and Jetty objects from it. A second overload additionally subscribes a reload callback (`Consumer onLoadOrReload`) that receives the instance on first load and a rebuilt instance whenever the underlying material changes. *How* the factory sources key material and builds the objects — files, a KMS API, an HSM-backed `KeyManagerFactory` — is entirely factory-internal; nothing material-shaped appears in the SPI, and key material never crosses a Pulsar API. The SPI deliberately omits PIP-337's consumer-driven rebuild choreography (`needsUpdate()` / `update()` / `createInternalSslContext()` call-ordering) and its kitchen-sink `PulsarSslConfiguration`. + +Rotation is push-based: the reload callback delivers the rebuilt instance directly. Server-side consumers subscribe — the broker/proxy binary listeners swap the `SslContext` they use for new connections, and the Jetty web service either receives a self-reloading `SslContextFactory.Server` from the factory or has the framework drive Jetty's documented `SslContextFactory.reload(...)` API from an `SSLContext` subscription (see the [Detailed Design](#detailed-design)). Most client-side consumers don't subscribe: they request the instance per new connection, picking up rotated material naturally. (Synchronous client integration points — an HTTP library's engine factory, or the proxy's broker-facing data path — instead hold the subscribing overload's current instance; see the [Detailed Design](#detailed-design).) The default `FileBasedTlsFactory` loads PEM and keystore files and reloads on rotation; it is **immutable after construction** — the v5 builder composes the final purpose→policy map (including material contributed by the v4 `AuthenticationTls` bridge) before creating it. A custom factory can integrate a Key Management System (KMS) and may differ between client and broker sides. + +Because the well-known classes include Netty (and Jetty) types, a custom factory must be packaged to match the relocation of the artifact it plugs into. The concern is confined to the **shaded v4 client distributions** — `pulsar-client-shaded`, `pulsar-client-admin-shaded`, and the `pulsar-client-all` fat jar — which relocate Netty (`io.netty` → `org.apache.pulsar.shade.io.netty`). A factory used with those must be published as a **relocated** plugin artifact whose Netty/Jetty references are rewritten to the shaded names (without bundling the relocated classes); the plain, unrelocated artifact serves `pulsar-client-original` / `pulsar-client-admin-original` and all server-side components, which are not shaded. The **v5 client** (`pulsar-client-v5` / `pulsar-client-api-v5`, PIP-466) is published **unshaded** and therefore has no such problem — a single plain plugin artifact works. Applications on the v5 client that nonetheless need shading to resolve a dependency conflict perform that relocation themselves, in their own build, and relocate the plugin along with everything else as part of it. PIP-337 has the same constraint on the shaded v4 client today, undocumented; this PIP makes it explicit (see the shading note in the [Detailed Design](#detailed-design)). + +**Three usage tiers.** The TLS surface is layered so that complexity is proportional to need: + +1. **File paths (almost everyone).** TLS material in PEM or keystore files, configured essentially as today — broker/proxy via the existing `ServiceConfiguration` properties, the v5 client via a simple builder-level TLS configuration object. The default `FileBasedTlsFactory` is wired automatically; the user never names a factory, a purpose, or an instance class. +2. **Per-purpose policies (multi-trust-domain deployments).** Deployments where, say, the OAuth2 IdP needs different trust anchors or a different client certificate than the brokers configure an additional `TlsPolicy` for that specific purpose (`CLIENT_OAUTH2`). Still no custom code. +3. **Custom factory (keys never touch a file).** Organizations whose policy forbids on-disk private keys implement `PulsarTlsFactory` against their KMS / HSM / workload-identity system. The mandatory part is small: build a JDK `SSLContext` — typically from a `KeyManagerFactory`/`TrustManagerFactory` initialized with a provider-backed `KeyStore`, so keys stay non-extractable behind the JCA provider (PKCS#11, KMS JCA provider; signing happens inside the HSM/KMS) — and deliver rebuilt instances through the reload callbacks on rotation. Engine-level policy (protocol/cipher restrictions, client-auth mode, endpoint identification) can optionally accompany the context as a `javax.net.ssl.SSLParameters` — still zero non-JDK dependencies (see the well-known-class table). Richer per-stack objects (a native OpenSSL-based Netty context, even a BoringSSL "keyless" one; a keystore-backed Jetty factory with SNI support) are optional extras the factory may choose to supply. Key material never crosses a Pulsar API at all. + +**Two provider axes, orthogonal to the tiers.** A `TlsPolicy` carries two optional `java.security.Provider` names, because they answer different questions and neither implies the other: `jsseProvider` names the **JSSE** provider supplying the `SSLContext` and the `KeyManagerFactory`/`TrustManagerFactory`, while `jcaProvider` names the **JCA** provider supplying the `KeyStore`, `CertificateFactory` and `KeyFactory` engines that parse and hold the material. Both are plain value-level fields — a line in `broker.conf` or `client.conf`, i.e. tier 1, not a reason to write a custom factory — and both default to unset, meaning today's behaviour. The semantics, the FIPS pair, the resolution rules, and why both axes are required are in the [Detailed Design](#redesigned-pip-337-ssl-provider-pulsartlsfactory); a custom factory may interpret either field or ignore both. + +## Compatibility bridge for v4 plugins + +`LegacyV4AuthenticationAdapter` wraps an arbitrary v4 `Authentication` as a v5 `Authentication`, declaring the capability interfaces that match the v4 plugin's style. All v4 calls are off-loaded to a separate, dedicated executor (`ctx.blockingExecutor()`, kept distinct from the scheduler) so the Netty event loop never runs v4 plugin I/O. v4 plugins that supply TLS material (`hasDataForTls() == true`, e.g., `AuthenticationTls`) have that material registered with the client's `PulsarTlsFactory` and are represented by the built-in `TlsAuthentication` plugin. + +## `ClientCnx` async-driver carve-out + +The only change to the otherwise-untouched v4 client is a marker interface `AsyncAuthenticationDriver` (in `pulsar-client-api`, `.internal` subpackage). `ClientCnx` detects it and routes connect / refresh / challenge handling through the async API; plain v4 instances keep the existing synchronous path verbatim. This is generic across all challenge types. + +## Error model + +All async **authentication** failures complete the returned future exceptionally with a v5 `org.apache.pulsar.client.api.v5.PulsarClientException` — `AuthenticationException` (terminal), `GettingAuthenticationDataException` (transient/retryable), or `UnsupportedAuthenticationException` (capability not supported by a wrapped v4 impl). The `PulsarTlsFactory` and `PulsarHttpClient` SPIs, which don't depend on the v5 client API, instead report ordinary exceptions. Details and the v4-exception translation are in the Detailed Design. + +# Detailed Design + +## Design & Implementation Details + +### The `Authentication` core and capability interfaces + +The new core `Authentication` interface lives in `pulsar-client-api-v5`: + +```java +package org.apache.pulsar.client.api.v5.auth; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +public interface Authentication extends AutoCloseable { + /** + * Configuration step. Called once by the framework AFTER no-arg construction + * when the plugin was loaded reflectively from authPluginClassName + + * authParams. NOT called when the plugin was constructed programmatically + * by the user — that path presumes the instance is already configured. + * Default: no-op. Implementations override to read their parameters. + * + *

Configuration is intentionally separate from {@link #initializeAsync}: + * configuration is static plugin-level data (file paths, URLs, scopes); + * initialization gives the plugin runtime services and may do I/O. + */ + default void configure(Map authParams) {} + + /** + * Initialization step. Called once by the framework with runtime services + * after configuration. May do I/O; the returned future completes when the + * implementation is ready to serve credentials. + */ + CompletableFuture initializeAsync(AuthenticationInitContext ctx); + + /** + * Capability factory — the framework's ONLY discovery mechanism. The + * framework never uses {@code instanceof} on a plugin instance; it asks this + * method for each capability it may drive. An implementation may therefore + * serve a capability from a separate internal class, or forward a wrapped + * delegate's capability, rather than implementing it on this type. The + * default implementation returns {@code this} when the plugin implements the + * requested interface directly, covering the common case with no override. + * + *

Contract: results are STABLE once {@link #initializeAsync} has completed — + * the framework may look a capability up once and cache it for the client's + * lifetime. One object may serve several capabilities. Capability objects are + * owned by this plugin and released by {@link #close()}; the framework never + * closes them individually. All capability methods must tolerate concurrent + * invocation (multiple connections authenticate in parallel); only the rounds + * of a single exchange are serialized by the framework. + */ + default Optional capability(Class kind) { + return kind.isInstance(this) ? Optional.of(kind.cast(this)) : Optional.empty(); + } + + @Override default void close() throws Exception {} +} +``` + +The four capability interfaces, each in its own file in the same package — segregated by transport (binary vs HTTP) and style (single-pass vs challenge-response). An implementation declares the ones that match what it supports (e.g., a one-pass plugin that serves both transports implements both `BinaryAuthDataProvider` and `HttpAuthHeadersProvider`): + +```java +package org.apache.pulsar.client.api.v5.auth; + +import java.util.concurrent.CompletableFuture; + +/** Single-pass credential exchange for Pulsar binary protocol. */ +public interface BinaryAuthDataProvider { + /** Stable identifier sent in CommandConnect.auth_method_name */ + String authMethodName(); + + /** + * Produce a credential for the binary protocol connection. The returned + * {@link BinaryAuthData} carries the {@code auth_data} bytes for + * {@code CommandConnect}; the framework pairs them with {@link #authMethodName()}. + */ + CompletableFuture getAuthDataAsync(AuthenticationCallContext ctx); +} + +/** Single-pass credential exchange for Pulsar HTTP transport. */ +public interface HttpAuthHeadersProvider { + /** + * Produce the authentication headers for an outgoing HTTP request (e.g. + * {@code Authorization: Bearer }, or a custom header such as + * {@code Athenz-Role-Auth}). The returned {@link HttpAuthHeaders} are attached + * to the request; most implementations produce the same credential for every + * call. Completes exceptionally on failure (see the error model). + */ + CompletableFuture getHttpHeadersAsync(HttpAuthCallContext ctx); +} + +/** Multi-round challenge/response (SASL-style and custom protocols) for Pulsar binary protocol. */ +public interface BinaryAuthChallengeHandler { + /** + * Respond to a binary-protocol {@code CommandAuthChallenge}. The framework places + * the returned bytes in {@code CommandAuthResponse}. Completion is decided by the + * broker (it replies with {@code CommandConnected} when satisfied), so the handler + * does not signal it; cross-round conversation state is kept in the context's + * state slot. + */ + CompletableFuture respondToChallengeAsync(AuthenticationCallContext ctx, + AuthChallenge authChallenge); +} + +/** + * SASL-style multi-round challenge/response over HTTP. Handles the existing Pulsar + * SASL-over-HTTP mechanism, where the server returns HTTP {@code 401} carrying the + * challenge in Pulsar's custom SASL headers (e.g. {@code SASL-Token} / {@code State} / + * {@code SASL-Server-ID}) rather than the standard {@code WWW-Authenticate} header. + * Standard {@code WWW-Authenticate}-based schemes (e.g. HTTP digest) will be served by a + * separate interface; the framework's HTTP auth driver dispatches to whichever a plugin + * implements. + */ +public interface HttpAuthChallengeHandler { + /** + * Compute the headers for the next round. The server's challenge headers from the + * prior {@code 401} are available via {@link HttpAuthCallContext#serverChallengeHeaders()}; + * the returned {@link HttpAuthHeaders} are attached to the resubmitted request. + * Cross-round conversation state is kept in the context's state slot. + */ + CompletableFuture respondToHttpChallengeAsync(HttpAuthCallContext ctx); +} +``` + +**Binary challenge routing (normative).** The binary transport's driver (`ClientCnx`) routes exactly as follows, for every plugin: + +1. **Initial connect** → `capability(BinaryAuthDataProvider.class).getAuthDataAsync(ctx)`; the capability is required for any plugin used on the binary transport — absent, client construction fails with `UnsupportedAuthenticationException`. +2. **`CommandAuthChallenge` carrying the refresh sentinel** (`AuthData.REFRESH_AUTH_DATA_BYTES`, `"PulsarAuthRefresh"`) → `ClientCnx` **terminates the current exchange and starts a fresh one**, producing the new credential via that fresh exchange's `getAuthDataAsync(ctx)` — a new call context and a new state slot — and sends it as `CommandAuthResponse`. **Conversation state does NOT survive a REFRESH**: refresh is by definition "produce your current credential again", a fresh single-pass exchange, not a continuation of the prior conversation. The sentinel never reaches the challenge handler. +3. **Any other `CommandAuthChallenge`** → `capability(BinaryAuthChallengeHandler.class).respondToChallengeAsync(ctx, challenge)`; if the plugin does not expose the capability, the connection fails with `AuthenticationException` (matching v4, where a plugin without `authenticate(AuthData)` cannot answer a challenge). + +A mechanism spanning both cells (SASL) therefore sees: initial frame via rule 1, conversation rounds via rule 3, and — should the broker ever push the sentinel mid-conversation — a fresh exchange via rule 2 that restarts authentication from a new initial frame (the prior conversation's state does not carry over; a REFRESH begins a clean single-pass exchange). + +One optional **composite interface** bundles the overwhelmingly common combination. It adds no methods — an implementation may declare it instead of listing the individual capabilities, or implement the capability interfaces directly: + +```java +package org.apache.pulsar.client.api.v5.auth; + +/** Convenience: a single-pass credential served over both transports. */ +public interface SinglePassAuthentication + extends Authentication, BinaryAuthDataProvider, HttpAuthHeadersProvider {} +``` + +The built-in **`TlsAuthentication`** plugin handles mTLS. It is an ordinary `Authentication` implementation, **not** a capability interface: it implements `BinaryAuthDataProvider` with `authMethodName()` fixed at `"tls"` and empty `auth_data`, carries no TLS material (that is configured at the builder), and exists only to drive `auth_method_name=tls` on the binary protocol so the broker authenticates from the TLS-handshake certificate: + +```java +package org.apache.pulsar.client.impl.v5.auth; + +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.client.api.v5.auth.*; + +/** + * Built-in mTLS plugin. The certificate and private key are configured at the + * {@code PulsarClient} builder (see "A note on mTLS"), not here. This plugin only + * reports {@code authMethodName()} (default {@code "tls"}) with empty {@code auth_data}, + * so the binary protocol sends {@code auth_method_name=tls} on {@code CommandConnect} + * and the broker authenticates from the certificate presented during the TLS handshake. + */ +public class TlsAuthentication implements Authentication, BinaryAuthDataProvider { + public static final String DEFAULT_AUTH_METHOD_NAME = "tls"; + + public TlsAuthentication() { } + + @Override public String authMethodName() { return DEFAULT_AUTH_METHOD_NAME; } + + @Override public CompletableFuture initializeAsync(AuthenticationInitContext ctx) { + return CompletableFuture.completedFuture(null); + } + + @Override public CompletableFuture getAuthDataAsync(AuthenticationCallContext ctx) { + return CompletableFuture.completedFuture(new BinaryAuthData(new byte[0])); + } +} +``` + +**`AuthenticationFactory.tls()`.** The v5 `AuthenticationFactory` exposes a single **no-arg** `static Authentication tls()` that returns the built-in `"tls"` marker plugin (`TlsAuthentication`): it selects the `tls` auth method and carries **no** TLS material. mTLS material is configured in exactly one place — `builder.tlsPolicy(TlsPolicy.pem(trustCerts, certFile, keyFile))` — from which the transport reads it via the client TLS factory; the marker plugin only drives `auth_method_name=tls` on the binary protocol so the broker authenticates from the certificate presented during the TLS handshake. There is deliberately **no** material-carrying `tls(cert, key)` overload: it would re-open the mTLS-material-in-an-auth-entry-point shape this PIP otherwise removes and give a client two ways to say the same thing. (The separate v4 `AuthenticationTls` → `CLIENT_DEFAULT` bridge fold — where a *legacy* plugin's cert/key merge into the client policy at build time — is a compatibility mechanism and is retained; see the [TLS override hook](#legacyv4authenticationadapter-v4--v5).) + +### Value types + +The credential/challenge value types in `org.apache.pulsar.client.api.v5.auth`. `BinaryAuthData` carries the binary-protocol credential; `HttpAuthHeaders` carries the HTTP headers; `AuthChallenge` / `ChallengeResponse` carry the multi-round binary exchange. + +```java +/** + * Binary-protocol credential: the auth_data bytes for CommandConnect. The + * auth-method name is deliberately NOT part of this value — it comes from + * exactly one place, {@link BinaryAuthDataProvider#authMethodName()}, + * so a plugin cannot contradict itself (carrying the name on both the provider + * and the data value would leave "which one wins?" to the + * javadoc). Kept as a record rather than a bare byte[] so a future field is an + * additive change — provided the current constructor is retained as an explicit + * overload (adding a record component regenerates the canonical constructor, so + * the prior shape must be kept). + */ +public record BinaryAuthData(byte[] bytes) {} +``` + +```java +/** + * HTTP authentication headers an implementation produces for an outgoing request + * (and, for challenge-response, the headers carrying a server's challenge). Header + * names are canonicalised on construction (RFC 7230 §3.2); {@link #get} is + * case-insensitive; {@link #asMap} returns the canonical-cased view. + */ +public final class HttpAuthHeaders { + static HttpAuthHeaders empty(); + static HttpAuthHeaders of(String name, String value); + static HttpAuthHeaders of(Map headers); + Optional get(String name); // case-insensitive + Map asMap(); + // ... +} +``` + +```java +/** A binary-protocol CommandAuthChallenge payload handed to a BinaryAuthChallengeHandler. */ +public record AuthChallenge(byte[] bytes) {} + +/** + * Reply from respondToChallengeAsync — just the bytes for the next + * CommandAuthResponse. There is no completion flag: the broker decides when the + * handshake is finished (it sends CommandConnected); the implementation tracks any + * cross-round state of its own in the call context's state slot. Kept as a record + * rather than a bare byte[] so a future field is an additive change — provided the + * current constructor is retained as an explicit overload (adding a record component + * regenerates the canonical constructor, so the prior shape must be kept). + */ +public record ChallengeResponse(byte[] bytes) {} +``` + +### `AuthenticationInitContext` + +```java +public interface AuthenticationInitContext { + /** + * Factory for {@link PulsarHttpClient} instances. The framework manages + * lifecycle (event loop, timer, DNS cache, TLS material refresh integration); + * plugins describe what they need via {@link PulsarHttpClientConfig} and + * receive a configured instance owned by the framework. Multiple instances + * with different TLS / timeouts may be obtained for different uses + * (e.g., OAuth2's mTLS exchange to the IdP vs a third-party plugin's own + * HTTP endpoint). + * + *

A plugin that needs framework-managed HTTP MUST obtain it here rather + * than constructing its own — a private client defeats the framework's + * shared event-loop / DNS / refresh integration. + */ + PulsarHttpClientFactory httpClientFactory(); + + /** Scheduler for delayed / periodic authentication work (e.g. proactive + * credential renewal). Never the Netty event loop. Reserved for scheduled + * tasks — potentially-blocking work belongs on {@link #blockingExecutor()}. */ + ScheduledExecutorService scheduler(); + + /** + * Dedicated executor for off-loading potentially-blocking authentication + * work, kept separate from {@link #scheduler()} so that blocking calls + * cannot starve the scheduler's threads or delay scheduled tasks. The + * {@link LegacyV4AuthenticationAdapter} offloads every synchronous v4 + * plugin call here. Never the Netty event loop. Framework-owned and shared + * per PulsarClient: a small bounded cached pool (max 16 threads, created on + * demand and reaped after 60s idle) — a fixed framework default, not a + * client-builder knob. It is re-entrancy aware: a task submitted from one of + * its own threads runs inline on that thread, so a nested credential fetch + * (the v4 HTTP-lookup branch blocks a pool thread on a v5 future whose body + * off-loads here again) can never be rejected by the pool it is waiting on. + */ + Executor blockingExecutor(); + + /** Used by implementations that schedule against wall-clock. */ + Clock clock(); + + /** Telemetry root; framework defaults to OpenTelemetry.noop() if unset. */ + OpenTelemetry openTelemetry(); + + /** Stable id of the owning PulsarClient for logging correlation. */ + String clientInstanceId(); +} +``` + +The lifecycle is: the framework constructs a single `AuthenticationInitContext` per `PulsarClient`, calls `initializeAsync(ctx)` once when the client is built, and the implementation may retain references for the lifetime of the client. `Authentication.close()` releases any resources the implementation acquired; the framework owns and closes the shared services (HTTP clients, scheduler, blocking executor). Per-call `AuthenticationCallContext` instances are cheap to allocate. + +### `AuthenticationCallContext` / `HttpAuthCallContext` + +A per-call context carries any transport-specific routing details, and exposes a **per-exchange state slot** for the implementation to retain conversation state across calls (challenge-response rounds, multi-stage HTTP auth, etc.): + +```java +public interface AuthenticationCallContext { + String brokerHost(); + + /** + * Retrieve an implementation-controlled state object previously stored + * with {@link #setStateObject}. The slot is keyed by class so the pieces + * of one plugin — which, under the capability-factory model, may be + * separate internal classes participating in the same exchange (e.g. the + * initial-data provider and the challenge handler on one binary connect) — + * can each keep state without collision; impls typically store one object + * of their own type (e.g., their SASL conversation state). + * + *

The slot's lifetime equals the authentication exchange: for binary + * protocol, the lifetime of one {@code ClientCnx} setup (including all + * {@code CommandAuthChallenge}/{@code CommandAuthResponse} rounds); + * for HTTP, one request's retry sequence. Concurrent authentications + * to different brokers get their own context with their own slots, + * so multiple in-flight handshakes don't collide. + */ + Optional getStateObject(Class clazz); + + /** Store a state object keyed by its (or any) class; a {@code null} value + * removes the entry. Implementations should key with a private class of + * their own (e.g. their conversation record) to avoid collisions. Rounds + * of one exchange are serialized by the framework, so slot access within + * an exchange needs no synchronization. */ + void setStateObject(Class clazz, T value); +} +``` + +```java +public interface HttpAuthCallContext { + URI requestUri(); + + /** + * For SASL-style HTTP challenge/response: the challenge headers from the server's + * prior {@code 401} response (e.g. {@code SASL-Token} / {@code State} / + * {@code SASL-Server-ID}). Empty on the first request, before any challenge. + */ + Optional serverChallengeHeaders(); + + // Per-exchange state slot — same contract as on AuthenticationCallContext + Optional getStateObject(Class clazz); + void setStateObject(Class clazz, T value); +} +``` + +(The state slot is defined on each of the two contexts rather than hoisted into a shared `StatefulCallContext` supertype — nothing is generic over both contexts, so the extra public type would buy nothing.) + +Most implementations don't need to inspect any of the routing fields — they produce the same credential for every call. Implementations of `BinaryAuthChallengeHandler` typically use the state slot to track their conversation across rounds (the server's challenge bytes arrive as the `AuthChallenge` parameter; cross-call state lives in `getStateObject(...)`). For HTTP, `HttpAuthChallengeHandler` reads the server's challenge from `HttpAuthCallContext.serverChallengeHeaders()` and keeps its own conversation state in the same state slot. + +### The `PulsarHttpClient` SPI + +The pluggable HTTP client lives in `org.apache.pulsar.http`. The **framework manages HTTP client lifecycle**, including the Netty event loop, timer, DNS cache, and `PulsarTlsFactory` integration. Plugins describe what kind of HTTP client they need; the framework constructs, pools, and closes instances: + +```java +package org.apache.pulsar.http; + +public interface PulsarHttpClient extends AutoCloseable { + CompletableFuture execute(HttpRequest request); + + /** Release this instance. Idempotent. Instances are framework-owned: a plugin + * MAY close an instance it no longer needs, and the framework closes every + * remaining instance when the owning PulsarClient closes — so calling this + * is optional for plugins. */ + @Override void close(); +} + +/** + * Framework-owned factory. A plugin that needs framework-managed HTTP + * MUST obtain a client via {@link AuthenticationInitContext#httpClientFactory()} + * rather than constructing its own. Multiple instances per PulsarClient + * are supported — e.g., OAuth2's mTLS exchange to the IdP uses a different + * TlsPurpose-driven TLS configuration than a third-party plugin's own HTTP + * endpoint, but they share the underlying event loop / timer / DNS resources. + */ +public interface PulsarHttpClientFactory { + PulsarHttpClient newHttpClient(PulsarHttpClientConfig config); +} +``` + +`HttpRequest` is an immutable value type with method, URI, headers, optional body (`Bytes` — raw content plus a `Content-Type`; the body slot is a single-member sealed `Body` so a future streaming or structured variant can be added additively). Headers are set per request (there is no separate config-level default-headers path). `HttpResponse` exposes status, headers, and the body bytes (buffered-only in v1; 16 MiB default cap configurable via `PulsarHttpClientConfig.maxResponseBodyBytes()`). `PulsarHttpClientConfig` carries per-instance concerns: a `TlsPurpose` (the framework consults the configured `PulsarTlsFactory` for the TLS material bound to that purpose), the request timeout, and the Pulsar user-agent string; tracing and metrics ride on the OpenTelemetry handle from `AuthenticationInitContext`, not on a bespoke hooks type. It carries no per-instance proxy setting — SOCKS5 proxying is wired once from the client configuration. The configuration deliberately does NOT carry the cert/key/trust material directly — that lives in the `PulsarTlsFactory` and is looked up by purpose. For a plugin that needs its own trust domain (the OAuth2 IdP being the canonical case), the plugin selects a distinct purpose — `TlsPurpose.CLIENT_OAUTH2`, or a minted variant such as `TlsPurpose.client("oauth2.myPlugin")` — and the operator configures a `TlsPolicy` for that purpose; the plugin never handles raw material. This purpose-key indirection is how a plugin gets its own trust domain — expressed as an ordinary `TlsPurpose` rather than a separate addressing type. + +**Header representation — single-valued by contract.** `HttpResponse` and `HttpAuthHeaders` model **one value per header name**: two source entries that canonicalise to the same name collapse (last-wins), and multi-valued headers are not representable. This is sufficient for the SASL / single-pass mechanisms that ship — one value per `Authorization` / `SASL-Token` / `State` / `SASL-Server-ID` name — and keeps the value types small. It is a deliberate contract, not a limitation to be patched later by widening these types: a future multi-round or standard `WWW-Authenticate` / digest capability that genuinely needs multi-valued headers will bring its **own richer header representation**, added as a sibling capability the way the second HTTP challenge interface is (see [Resolved decision 1](#1-http-multi-round-driver-design-and-the-sasl-vs-standard-interface-split)), rather than reshaping `HttpResponse` / `HttpAuthHeaders`. So the single-valued types never take a forced backward-incompatible change, and no multi-valued machinery ships now — the minimal durable resolution. + +**Header case-handling — a deliberate asymmetry.** Request and response header names are canonicalised differently, on purpose. `HttpRequest` names are stored **verbatim, case-preserving** — outbound wire fidelity: the client sends exactly the name the author (or an RFC) specified, so an integration is never surprised by a silently reshaped header, and there is no interop risk with a peer that treats a header name case-sensitively. `HttpResponse` and `HttpAuthHeaders` names are **canonicalised to lower case**, but purely as a **client-side lookup convenience** backing case-insensitive `header(...)` / `get(...)` — never a transformation applied on the wire. The direction dictates the rule: inbound the client is the reader, so a normalised key helps it look values up; outbound the client is the writer, so verbatim fidelity wins. + +**Normative — no credential or payload logging.** A `PulsarHttpClient` implementation, and any framework logging around it, MUST NOT log request or response **bodies**, **header values**, or **authentication bytes** (the `Authorization` header, SASL `SASL-Token` / `State` values, cookies, or a plugin's custom credential headers) at any log level. Only the request method, the request URI (with any userinfo or secret query parameters elided), and the response status are safe to record. + +**Why multiple HTTP clients?** When the binary connection uses mTLS, the OAuth2 client may need a DIFFERENT mTLS configuration to talk to the identity provider — the IdP and the broker are different trust domains. Sharing one HTTP client instance across both is wrong. But the *resources* — event loop threads, DNS resolver cache, refresh scheduler — should be shared. The framework owns the shared resources and hands out distinct `PulsarHttpClient` instances per `PulsarHttpClientConfig`. + +**Design decision — the backend is framework-owned, not pluggable.** Every `PulsarHttpClient` is built by the framework on AsyncHttpClient (in `pulsar-client-v5`), with TLS wired through the purpose-driven `PulsarTlsFactory` lookup. A pluggable backend was considered — a `ServiceLoader`-discovered `PulsarHttpClientProvider` SPI with name/priority resolution, a `builder.httpClientProvider(name)` selector, an opt-in JDK-`HttpClient` provider module, and a caller-supplied `builder.httpClient(instance)` escape hatch — and deliberately rejected. What plugins need (Motivation #3) is to *obtain* a framework-managed client, not to choose its transport; `PulsarHttpClientConfig` already covers the things operators actually asked for in [#24795](https://github.com/apache/pulsar/issues/24795) (timeouts, and observability via the OpenTelemetry handle; SOCKS5 proxying is wired from the client configuration). The `PulsarHttpClient` interface boundary by itself keeps the framework free to change its internal backend later, whereas a public discovery contract would have to be documented, tested, and maintained forever — with third-party backends of varying fidelity (a JDK backend, for instance, lacks SOCKS5 and custom DNS resolution and would need a client rebuild on TLS rotation). Backend pluggability can be reintroduced additively if a concrete need ever emerges. + +**TLS rotation behind `PulsarHttpClient`.** AsyncHttpClient fixes its TLS configuration at instance build, so the framework does not hand it a static context: it installs its own `SslEngineFactory` and subscribes to the Netty `SslContext` for the instance's `TlsPurpose` (`createInstance(purpose, SslContext.class, onLoadOrReload)`); each new engine — i.e. each new connection — is created from the most recently delivered context, so rotated material takes effect for new connections. Established pooled connections do not renegotiate: on reload delivery the framework evicts **idle** pooled connections, and a pooled-connection TTL that the **framework configures to a bounded value** (AsyncHttpClient's own `connectionTtl` default is `-1` — unbounded — so the bound is the framework's doing, not a library default) caps how long an active connection can keep using pre-rotation material — rotation is therefore effective within the TTL bound, not merely "eventually". Plugins see none of this — rotation is invisible behind `PulsarHttpClient`. + +Consumers of `PulsarHttpClient`: + +- The built-in OAuth2 plugin, for token-endpoint and well-known-metadata fetches. The OAuth2 credential flow (`Flow`/`FlowBase`) that makes these calls stays on the v4 `AuthenticationOAuth2` class rather than being reimplemented v5-native — see the layering note under [the v4 internal migration](#class-name-compatibility-and-the-v4-internal-migration). +- Any third-party plugin that needs HTTP. + +The HTTP topic-lookup client (`HttpClient`) and the admin client's `AsyncHttpConnector` are deliberately **not** `PulsarHttpClient` consumers: each still constructs its own `DefaultAsyncHttpClient`. They do, however, adopt the same two framework mechanisms this PIP introduces — the rotating `SslEngineFactory` backed by a `PulsarTlsFactory` subscription to the `CLIENT_DEFAULT` `SslContext` (so rotated TLS material takes effect for new connections, with idle pooled connections evicted on reload), and the shared `HttpAuthenticationDriver` for the SASL `401`→resubmit→`200` loop — so they gain the TLS-rotation and multi-round-auth behaviour without moving onto the `PulsarHttpClient` SPI itself. + +Athenz is deliberately **not** in this list: its ZTS role-token exchange runs on the Athenz SDK's own HTTP transport (the SDK owns that connection), so `AuthenticationAthenz` does not obtain a framework `PulsarHttpClient` — only OAuth2 among the built-ins does. + +**Standalone plugin usage.** The v4 `Authentication` contract allows a plugin to be created and started outside any `PulsarClient`/`PulsarAdmin` (`AuthenticationFactory.create(...).start()` — the proxy's own broker-client authentication and CLI tools do exactly this). Such a plugin has no framework services bound, so OAuth2's HTTP needs are served by a self-contained standalone fallback factory: a framework-owned HTTP client on its own minimal resources, honoring the plugin's IdP TLS parameters (with rotation) when present and platform-default trust otherwise, closed with the plugin. Bound usage always prefers the client's shared resources; the standalone form exists solely to preserve the v4 contract. + +### Redesigned PIP-337 SSL provider: `PulsarTlsFactory` + +A small set of new types replaces the existing `PulsarSslFactory` / `PulsarSslConfiguration` surface as the v5 client's (and Pulsar 5.0's) TLS integration point. The SPI is a typed **instance factory**: consumers request fully configured TLS objects per purpose; how the factory sources material and builds them is factory-internal. + +- **`PulsarTlsFactory`** (interface, in `org.apache.pulsar.tls`) — the v5 TLS SPI. The framework requests an instance of a well-known class for a `TlsPurpose` (described next); the factory returns a configured instance when it supports the combination, or `Optional.empty()` when it does not: + + ```java + public interface PulsarTlsFactory extends AutoCloseable { + CompletableFuture initialize(TlsFactoryInitContext context); + + /** + * A fully configured instance of {@code instanceClass} for the purpose, or + * {@code Optional.empty()} when this factory does not support the + * (purpose, class) combination. Failure to build a SUPPORTED combination + * completes the future exceptionally instead. + */ + CompletableFuture>> createInstance( + TlsPurpose purpose, Class instanceClass); + + /** + * One-shot variant carrying the destination endpoint as a per-request HINT. + * Client-side consumers pass the target host/port when they know it (one + * connection, one instance); a factory that serves per-destination material + * (multi-cluster deployments, per-target workload identities) may key on it. + * The default implementation ignores the endpoint and delegates to + * {@link #createInstance(TlsPurpose, Class)} — most factories, including the + * default file-based one, never look at it. The endpoint does NOT replace + * hostname verification or SNI: those are applied at engine creation from + * the same peer address (see the usage notes below). + */ + default CompletableFuture>> createInstance( + TlsPurpose purpose, TlsEndpoint endpoint, Class instanceClass) { + return createInstance(purpose, instanceClass); + } + + /** + * Like the one-shot form, but additionally subscribes to reloads: + * {@code onLoadOrReload} receives the instance on initial load and a REBUILT + * instance whenever the underlying material changes. The returned future + * completes after the first delivery. Subscriptions are purpose-scoped and + * carry no endpoint — they serve server-side listeners, which have no + * destination. + */ + CompletableFuture>> createInstance( + TlsPurpose purpose, Class instanceClass, Consumer onLoadOrReload); + + @Override + void close(); + } + + /** Destination of an outbound connection, passed to the factory as a hint. */ + public record TlsEndpoint(String host, int port) {} + + /** + * Handle for a built instance — returned by every {@code createInstance} form. + * For a one-shot request, {@link #get()} returns the built instance; for a + * subscribing request, it returns the value most recently delivered to + * {@code onLoadOrReload} (the initial load, or the latest rebuild), so a + * consumer can read the live instance on demand without caching callback + * deliveries. A single handle type is used rather than separate + * ({@code TlsInstance} / {@code TlsSubscription}) types, which would be + * byte-identical — differing only in prose. + */ + public interface TlsHandle { + T get(); + /** Unregister the reload callback (if any) and release the factory-side + * resources backing this handle (background refresh, caches). */ + void dispose(); + } + ``` + + How a factory is *selected* is component-specific configuration (see [Configuration](#configuration)); at initialization every factory receives a **`TlsFactoryInitContext`** carrying its parameters and the framework's runtime services (the default `FileBasedTlsFactory` additionally receives its purpose→policy map through its constructor, composed by the owning component — see the build-time composition note below): + + ```java + public interface TlsFactoryInitContext { + /** Factory-specific parameters from the owning component's configuration + * (the tlsFactoryConfig key on the server side; builder-supplied on the client). */ + Map params(); + + /** Scheduler for file-watch polling and rotation work. Framework-owned. */ + ScheduledExecutorService scheduler(); + + /** Executor for potentially-blocking material loading. Never a consumer event loop. */ + Executor blockingExecutor(); + + Clock clock(); + OpenTelemetry openTelemetry(); + } + ``` + + The context is constructed by whichever component owns the factory — the v5 client builder on the client side; the broker / proxy / websocket / functions-worker service on the server side — and `initialize(...)` completes before the first `createInstance` call. A failure during `initialize` is fatal to the owning component's startup. + + The **well-known instance classes** and their support contract: + + | Requested class | Consumer | Contract | + |---|---|---| + | `io.netty.handler.ssl.SslContext` | binary protocol; AsyncHttpClient-based HTTP | optional — on `empty()` the framework wraps the JDK `SSLContext` fallback with Netty's `JdkSslContext` adapter (the default file-based factory supports it natively, keeping OpenSSL-based contexts available) | + | `org.eclipse.jetty.util.ssl.SslContextFactory.Server` | broker / proxy / function-worker web server | optional — the framework asks the factory first; on `empty()` it synthesizes one from an `SSLContext` subscription and Jetty's `reload(...)` (see the Jetty section below) | + | `org.eclipse.jetty.util.ssl.SslContextFactory.Client` | proxy `AdminProxyHandler` admin `HttpClient` (proxy→broker) | optional — the framework asks the factory first; on `empty()` it synthesizes one from an `SSLContext` subscription and Jetty's `reload(...)`, mirroring the Server row | + | `javax.net.ssl.SSLContext` | fallback source for the classes above; non-Netty consumers | **required only as a fallback** — a factory must support it for a purpose *unless* it natively supplies every richer (Netty / Jetty) class that purpose consumes; the framework builds the Netty `SslContext` / Jetty `SslContextFactory.Server` from it whenever the factory returns `empty()` for them | + | `javax.net.ssl.SSLParameters` | optional companion to the `SSLContext` fallback | **optional** — consulted by the framework *only* when it synthesizes from the `SSLContext` fallback; carries the factory's engine-level baseline policy (enabled protocols and cipher suites, client-auth mode, endpoint identification, algorithm constraints) that a bare `SSLContext` cannot express — the JDK API has no setter for a context's default parameters. `empty()` means the consumer's own configuration applies, as before | + + `javax.net.ssl.SSLContext` is deliberately listed **last**: it is not an end in itself but the **universal fallback** the framework synthesizes the richer objects from. For each purpose the framework first asks the factory for the object a consumer actually needs — the Netty `SslContext`, or Jetty's `SslContextFactory.Server` / `SslContextFactory.Client`. Only when the factory returns `Optional.empty()` for that richer class does the framework fall back to requesting `SSLContext` for the same purpose and build the richer object itself (wrapping it with `JdkSslContext` for Netty, or configuring and driving `reload(...)` for Jetty). The consequence: a factory that natively supplies every richer object a purpose consumes is never asked for `SSLContext` for that purpose and need not implement it at all; a factory that supplies neither Netty nor Jetty objects must implement `SSLContext` so the framework can synthesize both from it. + + On that synthesis path the framework additionally asks the factory for `createInstance(purpose, SSLParameters.class)` — the optional engine-policy companion. Merge order is deterministic: (1) the factory's `SSLParameters` (non-null members only) form the engine baseline; (2) `endpointIdentificationAlgorithm` — the factory's value wins when set, otherwise the consumer's hostname-verification configuration applies `"HTTPS"` on client purposes; when neither sets it the overlay leaves it unset, and because the synthesized engine is JDK-backed (a `JdkSslContext` delegate that carries no endpoint-identification algorithm) it defaults to no verification, matching the consumer that disabled it — the synthesis path never has to counter an engine default. (The one place the algorithm is *explicitly* cleared is a different path: the native default-factory client-context build (`TlsContexts.buildNettyClientContext`) sets `endpointIdentificationAlgorithm(null)` to counter Netty 4.2's client-engine `"HTTPS"` default, which would otherwise re-enable verification the policy disabled.) (3) SNI server names are always set per connection from the target endpoint, overriding any factory baseline (a factory should not pin SNI); (4) on server purposes, a factory-supplied `SSLParameters` is authoritative for `needClientAuth`/`wantClientAuth`, otherwise the consumer's client-auth flag maps as usual. On subscriptions, the parameters are re-requested with each `SSLContext` delivery, so engine policy may rotate with material. The synthesized *Jetty* factories consult the protocols, cipher suites, and client-auth members only (Jetty's `SslContextFactory` exposes no setters for the finer members; endpoint identification is a client-engine concern the Jetty server path never needs) — the full member set applies on the Netty synthesis path. Rationale: without this companion, an `SSLContext`-only factory could not restrict protocols or ciphers, or state its own verification and client-auth policy — those are engine-level parameters, and `SSLContext` offers `getDefaultSSLParameters()` but no setter. With it, the tier-3 story is complete at **zero non-JDK dependencies**. + + Jetty's `SslContextFactory.Client` is a well-known class too, mirroring the Server variant. Its one in-tree consumer — the proxy's `AdminProxyHandler`, whose Jetty `HttpClient` forwards admin requests to brokers over TLS — obtains it through the same ask-then-synthesize path: the framework first asks the configured factory for a native `SslContextFactory.Client` for the `BROKER_CLIENT` purpose, and a custom factory MAY supply one — taking the same native-supply obligations as the Server variant (unstarted hand-over, same instance per purpose, internal `reload(...)` on rotation) and additionally owning its own endpoint identification — to customize proxy→broker admin TLS. When the factory returns `empty()` — which the default `FileBasedTlsFactory` does — the framework synthesizes a plain (non-subclassed) `SslContextFactory.Client` configured via `setSslContext(...)` from the same `SSLContext` fallback and drives its `reload(...)` on rotation (so rotated broker-client material reaches the long-lived admin `HttpClient`'s new connections). On that synthesized path the framework also applies the consumer's client-side hostname verification (disabling endpoint identification when the proxy has it off); a natively-supplied client owns that itself and is left untouched. + + The contract is enforced **fail-fast**: at startup the framework probes every (purpose, class) pair it needs; a purpose for which the framework can obtain *neither* the richer class *nor* its `SSLContext` fallback is a boot-time configuration error rather than a failure at first connection. Contract violations discovered later (e.g. a subscription that never delivers) surface as runtime exceptions. Probing mechanics: the probe uses the one-shot, endpoint-less `createInstance` form, and the returned handle is **retained as the initial cached instance** rather than disposed — the probe is the first load, not a throwaway. Statically known purposes are probed when the owning component starts (`PulsarClient` build on the client side, service start on the server side); plugin-minted purposes are probed when the plugin first requests them during `initializeAsync`. TLS material therefore loads **eagerly at build time** — a deliberate behavior change from the v4 client's lazy load at first connection, accepted because surfacing misconfiguration at build is the point of the fail-fast contract. + + Usage patterns per consumer: server-side consumers use the **subscribing** overload — the broker/proxy binary listeners swap the `SslContext` they use to build `SslHandler`s for new connections; Jetty instances are requested with the **one-shot** overload since a factory-supplied `SslContextFactory.Server` handles reloading internally (obligations below). Client-side consumers use the one-shot overload per new connection, passing the destination as a `TlsEndpoint` hint when they know it; the factory returns its cached instance until material changes (per-destination factories may cache per endpoint), and `dispose()` releases the consumer's interest (disposal on connection close also balances any Netty reference count). The one-shot form presumes an *asynchronous* connection path — the instance future composes into the connect chain before the transport connects, never blocking an event loop; Pulsar's binary connect already has this shape. A client-side consumer whose integration point is synchronous (an HTTP library's engine factory that must produce an `SSLEngine` on demand) instead holds the **subscribing** overload's current instance; the proxy's broker-facing data path, whose channel initializer is synchronous, does the same. Both are first-class client patterns. Per-instance settings — protocols, ciphers, server client-auth mode, insecure trust-all modes, and **client-side hostname verification** — are baked into the built objects by the factory from the configuration it was initialized with. Hostname verification cannot be left to the consumer to toggle per connection: Netty's OpenSSL-based `SslContext` fixes the endpoint-identification algorithm at build time (it is a `final` field on `ReferenceCountedOpenSslContext`, settable only via `SslContextBuilder.endpointIdentificationAlgorithm(...)`), and a per-`SSLEngine` `setSSLParameters` override is asymmetric on that backend — it can force verification on but does not relax it off. The factory therefore sets the algorithm when it builds the client context for a purpose, driven by the client/builder configuration. (The JDK backend would permit a clean per-`SSLEngine` override either way, but the SPI treats hostname verification as a per-context setting for backend independence.) Reload callbacks are serial per subscription, never concurrent, never invoked on a consumer event loop, and the first delivery happens-before the returned future completes. + + **Custom factories and the framework's TLS security defaults (normative).** The framework applies its TLS security defaults — the enabled-protocol floor, client-side hostname verification (`"HTTPS"` endpoint identification), trust-all / insecure handling, and the server client-auth mode — only on the objects it **synthesizes** from a factory's `SSLContext` (plus the optional `SSLParameters`) fallback; the default `FileBasedTlsFactory` bakes those same defaults natively into the Netty / JDK contexts it builds. A custom factory that **natively supplies** a richer object (Netty `SslContext`, or Jetty `SslContextFactory.Server` / `SslContextFactory.Client`) has that object used **verbatim** — the framework overlays no policy on it, and such a factory has **no framework `TlsPolicy` behind it**. A custom factory therefore MUST source all engine policy from its own configuration (`TlsFactoryInitContext.params()`, and the `SSLParameters` it returns on the synthesis path). In particular it **MUST** enable client-side hostname verification unless the operator has explicitly opted out, **MUST** pin an acceptable protocol floor, and **MUST NOT** trust all peer certificates unless explicitly configured. The phrase "driven by the client/builder configuration" above describes the default factory and the framework-synthesis path only; for a natively-supplying custom factory the security posture is the factory's own responsibility — one that ships with endpoint identification off by default silently exposes its consumers to man-in-the-middle attacks. This obligation is restated in [Security Considerations](#security-considerations). + + **Reload failure semantics.** A failed rebuild on rotation (e.g. a half-written or invalid rotated file — the canonical incident) must not tear down a working subscription: the factory keeps serving the **last-good** instance, logs the failure at WARN, records it in the reload-failure metric (see [Metrics](#metrics)), and retries on the next observed material change. A consumer callback that throws is caught and logged; the subscription stays live and later deliveries proceed. `initialize()` failing at boot fails the owning component's startup, per the fail-fast contract above. A custom factory MUST emit the `pulsar.tls.reload` and `pulsar.tls.last_reload_success` instruments (via the `OpenTelemetry` handle from `TlsFactoryInitContext`) on every load/reload attempt — success and failure alike — so operators keep rotation-health visibility regardless of which factory is installed (see [Metrics](#metrics) and [Monitoring](#monitoring)). + + **Ownership and disposal.** The component that creates a factory owns and closes it: the default factory, and any custom factory instantiated from configuration, are closed by the framework when the owning client or service closes. A factory *instance* supplied programmatically to the v5 builder is **adopted** — the client closes it on `PulsarClient.close()`, matching the v4 precedent for builder-supplied `Authentication` instances. Handles follow their consumer: server-side subscriptions are disposed when the listener or web service stops; a client connection disposes its one-shot handle when the connection closes. A leaked (undisposed) handle costs at most a factory-side cache/refresh registration — the default factory shares one `TlsMaterialSource` per purpose, so a leak never duplicates file watchers and is bounded. + + A factory that natively supplies a Jetty factory — `SslContextFactory.Server` **or** `SslContextFactory.Client` — takes on three obligations: the instance must be handed over **unstarted** (Jetty starts it with the connector / `HttpClient` lifecycle), repeated requests for the same purpose must return the **same instance** (it is a stateful lifecycle bean, not a value), and the factory must drive `reload(...)` on it internally when material rotates. For the client variant it additionally owns endpoint identification / hostname verification (the framework applies that only on its synthesized fallback). The framework-synthesized fallback carries these obligations instead, which is why most factories should simply return `empty()` for both Jetty classes. The native-supply path is retained (rather than making Jetty integration framework-only) because it is the only way a factory can control Jetty-level engine settings — protocol/cipher selection, SNI keystores — that a bare `SSLContext` cannot carry. + + `Optional.empty()` from `createInstance` means exactly one thing: the factory does not support the requested (purpose, class) combination. It is **not** a purpose-resolution signal — how a factory maps a purpose to configured material is factory-internal. The default `FileBasedTlsFactory` resolves the requested purpose directly against its configured `TlsPurpose → TlsPolicy` map (see `TlsPurpose`); when nothing is configured, a client-role purpose resolves terminally to the system default (OS trust store, no client certificate) and a server-role purpose fails the request. Two guardrails keep custom factories from misusing the signal: a factory that has resolved a purpose to material (or to the system default) and *supports the requested class* must **never** return `empty()` for it — a resolved-but-unbuildable request completes exceptionally, so a real configuration error can't be masked by the framework quietly falling back to `SSLContext` synthesis; and the framework's default factory follows the terminal resolution rule specified above (a direct map lookup, then the role's terminal rule — client → system default, server → error), so its resolution is objectively testable. A custom factory owns its own purpose→material resolution, but MUST document any divergence from that default so operators can reason about it. Example: `createInstance(CLIENT_OAUTH2, SslContext.class)` on the default factory with no OAuth2 policy configured returns a system-default-trust context (client role, no configured policy → system default) — not `empty()`, and not an error. Like every future-returning method in this PIP, `createInstance` reports all failures through the returned future and never throws synchronously. All `createInstance` overloads MUST be **thread-safe**: the framework calls them concurrently from independent connection paths, and a subscription's reload fan-out may run while other `createInstance` calls are in flight. + + **Instance ownership.** Built TLS objects are **factory-owned snapshots**: consumers never close, release, or mutate them — in particular they must not `release()` a Netty reference-counted OpenSSL context. `TlsHandle.dispose()` only signals that this consumer is done; the factory releases a superseded or fully-disposed instance's native resources itself, once the last handle referencing it is gone. A factory that builds reference-counted Netty contexts therefore manages their refcounts internally; consumers treat every returned object as an immutable borrow. Calling `TlsHandle.get()` **after** `dispose()` is a programming error: the handle no longer references a live instance (an implementation may return the last value or throw `IllegalStateException`, and MUST NOT return a released native context) — a consumer that has disposed its handle must re-acquire through `createInstance`. `dispose()` is idempotent. + + **Shading and plugin packaging.** The well-known classes include Netty and Jetty types, and `pulsar-client-shaded` / `pulsar-client-admin-shaded` relocate Netty (`io.netty` → `org.apache.pulsar.shade.io.netty`). A custom `PulsarTlsFactory` intended for the shaded artifacts must therefore be **relocated the same way** — its bytecode references rewritten to the shaded class names (e.g. with the Gradle Shadow or Maven Shade plugin), *without bundling the relocated classes themselves*. The plain, unrelocated plugin artifact serves `pulsar-client-original` / `pulsar-client-admin-original` and all server-side components, which are not shaded. Plugins targeting both client flavors should publish both artifacts (e.g. a `-shaded` classifier). PIP-337 has exactly the same constraint through `getInternalNettySslContext()`, but leaves it undocumented and unresolved; this PIP makes the requirement explicit. + + A support class (**`TlsContexts`**) lets factories compose instead of reimplement: build a JDK `SSLContext` or Netty `SslContext` from PEM/keystore inputs or from a `KeyManagerFactory`/`TrustManagerFactory`, and wrap JDK→Netty via `JdkSslContext`. It lives in **`pulsar-common`** (package `org.apache.pulsar.common.tls.impl`), *not* in the dependency-light `pulsar-tls-factory-api` SPI module — a custom factory that wants it therefore depends on `pulsar-common`. Its exact method surface is an implementation detail to be settled in code review — it is a convenience helper, not part of this PIP's SPI contract. + +- **`TlsPurpose`** (a value type, in `org.apache.pulsar.tls`) — identifies *why* TLS is requested and in what role. It is a **simple named key**, not a type hierarchy: a role (client or server) and an open name — nothing more. Well-known purposes are exposed as constants; a factory serves distinct material per purpose. Instances are used as map keys, and identity is plainly `(role, name)`: `equals`/`hashCode` are defined over exactly those two fields. This deliberately avoids a sealed `Client`/`Server` split, a `UsageIdentifier(Class, String)` addressing dimension, a structured `host()` (SNI cert-*selection* is unused — Pulsar sets `setSniRequired(false)`), any fallback *chain* or single-level fallback field, and a second `qualifier()` addressing dimension — a flat role-plus-name is the whole key, and resolution is a single terminal step (below), not a walk. + + ```java + package org.apache.pulsar.tls; + + public final class TlsPurpose { + public enum Role { CLIENT, SERVER } + + public Role role(); + /** Well-known or plugin-minted name, e.g. "default", "oauth2", "broker-client", "broker". */ + public String name(); + + /** Mint open-named purposes, e.g. TlsPurpose.client("oauth2.myPlugin"). When nothing is + * configured for a purpose, resolution is terminal: a CLIENT purpose resolves to the system + * default (OS trust store, no client certificate); a SERVER purpose is a configuration error. */ + public static TlsPurpose client(String name); + public static TlsPurpose server(String name); + + // equals/hashCode are defined over (role, name). + + // Well-known CLIENT purposes + public static final TlsPurpose CLIENT_DEFAULT; // Pulsar-cluster traffic: binary, HTTP lookup, admin + public static final TlsPurpose CLIENT_OAUTH2; // OAuth2 / IdP calls — must NOT reuse cluster material + public static final TlsPurpose BROKER_CLIENT; // a server component's own outbound Pulsar-client traffic: + // replication, proxy→broker, websocket→broker, worker→broker + // Well-known SERVER purposes + public static final TlsPurpose BROKER; + public static final TlsPurpose PROXY; + public static final TlsPurpose WEB; + } + ``` + + Users configuring the client only ever reference client purposes, and in practice only two buckets matter: `CLIENT_DEFAULT` (all Pulsar-cluster traffic — binary protocol, HTTP topic lookup, and the admin client all resolve here) and `CLIENT_OAUTH2` for OAuth2/IdP calls, which — when unconfigured — resolves terminally to the **system default** (OS trust store, no client certificate) rather than to the cluster material, since the IdP is a different trust domain. A plugin that needs its own trust domain **mints** an open-named purpose — `TlsPurpose.client("oauth2.myPlugin")` — giving it a dedicated config key; the operator configures a `TlsPolicy` for that key, and until they do the purpose resolves to the system default. The framework does **not** chain a minted purpose back to `CLIENT_DEFAULT`: a finer client purpose that should share cluster material is seeded with that material at composition time, not resolved through a fallback link. + + On the server side, `BROKER` / `PROXY` / `WEB` cover the binary listeners and the web server, and `BROKER_CLIENT` covers a server component's *own outbound* Pulsar-client connections — geo-replication, proxy→broker, websocket→broker, and functions-worker→broker traffic, which is configured through the dedicated `brokerClient*` keys (including `brokerClientTlsFactoryClassName`), a distinct trust domain from both the server listeners and any application client. Unconfigured server purposes resolve to the component's configured default server material. This reconciles with `TlsPurpose`'s terminal rule ("SERVER role → configuration error") through **composition-time pre-population**: the owning component seeds every server purpose's slot with its configured default server material when it builds the factory, so a server-side `createInstance` never actually reaches the terminal error unless even that component default is unset — which is then the genuine configuration error the rule names. `BROKER_CLIENT` resolution additionally **folds the component's broker-client `Authentication` TLS material** when the plugin supplies any (`brokerClientAuthenticationPlugin=AuthenticationTls`): the plugin's in-memory cert/key override the configured `brokerClient*` file paths — the server-side mirror of the client TLS override hook, preserving the PIP-337 behavior that `PulsarSslConfiguration.authData` carried. Without the fold, a proxy would present the wrong identity to the broker and break forwarded-principal authorization. Per-advertised-listener material is deferred to the separate listener-aware broker PIP; when it lands, dotted minted names (e.g. `server("broker.internal")`) seeded with their material at composition time are the natural additive extension — no new addressing dimension is required. + + **Design decision — destination endpoint as a hint, not part of the purpose.** Three options were considered for letting TLS material vary by destination: (a) no destination in the SPI at all (premise: material lookup never needs it); (b) *(chosen)* keep `TlsPurpose` a static key and pass the destination as an optional per-request `TlsEndpoint` hint on the one-shot `createInstance`, which factories may ignore (the default file-based factory does); (c) fold the destination into the purpose value itself. Rationale: (a) forecloses per-destination material — multi-cluster deployments and workload-identity systems that mint per-target credentials have no way to see the target — and retrofitting the parameter later would ripple through every factory; (c) would make the purpose a dynamic value, destroying its role as a stable configuration key (purpose names map to config entries — see the config-file note below). Option (b) costs one default method that simple factories never notice, and keeps the two concerns cleanly separated: the *purpose* says which trust domain, the *endpoint* says which peer within it. Hostname verification and SNI are unaffected either way — they are applied at engine creation from the same peer address by whichever component builds the `SSLEngine`, never by the factory. **Scope note:** per-endpoint material selection is a feature of the *one-shot, binary-transport* path only. Subscriptions are purpose-scoped (no endpoint overload), and `PulsarHttpClient` instances select TLS material **by purpose only** — an HTTP client instance talks to one logical service, so its trust domain is fixed at instance creation; if per-destination HTTP material is ever needed, the framework's `SslEngineFactory` already sees the peer address per engine and an endpoint-aware lookup can be added there without SPI changes. + +**Jetty web server integration (replacing the `getSslContext()` override).** Pulsar's current `JettySslContextFactory` subclasses Jetty's `SslContextFactory.Server` and overrides `getSslContext()` to return the rotating context on every call. That pattern is unsound: Jetty selects protocols and cipher suites at `load()` time *from the context it loads*, so with the override the selection runs against a throwaway context, handshakes use a different one, and rotation never re-runs selection — it works today only by accident (see [Appendix A: Jetty getSslContext override analysis](#appendix-a-jetty-getsslcontext-override-analysis)). v5 abandons the override entirely. + +The correct, documented mechanism — the same one Jetty's own `KeyStoreScanner` hot-reload module uses — is `SslContextFactory.reload(Consumer)`: the consumer mutates the configuration under the factory's lock, then Jetty unloads and re-runs `load()`, atomically swapping the internal state and re-selecting protocols/ciphers from the new context. Under the v5 SPI there are two paths. A factory that **natively supplies** `SslContextFactory.Server` owns this itself (the obligations listed above: unstarted hand-over, same instance per purpose, internal `reload(...)` on rotation). When the factory returns `Optional.empty()` for the Jetty class — the recommended default — the **framework synthesizes** the integration: it subscribes to `javax.net.ssl.SSLContext` for the purpose via `createInstance(purpose, SSLContext.class, onLoadOrReload)`, configures a plain (non-subclassed) `SslContextFactory.Server` with `setSslContext(initialContext)` before start, and on each callback delivery runs: + +```java +sslContextFactory.reload(f -> f.setSslContext(newContext)); +``` + +Existing connections keep their negotiated sessions; new connections use the new context. One caveat: with a directly-set `SSLContext`, Jetty cannot wrap `KeyManager`s for SNI-based certificate selection — Pulsar does not use that feature (it sets `setSniRequired(false)` today), and a factory that needs it can supply a keystore-backed `SslContextFactory.Server` natively. + +**Default file-based factory.** `FileBasedTlsFactory` implements `PulsarTlsFactory`, natively supplying the Netty `SslContext` (on the configured engine — JDK, or OpenSSL-based where available) and the JDK `SSLContext`; it returns `empty()` for the Jetty class, which the framework synthesizes — the recommended default for every factory (see the Jetty section). It lives in **`pulsar-common`** (package `org.apache.pulsar.common.tls.impl`), not in the SPI module: `pulsar-common` already carries `netty-handler` and the `netty-tcnative` OpenSSL binding the native contexts need, while `pulsar-tls-factory-api` stays dependency-light; the distinct `.impl` package keeps it clear of the hostname-verification helpers that share `org.apache.pulsar.common.tls`. It is configured with a `TlsPolicy` per `TlsPurpose` (see below), and internally turns each policy into a **`TlsMaterialSource`** — a package-internal object that owns the runtime behavior: it polls its backing files/keystore for changes, caches the loaded material while the source is unchanged (value-equality on the loaded material suppresses spurious rebuilds when files are touched but unchanged), and notifies subscribers on change. The background poll is not the only notifier: because re-stating the source *consumes* its change signal (the mtime baseline is committed by the same refresh that observed the change), a one-shot acquisition that happens to observe a rotation first also fans that rotation out to every subscriber on the same purpose. Without that, whichever caller looked first would silently swallow the rotation and leave the subscribers wedged on the pre-rotation instance until the files changed again — e.g. an HTTP-lookup client whose engine factory subscribes to `CLIENT_DEFAULT` while another path acquires one-shot on it. The fan-out shares the single `pulsar.tls.reload` record for that (re)load rather than double-counting, and a partial delivery failure arms the same redelivery retry the poll uses. The factory itself owns the purpose→source registry, the purpose matching and terminal resolution, and the reload fan-out to subscribers; `TlsMaterialSource` owns only the load/watch/cache of one material set. This is a clean split — `TlsPolicy` is the declarative *what* (user-facing), `TlsMaterialSource` is the runtime *how* (internal). + +**Immutable after construction — policies compose at build time.** `FileBasedTlsFactory` is constructed with its complete purpose→`TlsPolicy` map and never mutates it afterwards: no public reconfiguration API, no concurrency contract between reconfiguration and in-flight `createInstance` calls. The one case that must *merge* TLS configuration from two places — the v4 compatibility path, where some TLS settings come from the client builder and some from a legacy `AuthenticationTls` / `AuthenticationKeyStoreTls` plugin — is resolved by the v5 client **builder before the factory exists**: the builder extracts the plugin's material paths (see the [TLS override hook](#legacyv4authenticationadapter-v4--v5)), merges them with any builder-configured policy (plugin-supplied key/cert combines with builder-supplied trust; on a per-field conflict the plugin value wins, preserving v4's "`AuthenticationTls` overrides TLS" semantics), and constructs the factory with the final map. Mutation methods (`setPolicy`/`addPolicy`) on the factory were considered for this and rejected: their overlap-merge semantics could not be stated crisply as a public contract, and nothing needs to reconfigure TLS after the client is built — file *rotation* is not reconfiguration, it is handled inside `TlsMaterialSource`. A user who installs a fully custom `PulsarTlsFactory` configures its material directly, and the v4 bridge then has nothing to contribute. + +The broker's default `DefaultBrokerTlsFactory` is a thin `FileBasedTlsFactory` wrapper whose constructor takes the purpose→policy map composed from the existing `ServiceConfiguration` properties; it lives in **`pulsar-broker-common`** (which owns the Jetty integration today), keeping both `pulsar-tls-factory-api` and `pulsar-common` free of broker configuration knowledge. + +**Client configuration: the `TlsPolicy` value.** The default client-side `PulsarTlsFactory` is wired from the client builder, and **`TlsPolicy` is the single TLS type a user ever touches** — a flat, immutable value describing *what* material to use and the policy flags. It subsumes the experimental PIP-466 `TlsPolicy` (which was PEM-only and lived in `org.apache.pulsar.client.api.v5.config`) and the internal material-source machinery; the type lives in `org.apache.pulsar.tls` in the `pulsar-tls-factory-api` module so the client builder and the server components consume the same value. The builder turns each `TlsPolicy` into an internal `TlsMaterialSource` and configures the `FileBasedTlsFactory` for the relevant purpose(s). + +To keep it friendly to a future configuration file (see below), `TlsPolicy` is a **flat value with a `format` discriminator** rather than a polymorphic hierarchy — one type covering PEM and keystore/truststore, plus the common flags, with static factories for the common shapes: + +```java +package org.apache.pulsar.tls; + +public final class TlsPolicy { + public enum Format { PEM, KEYSTORE } + + public Format format(); + // format == PEM + public String trustCertsFilePath(); + public String certificateFilePath(); + public String keyFilePath(); + // format == KEYSTORE + public String trustStorePath(); + public String trustStorePassword(); + public String keyStorePath(); + public String keyStorePassword(); + public String keyStoreType(); // JKS / PKCS12 (blank -> JDK default) + public String trustStoreType(); // JKS / PKCS12 (blank -> JDK default) + // common flags (both formats) + public boolean allowInsecureConnection(); + public boolean enableHostnameVerification(); + public List protocols(); // optional + public List ciphers(); // optional + public String jsseProvider(); // optional: names the JSSE java.security.Provider that supplies the + // SSLContext, KeyManagerFactory and TrustManagerFactory (e.g. BCJSSE + // for FIPS). When set, the default FileBasedTlsFactory builds the JDK + // Netty engine with this provider (SslProvider.JDK + + // SslContextBuilder.sslContextProvider), taking precedence over the + // factory-level OpenSSL/JDK engine choice. + // Blank/unset = the platform default JSSE provider (today's behaviour). + public String jcaProvider(); // optional: names the JCA java.security.Provider used to create the + // KeyStore / CertificateFactory / KeyFactory engines that parse the + // TLS material (e.g. BCFIPS for FIPS, alongside jsseProvider=BCJSSE). + // JSSE service types (SSLContext, KeyManagerFactory, + // TrustManagerFactory) are NEVER taken from this provider. + // Blank/unset = the JVM provider search order (pre-PIP-478 behaviour). + + public static TlsPolicy pem(String trustCerts, String cert, String key); + // keyStore(...) sets a single storeType on BOTH stores (the common case); for a mixed setup + // (e.g. a PKCS12 keystore with a JKS truststore) use builder().keyStoreType(..).trustStoreType(..). + public static TlsPolicy keyStore(String trustStore, String trustStorePw, + String keyStore, String keyStorePw, String storeType); + public static TlsPolicy insecure(); + public static Builder builder(); +} +``` + +The tier-1 case is one expression — `builder.tlsPolicy(TlsPolicy.pem(trust, cert, key))` — which binds the policy to `CLIENT_DEFAULT`, the purpose all Pulsar-cluster client traffic (binary protocol, HTTP topic lookup, admin) resolves against directly. `CLIENT_OAUTH2` is separate: when unconfigured it resolves terminally to the system default, unless the OAuth2 plugin contributes its own IdP material. The tier-2 case binds a policy to a specific purpose — `builder.tlsPolicy(TlsPurpose.CLIENT_OAUTH2, idpPolicy)` — and any client purpose left unconfigured resolves terminally to the system default (there is no fallback link to `CLIENT_DEFAULT`; cluster-facing consumers use `CLIENT_DEFAULT` directly, so tier-1 already covers them). `TlsPolicy` describes material *locations*, not the loaded material, so it stays a small serializable value; the loading, caching, and rotation are the internal `TlsMaterialSource`'s job. + +**A policy configures a trust domain; it does not switch the broker transport to TLS.** Whether the binary/HTTP connection to the cluster uses TLS is decided by the service URL (`pulsar+ssl://`) — exactly as in v4 — with one narrow addition: `tlsPolicy(CLIENT_DEFAULT, …)` also enables it, because that is the only v5 expression of the legacy `client.conf` `useTls=true` combined with a plain `pulsar://` URL (the v5 builder has no `useTls` knob, and `PulsarClientTool` maps the conf key through it). Every *other* purpose — `CLIENT_OAUTH2` above all — configures its own trust domain only: an HTTPS identity provider behind a private CA must not turn a plaintext `pulsar://` broker connection into a TLS one. `tlsFactory(PulsarTlsFactory)` likewise supplies material for all purposes without enabling transport TLS. + +**Not enabling the transport is not the same as being ignored: the client composes a TLS factory whenever TLS material is configured for *any* purpose.** A client TLS factory is composed (or an adopted one initialized) when *any* of these holds — the broker transport is TLS (`useTls` / `pulsar+ssl://`), a `tlsPolicy(purpose, …)` entry exists for *any* purpose (notably `CLIENT_OAUTH2`), a `tlsFactory(…)` instance was adopted, or the configured OAuth2 plugin carries its own IdP TLS material (folded into `CLIENT_OAUTH2`). This matters because the framework HTTP client resolves its purpose *from the factory when one exists*: without composition a `CLIENT_OAUTH2` policy on a plaintext `pulsar://` client would be dead configuration and the IdP connection would silently fall back to the platform trust store — the v4 behaviour it replaces honoured IdP TLS regardless of broker TLS. Composition is not enablement: binary transport TLS stays strictly `useTls`-gated (in the channel initializer) and HTTP stays scheme-gated, so the invariant above is unaffected. An adopted factory is `initialize()`d with the client's shared scheduler / blocking executor / OpenTelemetry handle and closed with the client even when the broker connection is plaintext. One consequence of the v5-builder path: whenever a policy map or an adopted factory is present, the fail-fast `CLIENT_DEFAULT` probe runs — *including* on a plaintext client — so a custom factory must be able to serve `CLIENT_DEFAULT`. + +The corollary for tools: TLS *modifiers* (hostname verification, allow-insecure, trust/cert paths) shape a policy and never create one, so a secure default shipped in `conf/client.conf` cannot silently make a CLI speak TLS to a plaintext broker. In the CLIs the enabler set is therefore exactly: the `pulsar+ssl://` service-URL scheme, plus (for `pulsar-client`, which reads `client.conf`) an explicit `useTls=true`, which reaches the transport through `tlsPolicy(CLIENT_DEFAULT, …)`. The `https` axis of the v4 `isUseTls()` derivation is deliberately *not* mirrored: the v5 builder accepts only the broker binary protocol and rejects an `http(s)://` service URL with an actionable error, and the tools' HTTPS admin endpoint carries its trust material on the v4 `PulsarAdmin` builder rather than through a `TlsPolicy`. + +**`jsseProvider` — pinning the JSSE (SSLContext) provider.** `TlsPolicy.jsseProvider` names a `java.security.Provider` (matched by `Provider.getName()`) that supplies the JSSE service types the default file-based path builds — the `SSLContext` (`TLS`) itself *and* the `KeyManagerFactory` / `TrustManagerFactory` initialized from the loaded material, the pair that keeps the private-key side inside a FIPS JSSE provider. Naming a crypto-only provider here (BC-FIPS, say) still *resolves*, since it is an installed provider, and then fails one step later at `SSLContext.getInstance("TLS", provider)` with a `NoSuchAlgorithmException` — a different failure from the `IllegalArgumentException` an unresolvable name raises. Names are resolved by the shared **provider-name resolution** described below. When `jsseProvider` is set, the default `FileBasedTlsFactory` builds the **JDK** Netty engine (`io.netty.handler.ssl.SslProvider#JDK`) with that provider installed as `SslContextBuilder#sslContextProvider(...)`, **taking precedence over** the factory-level engine selection — a `jsseProvider` value therefore always pins the JDK engine and never the native OpenSSL one, because Netty's OpenSSL engine cannot delegate to an arbitrary JSSE provider. On the JDK `SSLContext` path the pin also decides where the context's randomness comes from: it is initialized with a `null` `SecureRandom` so the pinned provider's own DRBG is used, rather than a `new SecureRandom()` resolved through the JVM search order (which would seed a validated context from a non-validated module); with no pin the historical explicit instance is kept. + +Because providers differ in the algorithms they register, the two factory types are **negotiated** against the pinned provider rather than demanded from it: the platform default algorithm when the provider registers it, else `PKIX` (needed because BCJSSE registers `X.509` with `X509`/`PKIX` aliases but *not* the JDK's default `SunX509`), else — for a provider that registers no such service at all, Conscrypt for instance — the platform default *factory*. That last step is a bounded, deliberate degradation: the `SSLContext` pin still holds and consumes standard `X509KeyManager`/`X509TrustManager` instances, and a FIPS deployment never reaches it because BCJSSE registers both factories. `jsseProvider` is a **value-level, file-based-simple** field precisely because a deployment's JSSE provider travels with its material and security posture; this is a *different axis* from the raw JDK-vs-OpenSSL **engine** choice, which stays a factory concern and is deliberately not a `TlsPolicy` field (see the [engine-selection note](#pip-337-removal-impact)). A **custom** `PulsarTlsFactory` may interpret `jsseProvider` however it likes — or ignore it entirely; the field carries this defined meaning only for the default file-based path, which always sources its material from the file-based keys. + +**Provider-name resolution (both axes, one mechanism).** A non-blank provider name is resolved in three steps: (1) the **`ServiceLoader`** mechanism (`META-INF/services/java.security.Provider`) on the **thread-context** class loader — falling back to `JcaProviders`' own class loader when the thread has none — matching on `Provider.getName()`; (2) a provider already statically registered in the JVM (`Security.getProvider(name)`); (3) a loud `IllegalArgumentException` when the name resolves to nothing, rather than a silent revert to the JVM default. The first two steps differ in *which instance* answers a name: the `ServiceLoader` step constructs a fresh instance through the provider's no-arg constructor, so for a provider whose mode is a constructor argument (`new BouncyCastleJsseProvider("fips:BCFIPS")` registers under the plain name `BCJSSE`) a statically registered FIPS-mode instance is only reached when the name is not `ServiceLoader`-discoverable. The walk is defensive: a broken `META-INF/services/java.security.Provider` entry belonging to an unrelated provider is skipped rather than aborting resolution. The resolved provider is handed straight to the TLS layer and is never itself installed process-wide via `Security.addProvider` — though invoking the resolver initializes the `JcaProviders` class, whose class initialization does install the Bouncy Castle provider (and Conscrypt, when present) process-wide; that is pre-existing behaviour, independent of the pinned name. Resolution happens where the provider is first needed — when a purpose's material source is constructed and when its context is built — so an unresolvable name surfaces at client build / server start for the eagerly probed purposes (the fail-fast `CLIENT_DEFAULT` probe, a server listener at startup) and at first use for a purpose resolved lazily. + +**`jcaProvider` — pinning the JCA (crypto) provider that parses the material.** `TlsPolicy.jcaProvider` names a second, independent `java.security.Provider`, resolved exactly as above. It is used for exactly the JCA *material* engine classes on the default file-based path: + +- `KeyStore.getInstance(type, provider)` — the operator's keystore/truststore, and the process-local in-memory carrier store that ferries PEM material into the `KeyManagerFactory`; +- `CertificateFactory.getInstance("X.509", provider)` — PEM certificate and trust-chain parsing; +- `KeyFactory.getInstance(algorithm, provider)` — PEM private-key parsing, inside the existing per-algorithm loop (an algorithm the pinned provider does not supply is skipped, exactly as an unsupported algorithm is today). + +That carrier store is the one place where a pin changes something an operator can observe beyond *who* parses the material. Unpinned it is built exactly as before — the caller's default store type (`PKCS12` on the keystore multi-alias path, the JDK default type on the PEM carrier), with the **PEM** carrier's key entries under an empty password; the keystore multi-alias carrier generates a fresh per-build password either way, so a pin changes only its store type. Pinned, both follow the provider: the type becomes the first of **`BCFKS`, `PKCS12`** the pinned provider registers (failing loudly, listing what it does register, if it registers neither), and the entries are protected by a **freshly generated 32-character password**, generated per carrier store, never persisted or logged; the password is zeroed as soon as the `KeyManagerFactory` has been initialized from it — on the PEM carrier that clears the caller's copy while the holder keeps its own, so its entries stay readable, and on the keystore multi-alias carrier it clears the array itself, the store having already been consumed — because PKCS12 and BCFKS protect key entries with a password-based KDF whose SP 800-132 constraints a FIPS provider in approved-only mode enforces, rejecting a zero-length password outright. + +It is deliberately **never** applied to the JSSE service types — `SSLContext`, `KeyManagerFactory`, `TrustManagerFactory` stay pinned to `jsseProvider`. That is not a symmetry oversight: those are JSSE service types, and a crypto-only provider such as BC-FIPS registers none of them, so routing them through `jcaProvider` would throw `NoSuchAlgorithmException` and break a FIPS deployment rather than enable it. The Netty engine axis (`SslProvider.JDK|OPENSSL`) is likewise untouched. + +**Why two axes are required for FIPS.** BCJSSE registers `SSLContext`/`KeyManagerFactory`/`TrustManagerFactory` and **no** `KeyStore` or `CertificateFactory` services; BCFIPS registers `KeyStore` (`BCFKS`, `PKCS12`, …), `CertificateFactory` (`X.509`) and `KeyFactory` and **no** JSSE services. Pinning only `jsseProvider=BCJSSE` therefore leaves every material `getInstance` call falling through the JVM search order to SUN/SunRsaSign — a FIPS-*shaped* configuration whose private key object was manufactured outside the validated module. The FIPS configuration is the pair: **`jsseProvider=BCJSSE` + `jcaProvider=BCFIPS`**, with the JSSE provider registered in FIPS mode by the operator (`new BouncyCastleJsseProvider("fips:BCFIPS")`, or the `java.security` static-registration equivalent). Note the naming: the BouncyCastle FIPS JSSE provider registers under the name **`BCJSSE`** whether or not it was constructed in FIPS mode — the FIPS-ness is a constructor argument, not part of `Provider.getName()` — so there is no provider named `BCFIPSJSSE` to configure. + +**Semantics and defaults.** Blank/unset `jcaProvider` means *exactly today's behaviour*: every affected call site keeps its one-argument `getInstance` form, i.e. the JVM provider search order. Unlike `jsseProvider` it is **explicit-only** — no legacy v4 value is routed onto this axis — so nothing can start landing here by surprise. `jcaProvider` is orthogonal to the store type (`TlsPolicy.keyStoreType()` / `trustStoreType()`, fed by the v4-named `tls*StoreType` config keys listed under [Configuration](#configuration) — there is no config key literally named `keyStoreType`): the type chooses *which* store format, `jcaProvider` chooses *who supplies* it. When the pinned provider does not register the requested type, the failure is **loud** wherever the affected store is created — at *material load* for the operator's keystore/truststore, i.e. at component start and again on every rotation reload, and at context build for the in-memory carrier — with an actionable message naming the types the provider does register. That is the opposite terminal action from the `KeyManagerFactory`/`TrustManagerFactory` negotiation above, which degrades because those algorithms are interchangeable and the `SSLContext` pin still holds; a silent fallback here would instead void the exact property the operator set the field to obtain. A keystore-based FIPS deployment therefore sets every store-type key to `BCFKS` (BouncyCastle's FIPS-approved store format) or `PKCS12`; `JKS` — the v4-parity default on every such key — is not registered by BCFIPS and is intentionally rejected, so leaving one at its default fails the load loudly (the surface-by-surface key list is under [Configuration](#configuration)). A **PEM** deployment consults no store-type key at all, which makes PEM plus the provider pair the shortest FIPS configuration. Both provider fields are trimmed and blank-normalized to unset in the builder — server configuration surfaces pass raw config values straight through, and `TlsPolicy` value equality drives rotation-change suppression, so an empty `jcaProvider=` line must yield a policy equal to one where the key is absent rather than a value that looks like a material change on every reload. + +**One bounded exception, reported rather than silent.** On the outbound (`BROKER_CLIENT`) leg the TLS identity may come from an authentication plugin rather than the policy's own files. With `jcaProvider` pinned, the plugin's certificate/key **file paths take precedence over the objects it already parsed** — plugins such as `AuthenticationTls` parse their PEM eagerly in their constructor, through the JVM search order, so re-reading the same file through the pinned provider yields the same identity from the right module; the plugin's truststore stream and keystore parameters are likewise loaded through it. A plugin that exposes pre-parsed key objects without a certificate/key **file path** cannot be brought inside the pin, because the objects were manufactured before Pulsar saw them — and a keystore path does not rescue it, since the pre-parsed objects are consulted before the keystore parameters; that case emits a one-time WARN naming the pinned provider and the auth-data class and pointing at the file-path configuration. It is the only path on which a pinned deployment's key objects can originate outside the pinned provider. + +**Config-file (de)serialization (out of scope, but accommodated).** Some deployments keep client configuration in a properties or JSON file rather than building it programmatically. Wiring a config-file loader for the v5 client is out of scope for this PIP, but it shapes one decision here: keeping `TlsPolicy` a flat, discriminated value (and `TlsPurpose` a plain named key usable as a config key) means it can later map cleanly to flat keys (e.g. `tls.default.trustCertsFilePath=…`, `tls.oauth2.trustStorePath=…`) without polymorphic type-tag handling. + +**Custom factories** implement `PulsarTlsFactory` directly and may externalize the entire TLS configuration (e.g. to a KMS). A custom factory that still uses files can build on the public `TlsContexts` helper (in `pulsar-common`) to construct the well-known instance classes; the file-watching `TlsMaterialSource`, however, is **package-private** to `FileBasedTlsFactory`, so a custom factory implements its own load/cache/rotation (or, where the default behavior suffices, wraps `FileBasedTlsFactory` itself). The same implementation can serve both client and broker sides, or differ between them (a server-side KMS integration often differs from the client-side one). + +**Removal.** The existing v4 `org.apache.pulsar.common.util.PulsarSslFactory` and `PulsarSslConfiguration` are removed completely — an intentional breaking change. Because the SPI now answers ready-built TLS objects per purpose (rotation is factory-internal, surfaced through the reload callbacks), the `org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext` class is no longer needed either; the default file-based factory reads keystores (PKCS12/JKS) directly for the private key, key-cert chain, and trust certificates. The working assumption — one to confirm on `dev@pulsar.apache.org` before the vote — is that custom PIP-337 factories are rare in the field (the SPI is recent and niche); on that basis PIP-337 is removed rather than retained alongside the new SPI. Were that assumption to prove wrong, a thin PIP-337→PIP-478 compatibility adapter could be reintroduced additively as a fallback (the migration sketch under [Upgrade](#upgrade) shows how close the two shapes are). + +The internal TLS utility layer is decomposed at the same time, applying this PIP's own anti-kitchen-sink philosophy to the helpers: the generic `org.apache.pulsar.common.util.SecurityUtility` grab-bag is **removed**, its surviving primitives split into cohesive single-concern containers under `org.apache.pulsar.common.util.tls` — `PemReader` (PEM certificate/key parsing), `JcaProviders` (Bouncy Castle / Conscrypt provider resolution), and `JdkSslContexts` (JDK `SSLContext` assembly). The delegate-swap rotation helpers `org.apache.pulsar.common.util.TrustManagerProxy` / `KeyManagerProxy` — and `SecurityUtility.createAutoRefreshSslContextForClient`, their only caller — are **removed as obsolete**: the rebuild-the-context rotation model described next is precisely why they are no longer needed (see [Appendix B](#appendix-b-delegate-swap-rotation-and-netty-openssl)). Also removed with `SecurityUtility` are its unused Netty context-builder helpers (`createNettySslContextForClient` / `createNettySslContextForServer`), the per-connection `configureSSLHandler` (hostname verification is now baked into the built context, not re-applied per connection), and the unused `org.apache.pulsar.common.util.keystoretls.SSLContextValidatorEngine`. This layer carries no `@InterfaceStability` annotation and has no client-API importer, so its decomposition and removal are internal-only and not a public-API break. + +**Rotation model: rebuild the context, don't mutate behind a stable one.** The reload callback hands consumers a freshly built `SslContext` / `SSLContext` on each rotation, rather than keeping one context alive and swapping new material in behind mutable delegates — the approach of `org.apache.pulsar.common.util.TrustManagerProxy` / `KeyManagerProxy`. The delegate-swap pattern interacts badly with Netty's **OpenSSL** provider in three ways: a custom `KeyManager` forfeits the native/caching key-manager factories and requires an *extractable* private key, ruling out HSM/PKCS#11 keys — the very case this PIP targets; a mid-handshake delegate swap can race alias selection against material fetch; and a server's advertised client-CA list (the TLS *CertificateRequest*) is written into the native context once at build and never follows the swap. Rebuilding the whole context avoids all three, and its cost — one build per rotation rather than per connection — is negligible. The detailed analysis is in [Appendix B: delegate-swap rotation and Netty OpenSSL](#appendix-b-delegate-swap-rotation-and-netty-openssl). + +### `LegacyV4AuthenticationAdapter` (v4 → v5) + +Wraps an arbitrary v4 `Authentication` instance as a v5 `Authentication`. Selects the plugin's style by its advertised **auth-method name** — `"tls"` → `LegacyV4TlsAdapter`, `"sasl"` → `LegacyV4ChallengeResponseAdapter`, anything else → `LegacyV4CredentialAdapter` — and declares the matching v5 capability. Routing is deliberately by name, not by probing `has*()`: probing would run v4 credential I/O on the caller thread, breaking the PIP-478 offload discipline. The `has*()` methods are consulted only *post-start*, on the blocking executor, to decide which single-pass capabilities the credential adapter actually advertises: + +```java +package org.apache.pulsar.client.impl.v5.auth; + +public abstract class LegacyV4AuthenticationAdapter implements Authentication { + + protected final org.apache.pulsar.client.api.Authentication v4; + protected AuthenticationInitContext ctx; + + /** + * Wraps {@code v4} and returns an instance that declares the right + * capability interfaces for its style — typically + * {@link BinaryAuthDataProvider} (+ {@link HttpAuthHeadersProvider}) + * for one-pass plugins (Token, Basic, OAuth2, Athenz) or + * {@link BinaryAuthChallengeHandler} for SASL. + * Plugins reporting {@code hasDataForTls() == true} have their TLS material + * registered with the client's {@link PulsarTlsFactory} by the bridge + * and are represented by the built-in {@link TlsAuthentication} plugin (see "TLS + * override hook" below). + */ + public static Authentication wrap(org.apache.pulsar.client.api.Authentication v4); + + @Override public void configure(Map p) { v4.configure(p); } + @Override public CompletableFuture initializeAsync(AuthenticationInitContext c); + @Override public void close() throws Exception; +} +``` + +Internally, the concrete subclasses cover the cases: + +- `LegacyV4CredentialAdapter implements BinaryAuthDataProvider, HttpAuthHeadersProvider` — the default for any plugin whose method name is neither `"tls"` nor `"sasl"` (e.g. `AuthenticationToken`). *Post-start*, on the blocking executor, it probes the started plugin's `hasDataFromCommand()` / `hasDataForHttp()` and advertises (via `capability(...)`) only the single-pass capabilities the plugin actually supports; it renders the credential into binary bytes (`v4.getAuthData(host).getCommandData()`) and/or HTTP headers (`getHttpHeaders()`). +- `LegacyV4ChallengeResponseAdapter implements BinaryAuthDataProvider, BinaryAuthChallengeHandler` — selected for the method name `"sasl"`. It produces the initial frame via `getAuthDataAsync(...)` and routes binary challenges through `respondToChallengeAsync(...)`, driving the v4 `authenticate(AuthData)` method across rounds (the per-exchange v4 provider is kept in the call-context state slot). **Consequence of name-based routing:** a third-party challenge-response plugin whose method name is *not* `"sasl"` is routed to the credential adapter instead, so its multi-round `authenticate(AuthData)` flow is not driven — a known limitation of bridging arbitrary v4 challenge-response plugins. + +The v4 `Authentication.authenticationStage(...)` HTTP multi-round hook is **not** carried through the bridge. The built-in SASL plugin never needs it there — it becomes v5-native under the internal migration (in-scope item #7) and implements `HttpAuthChallengeHandler` directly. A *third-party* v4 plugin that drives HTTP challenge/response through `authenticationStage(...)` is explicitly unsupported on the v5 path (it keeps working unchanged on the v4 client API); bridging it would mean reproducing the v4 plugin-driven resubmit loop inside the v5 driver, and no such third-party plugin is known to exist. +- `LegacyV4TlsAdapter extends TlsAuthentication` — selected for the method name `"tls"` (the built-in `AuthenticationTls` / `AuthenticationKeyStoreTls`). It reuses the built-in `TlsAuthentication` plugin (so the binary protocol sends `auth_method_name=tls`), and the builder folds the plugin's TLS material into the client's TLS configuration — see the TLS override hook below. + +Three invariants apply across all adapters: + +1. **Always offload to a separate executor (`ctx.blockingExecutor()`).** Even calls to "fast" v4 implementations like `AuthenticationToken.getAuthData()` go through this dedicated executor rather than the scheduler, so a slow or blocking v4 plugin cannot tie up the scheduler's threads or the Netty event loop. The cost is a thread hop, but the simplicity is worth it: the adapter has no class-name allow-list, no per-impl heuristics, and no production hazard from a misclassified plugin. +2. **Translate v4 exceptions.** Any `org.apache.pulsar.client.api.PulsarClientException` thrown by the v4 call is wrapped in the v5 `org.apache.pulsar.client.api.v5.PulsarClientException` and used to complete the returned future exceptionally. +3. **Refresh is the plugin's concern.** The framework has no built-in refresh; proactive renewal and token reuse are internal to the authentication implementation. A short-lived-credential plugin (e.g. `AuthenticationOAuth2`, `AuthenticationAthenz`) renews its credential internally and returns the current one whenever the framework next calls its async credential method — and the broker's `CommandAuthChallenge` refresh sentinel simply triggers that same call. The legacy adapter therefore just re-invokes the wrapped v4 plugin per request; no refresh metadata crosses the capability surface. + +**TLS override hook.** When the v5 client builder is asked to use a v4 plugin that reports `hasDataForTls() == true` (notably the v4 `AuthenticationTls` and `AuthenticationKeyStoreTls` classes, or any third-party equivalent), the bridge does two things during client construction: (1) it **contributes** the plugin's TLS material to the client's TLS configuration — the builder merges it into the purpose→policy map for the relevant client purposes *before* constructing the default `FileBasedTlsFactory` (see the build-time composition note in [`PulsarTlsFactory`](#redesigned-pip-337-ssl-provider-pulsartlsfactory)), rather than mutating or replacing a factory the user may have configured — and (2) it represents the connection's authentication with the built-in `TlsAuthentication` plugin so the binary protocol sends `auth_method_name=tls`. There is no material-extraction hook on the adapter: `LegacyV4AuthenticationAdapter.unwrapV4(...)` recovers the wrapped v4 plugin, and the v5 client builder itself does the inspection — it reads the built-in `AuthenticationTls` / `AuthenticationKeyStoreTls` fields directly, and probes an arbitrary third-party plugin via `hasDataForTls()` — then merges the **file-based** PEM paths or keystore into the purpose→policy map, logging a `WARN` when the plugin's material is only in-memory and so cannot be represented as a file-based `TlsPolicy`. + +This preserves v4's "`AuthenticationTls` is an `Authentication` that overrides TLS" semantics for source compatibility, now expressed cleanly as builder-level TLS material **plus** the built-in `TlsAuthentication` plugin. + +### `ClientCnx` async-driver carve-out (generic, not SASL-specific) + +The v5 client wraps the v4 transport (per PIP-466), so the practical change is twofold: + +1. `V5ToV4AuthenticationAdapter` exposes the v4 `Authentication` interface that `ClientCnx` already drives. +2. A new public interface `org.apache.pulsar.client.api.internal.AsyncAuthenticationDriver` (in `pulsar-client-api`) lets `ClientCnx` detect when the wrapped authentication supports an async path. It is **exchange-scoped**: the driver hands `ClientCnx` an `AuthenticationExchange` for one connection attempt, and every round of that connect is driven through the single exchange object: + +```java +package org.apache.pulsar.client.api.internal; + +public interface AsyncAuthenticationDriver { + // One exchange per connection attempt; ClientCnx drives every round through it. + AuthenticationExchange newAuthenticationExchange(String brokerHostName); + + interface AuthenticationExchange { + CompletableFuture getAuthDataAsync(); // initial connect + CompletableFuture authenticateAsync(AuthData challenge); // ordinary challenge round; the REFRESH sentinel never reaches the exchange + } +} +``` + +The `.internal.` subpackage signals "stable internal — application code should not implement this." `ClientCnx` calls `newAuthenticationExchange(brokerHostName)` once per connection attempt and drives that connect through the returned exchange — the initial credential via `getAuthDataAsync()` and every ordinary `CommandAuthChallenge` round via `authenticateAsync(...)`. The broker's REFRESH sentinel is **not** driven through the current exchange: `ClientCnx` terminates it and opens a **fresh** exchange whose `getAuthDataAsync()` re-produces the credential (binary routing rule 2), so conversation state does not survive a REFRESH. **Exchange scoping matches the state slot's lifetime.** An authentication's per-connection conversation state — for the v4↔v5 bridge, one `AuthenticationCallContext` and its state slot — lives exactly one *exchange*, so the driver hands `ClientCnx` an object that *owns* that state for the duration of the exchange, rather than re-deriving it from a host argument on each call. (An alternative two-method, host-keyed shape — `getAuthDataAsync(String)` / `authenticateAsync(AuthData, String)` — was rejected: the bridge would allocate a fresh call context per call and so lose challenge-response state across rounds.) `ClientCnx.newConnectCommand()` and `ClientCnx.handleAuthChallenge(...)` check `authentication instanceof AsyncAuthenticationDriver async` and route through the exchange when present; a fresh exchange is created per connection attempt (and when `ClientCnx` restarts authentication on the REFRESH sentinel), so concurrent handshakes never share state. **Generic across all challenge types** — connect, REFRESH, SASL multi-round, custom challenge-response. The marker is the only carve-out from the otherwise-untouched v4 client. + +The async and sync handling are **one state machine, not two parallel paths**: credential resolution produces a `CompletableFuture` (already-completed for a sync plugin, computed inline; pending for an async driver), feeding a single continuation that performs every side effect the sync path performs today — assigning the connection's `authenticationDataProvider` (on connect *and* on the REFRESH sentinel), building the command, transitioning connection state, writing, and completing the connection future. The continuation runs inline when the future is already done (preserving today's sync behaviour verbatim) and hops to the channel's event executor only when it was pending. This single-assignment-site structure is a hard requirement: splitting into a *duplicated* async path risks silently dropping the `authenticationDataProvider` assignment. Two invariants are acceptance criteria, enforced by pre-existing tests on master: the connection's `authenticationDataProvider` observable (`getCommandData()`) must reflect the refreshed credential, and a broker-pushed refresh must not disconnect (`getLastDisconnectedTimestamp()` unchanged — see `TokenOauth2AuthenticatedProducerConsumerTest.testOAuth2TokenRefreshedWithoutReconnect`). Failures from the async path are unwrapped (`CompletionException` never leaks) to the matching v4 `PulsarClientException` subtype before failing the connection future. + +When `authentication` is a plain v4 instance (not `AsyncAuthenticationDriver`), `ClientCnx` preserves the existing sync path verbatim — no behavioral change for v4-only callers. + +After the v4 internal migration (in-scope item #7) completes, the **credential-fetching** built-in v4 classes (Token, Basic, OAuth2, Athenz, SASL) keep their v4 synchronous surface verbatim and **additively** expose `AsyncAuthenticationDriver`; each hands `ClientCnx` an exchange that drives a v5-native body through `V5BinaryAuthenticationDriver` (in `pulsar-client`, `org.apache.pulsar.client.impl.auth.v5`), so even users of v4 `PulsarClient` get the async path automatically for the plugins where it matters — the ones that perform credential I/O (Motivation #1). The **no-credential-I/O** built-ins (`AuthenticationDisabled`, `AuthenticationTls`, `AuthenticationKeyStoreTls`) intentionally stay on the verbatim sync path: async buys them nothing, and keeping the default no-auth hot path synchronous avoids routing every connect through the executor machinery. `V5BinaryAuthenticationDriver` shares the same `BinaryAuthenticationExchange` as the v5-builder-path `V5ToV4AuthenticationAdapter` (`pulsar-client-v5`), so on this v4-facing path v5 auth exceptions are mapped **back** to the corresponding v4 `org.apache.pulsar.client.api.PulsarClientException` subtypes — v4 exception types are part of the public API and must be preserved for v4 callers (see the Error model). + +### HTTP multi-round auth drivers + +The binary protocol runs its multi-round loop in one place (`ClientCnx`, above). HTTP is reached through two client APIs — the JAX-RS (Jersey) admin client and the raw-AsyncHttpClient HTTP-lookup client, both over the same AsyncHttpClient transport — so the `401`→resubmit→`200` loop is implemented as one framework-side **driver** (a single shared state machine behind a thin request/response adapter per client API) rather than inside the plugin (the v4 hazard the [Transport B](#transport-b--httphttps-rest-api-admin-client-http-topic-lookup) note describes). The driver detects the challenge style by asking the plugin for the corresponding capability — today only `capability(HttpAuthChallengeHandler.class)` — surfaces each server challenge through `HttpAuthCallContext.serverChallengeHeaders()`, and attaches the plugin's computed headers to the resubmitted request. The SASL driver reproduces v4 behaviour exactly, including re-issuing the exchange as a `GET` to the original URI. + +**HTTP challenge routing (normative).** The HTTP driver routes each request as follows (the HTTP analogue of the binary routing rules above): + +1. **Initial request** → if the plugin exposes `capability(HttpAuthHeadersProvider.class)` (single-pass), its `getHttpHeadersAsync(ctx)` supplies the request's auth headers; a plugin with **only** a challenge handler contributes no initial headers and relies on the server's first `401` to open the exchange (`HttpAuthCallContext.serverChallengeHeaders()` is empty on this first call). +2. **`401` carrying the server's challenge headers** → `capability(HttpAuthChallengeHandler.class).respondToHttpChallengeAsync(ctx)`, with the challenge surfaced through `serverChallengeHeaders()`; the returned headers are attached to the resubmitted request. If the plugin exposes **no** `HttpAuthChallengeHandler`, the `401` is returned to the caller unhandled — the driver does not resubmit. +3. **`2xx` (or any non-`401` response)** → the exchange completes and the response is returned to the caller. + +The style is selected purely by which capability the plugin exposes — today only the SASL-style `HttpAuthChallengeHandler`; a future `WWW-Authenticate` / digest handler would be dispatched the same way. The loop is bounded (see below). + +**Bounded exchange (normative).** The driver enforces limits so a misbehaving server or plugin cannot loop or hang: at most a fixed number of challenge rounds per request (`HttpAuthenticationDriver.MAX_CHALLENGE_ROUNDS`, a `public static final int` = 10), and the *original request's* timeout budget covers the whole exchange — challenge rounds do not reset it. A plugin failure (the handler's future completing exceptionally) fails the request with that error; the driver never retries plugin failures — any retry is the caller's ordinary request-retry policy applied to the whole request. Request replay: the SASL style sidesteps body replay entirely because the exchange is re-issued as a bodiless `GET` to the original URI (the v4 takeover, preserved); a future standard-style driver must define non-replayable-body behaviour before it ships. The standard `WWW-Authenticate` style is a documented future extension, not shipped API — see [Resolved design decisions](#resolved-design-decisions). + +**Who owns the future a budget is applied to (normative).** The framework bounds each stage of the exchange by timing out a defensive **copy** of the stage future, never the future the plugin or the transport returned. `CompletableFuture.orTimeout` returns — and completes — *the receiver*, so bounding an SPI-supplied future in place would permanently fail a plugin that legitimately memoizes one shared credential future (the natural shape of an OAuth2 access-token cache): one slow request would turn into a client-wide authentication outage. A corollary follows from bounding the copy: the framework does **not** cancel the underlying work on timeout — deliberately, since cancelling a shared plugin future is the same hazard — so cleanup on timeout belongs to the stage itself, and both HTTP transports self-bound against the `Duration` they are handed (request timeout on the AsyncHttpClient one; the JAX-RS one times out its own future and cancels the in-flight request). + +**The budget covers the v4-adapter path too.** Credential production is bounded not only in the v5-native driver but on the branch every adapter-backed v5 plugin actually takes on an HTTP lookup: the synthesized v4 `AuthenticationDataProvider.getHttpHeaders()` inside `V5ToV4AuthenticationAdapter` (which runs on the bounded blocking auth executor) waits on the v5 provider with a **timed** `get()` — default 60 s, matching the fallback lookup budget — and surfaces exhaustion as `PulsarClientException.TimeoutException`. The timed `get()` is the same ownership rule expressed on a blocking path: it abandons only that caller's wait and leaves the plugin's (possibly cached) future intact. Placing the bound in the shared synthesized provider, rather than at one call site, means no v4 caller of that provider can miss it. + +### Class-name compatibility and the v4 internal migration + +`authPluginClassName` strings in `ClientConfigurationData` continue to work without modification: + +- The existing v4 class names (`org.apache.pulsar.client.impl.auth.AuthenticationToken`, `AuthenticationTls`, `AuthenticationOAuth2`, `AuthenticationAthenz`, `AuthenticationBasic`, `AuthenticationSasl`, `AuthenticationDisabled`, plus the v4 `AuthenticationKeyStoreTls`) are retained. +- **After the v4 internal migration (in-scope item #7)**, each of those classes keeps its v4 synchronous surface **verbatim** and *additively* implements the async binary driver; it does **not** route its existing sync methods through a v5 delegate. The v5-native body is constructed only when the async path is entered. The migrated `AuthenticationToken` illustrates the shape: + ```java + public class AuthenticationToken + implements org.apache.pulsar.client.api.Authentication, + AsyncAuthenticationDriver, ClientAuthenticationServicesAware { + // v4 sync surface — unchanged from before this PIP: + @Override public String getAuthMethodName() { return "token"; } + @Override public void configure(String params) { /* parse the token locally, as before */ } + @Override public void configure(Map p) { /* no-op, as before */ } + @Override public void start() { /* no-op — no eager I/O */ } + @Override public AuthenticationDataProvider getAuthData() { + return new AuthenticationDataToken(tokenSupplier); // built directly — verbatim v4 + } + // PIP-478, additive: bind the framework's late-bound services, then drive the async binary path. + @Override public void bindClientAuthenticationServices(ClientAuthenticationServices s) { this.authServices = s; } + @Override public AuthenticationExchange newAuthenticationExchange(String brokerHostName) { + // The v5-native body is constructed HERE, only for the async binary path. + return new V5BinaryAuthenticationDriver(new TokenAuthenticationV5(tokenSupplier), authServices) + .newAuthenticationExchange(brokerHostName); + } + } + ``` + So the v5-native body (`TokenAuthenticationV5`) drives **only** the non-blocking binary path — via `V5BinaryAuthenticationDriver`, whose `ensureInitialized()` calls the body's `initializeAsync(...)` lazily on first use — while the v4 sync methods that predate this PIP are untouched. For the HTTP-capable built-ins (SASL over HTTP), the v5-native body additionally drives the framework's HTTP multi-round loop through the `AsyncHttpAuthenticationProvider` seam and the shared `HttpAuthenticationDriver`, rather than a per-class `authenticationStage(...)` hook. +- The v4 `AuthenticationTls` / `AuthenticationKeyStoreTls` are also shims, but rather than wrapping a v5 auth they surface their configured paths/keystore to the client builder, which merges them into the `FileBasedTlsFactory`'s purpose→policy map at client construction (the build-time composition above), and use the built-in `TlsAuthentication` plugin so the binary protocol sends `auth_method_name=tls`. +- **OAuth2 and Athenz invert the body direction — deliberately.** For the two credential-acquisition-heavy plugins the additive-driver shape above is reversed: rather than the v5-native body owning credential acquisition (as `TokenAuthenticationV5` does for the token above), the battle-tested provider-integration machinery is *not* reimplemented on the v5 side. `AuthenticationOAuth2` keeps its OAuth2 flow, early-refresh scheduling, and token cache (`Flow`/`FlowBase`); `AuthenticationAthenz` keeps its ZTS client (on the Athenz SDK transport) and role-token cache. The matching v5 bodies (`OAuth2AuthenticationV5`, `AthenzAuthenticationV5`) are thin *readers* that simply return the current credential the v4 layer supplies through an access-token / role-token `Supplier`. This is a reuse-not-duplicate choice: it exposes the v5 async capability surface — so both v4- and v5-`PulsarClient` users get off-loaded, non-event-loop-blocking credential I/O (Motivation #1) — without forking hard-won provider logic. It is why this PIP speaks of these plugins gaining the *async path*, not of a self-sufficient v5-native reimplementation of OAuth2/ZTS. +- The built-in → v5-native mapping is keyed by **exact fully-qualified class name** — a closed allow-list of the built-ins above — not by interface probing or heuristics. Third-party class names (or v4 classes without a v5-native equivalent) match nothing on that list and go through `LegacyV4AuthenticationAdapter` unchanged. +- `AuthenticationFactory.create(authPluginClassName, authParamsString)` and `AuthenticationFactory.create(authPluginClassName, Map)` continue to instantiate via `AuthenticationUtil.create()`, unchanged. + +The two configuration paths (programmatic vs string-based) both work for the v4 surface: + +- **Programmatic**: `new AuthenticationToken(jwt)` — the constructor stores the JWT exactly as before, and `getAuthData()` still serves it synchronously. The v5-native `TokenAuthenticationV5` is created lazily, only when `ClientCnx` first opens an async exchange; `start()` remains a no-op (no eager I/O). +- **String-based**: `AuthenticationUtil.create("...AuthenticationToken", "{...}")` — reflection invokes the no-arg constructor, then `configure(params)` runs the class's own (verbatim v4) parsing. `start()` remains a no-op; the async body is initialized on first use inside `V5BinaryAuthenticationDriver`, not from `start()`. + +### Error model + +A blanket contract applies to **every** future-returning method introduced by this PIP (`Authentication`, the capability interfaces, `PulsarTlsFactory`, `PulsarHttpClient`): failures — including argument validation and build errors — are reported by completing the returned future exceptionally; the method itself never throws synchronously (Pulsar's async convention; a synchronous throw on a Netty event loop is exactly the class of bug this PIP exists to remove). The exception *type* is SPI-specific, though: only the **auth** SPI completes with the v5 `PulsarClientException` subtypes below. The `PulsarTlsFactory` and `PulsarHttpClient` SPIs live in `pulsar-tls-factory-api` / `pulsar-http-client-api` and do **not** depend on `pulsar-client-api-v5`, so they complete with ordinary exceptions (`IOException`, `GeneralSecurityException`, `IllegalArgumentException`, and the like), not with a v5 `PulsarClientException`. + +All async **auth** failures complete the returned `CompletableFuture` exceptionally with a v5 `org.apache.pulsar.client.api.v5.PulsarClientException`. The relevant subclasses for auth are: + +- `AuthenticationException` — terminal authentication failure (rejected credential, untrusted certificate). Connection is failed. +- `GettingAuthenticationDataException` — transient failure during credential acquisition (token endpoint timeout, ZTS unavailable). The caller may retry; the v5 connection layer treats this the same way it treats network errors. +- `UnsupportedAuthenticationException` — the requested capability is not supported by the wrapped impl (used by `LegacyV4AuthenticationAdapter` when a v4 provider returns null/false from the corresponding `hasData*` method). + +Translation runs in **both** directions, since v4 and v5 exception types are both public API: + +- **v4 → v5** — `LegacyV4AuthenticationAdapter` wraps any v4 `org.apache.pulsar.client.api.PulsarClientException` thrown by a wrapped v4 plugin onto the v5 subclasses above. +- **v5 → v4** — on the v4-facing path (`V5ToV4AuthenticationAdapter` and the built-in v4 shims), v5 auth exceptions are mapped back to the matching v4 `PulsarClientException` subtypes so v4 callers see the exceptions they always have. This is a straightforward one-to-one mapping (v5 `AuthenticationException` → v4 `AuthenticationException`, and so on) that introduces no behavioural change; the acceptance criterion is that the existing v4 authentication tests pass unmodified after the migration. + +## Public-facing Changes + +### Public API + +New public types introduced by this PIP: + +- `org.apache.pulsar.client.api.v5.auth` — `Authentication` (replacing the PIP-466 sync stub); capability interfaces `BinaryAuthDataProvider`, `HttpAuthHeadersProvider`, `BinaryAuthChallengeHandler`, `HttpAuthChallengeHandler`; the convenience composite `SinglePassAuthentication`; the static `AuthenticationFactory` (`token` / `tls` / `create` entry points — the no-arg `tls()` returns the `"tls"` marker plugin, mTLS material goes through `tlsPolicy(...)`); contexts `AuthenticationInitContext`, `AuthenticationCallContext`, `HttpAuthCallContext`; value types `BinaryAuthData`, `HttpAuthHeaders`, `AuthChallenge`, `ChallengeResponse`. +- `org.apache.pulsar.client.api.v5` — the auth exception types are **nested classes on `PulsarClientException`**, not top-level types under `...v5.auth`: `PulsarClientException.AuthenticationException`, `PulsarClientException.GettingAuthenticationDataException`, and `PulsarClientException.UnsupportedAuthenticationException` (see the [Error model](#error-model)). +- `org.apache.pulsar.http` (in `pulsar-http-client-api`) — `PulsarHttpClient`, `PulsarHttpClientFactory`, `PulsarHttpClientConfig`, `HttpRequest`, `HttpResponse`. The `pulsar-http-client-api` module depends on `pulsar-tls-factory-api` (a `PulsarHttpClientConfig` carries a `TlsPurpose`); `pulsar-client-api-v5` depends on both `pulsar-http-client-api` and `pulsar-tls-factory-api` (the v5 client builder exposes `TlsPolicy`/TLS SPI types directly) (see [Resolved design decisions](#resolved-design-decisions)). +- `org.apache.pulsar.tls` (in `pulsar-tls-factory-api`) — the SPI: `PulsarTlsFactory` with the `TlsHandle` handle and `TlsFactoryInitContext`; the `TlsPurpose` named-key value type and the `TlsEndpoint` destination hint; the flat `TlsPolicy` value (PEM and keystore, via a `format` discriminator — the single user-facing TLS type, consumed by the client builder and the server components alike). (The well-known instance classes are `io.netty.handler.ssl.SslContext`, Jetty's `SslContextFactory.Server` and `SslContextFactory.Client`, and the JDK `javax.net.ssl.SSLContext` fallback plus its optional `javax.net.ssl.SSLParameters` engine-policy companion. The three non-JDK ones remain `compileOnly`-style optional references for factories; custom factories targeting the shaded client must publish a relocated artifact, see the shading note in the Detailed Design. The `SSLContext`/`SSLParameters` fallbacks carry no non-JDK dependency.) +- `org.apache.pulsar.common.tls.impl` (in `pulsar-common`) — `FileBasedTlsFactory`, its `FileBasedTlsFactorySettings` value (factory-wide engine provider / client-auth / refresh interval), and the `TlsContexts` build-support helper; `pulsar-common` already carries the Netty engine dependencies these need. The `TlsMaterialSource` that owns per-purpose load/watch/cache is **package-private** to `FileBasedTlsFactory` — an internal implementation detail, not public API. `DefaultBrokerTlsFactory` lives in `pulsar-broker-common`. +- `org.apache.pulsar.client.api.internal` — `AsyncAuthenticationDriver` (observed by `ClientCnx`; application code should not implement it). +- `org.apache.pulsar.client.api.v5.internal` (in `pulsar-client-api-v5`) — `ClientAuthenticationServices` and `ClientAuthenticationServicesAware`, the "stable internal" seam through which the framework late-binds its runtime services (scheduler, bounded blocking executor, HTTP client factory, client instance id) into an authentication driver after the `PulsarClient` is constructed. The `.internal.` subpackage marks them framework-driven — application code neither implements nor consumes them. +- `pulsar-client-v5` (`org.apache.pulsar.client.impl.v5.auth`) — the built-in `TlsAuthentication` mTLS plugin, and the compatibility adapters `LegacyV4AuthenticationAdapter` and `V5ToV4AuthenticationAdapter`. + +Modified v4 public API: `org.apache.pulsar.client.api.ClientBuilder` and `org.apache.pulsar.client.admin.PulsarAdminBuilder` each **gain** `tlsFactoryClassName(String)` and `tlsFactoryConfig(String)` — the by-name selector for a custom `PulsarTlsFactory` from the v4 API (see [Configuration](#configuration)) — and **lose** the PIP-337 `sslFactoryPlugin(String)` / `sslFactoryPluginParams(String)` methods (removed with PIP-337; a v4 source-compatibility break, listed under Removed below and in [PIP-337 removal impact](#pip-337-removal-impact)). + +Dependency note: the v4 `pulsar-client` module gains **three** `api` dependencies — `pulsar-client-api-v5`, `pulsar-tls-factory-api`, and `pulsar-http-client-api` — because the migrated built-in v4 plugins delegate to v5-native bodies (in-scope item #7) and surface the TLS/HTTP SPI types on their exported ABI. The shaded v4 artifacts consequently bundle the v5 auth API, the TLS-factory API, and the HTTP-client API classes — unrelocated, as `org.apache.pulsar.client.api.v5.*`, `org.apache.pulsar.tls.*`, and `org.apache.pulsar.http.*` are Pulsar's own API packages. + +Removed: `org.apache.pulsar.common.util.PulsarSslFactory`, `PulsarSslConfiguration`, and `org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext` (intentional breaking change — see Backward & Forward Compatibility); the PIP-337 v4 builder methods `ClientBuilder.sslFactoryPlugin(String)` / `sslFactoryPluginParams(String)` and `PulsarAdminBuilder.sslFactoryPlugin(String)` / `sslFactoryPluginParams(String)` (a v4 source-compatibility break); and the `ClusterData.brokerClientSslFactoryPlugin` / `...Params` accessors and their builder methods (per-cluster factory selection is now broker-level — see [PIP-337 removal impact](#pip-337-removal-impact)). Also removed, but **internal-only** (no `@InterfaceStability`, no client-API importer — not a compatibility break): the `org.apache.pulsar.common.util.SecurityUtility` grab-bag (decomposed into the cohesive `PemReader` / `JcaProviders` / `JdkSslContexts` containers), the obsolete `TrustManagerProxy` / `KeyManagerProxy` delegate-swap rotation helpers, and the unused `SSLContextValidatorEngine` — see *Removal* in the Detailed Design. + +### Binary protocol + +No new wire-protocol commands. The existing `CommandConnect`, `CommandAuthChallenge`, and `CommandAuthResponse` are used unchanged. + +### Configuration + +**Server-side (new keys, replacing the PIP-337 keys).** Each server component gains a pair of keys naming and parameterizing its `PulsarTlsFactory`: + +- Broker `ServiceConfiguration`: `tlsFactoryClassName` (default: blank — a blank value has the framework build the built-in `DefaultBrokerTlsFactory`) + `tlsFactoryConfig` for the listener/web purposes, and `brokerClientTlsFactoryClassName` + `brokerClientTlsFactoryConfig` for the broker's own outbound clients (the `BROKER_CLIENT` purpose). +- Proxy `ProxyConfiguration`, the WebSocket service configuration, and the functions-worker `WorkerConfig`: the same pattern. (The websocket service and the functions worker hard-instantiate the PIP-337 default factory today; they gain factory pluggability for the first time.) + +These keys are the successors of PIP-337's `sslFactoryPlugin` / `sslFactoryPluginParams` / `brokerClientSslFactoryPlugin` / `brokerClientSslFactoryPluginParams`, which are **removed outright** in 5.0 — not retained as deprecated fields; a stale, non-default value left in a config file is rejected loudly at startup by the removed-key validation. The disposition of every removed key is inventoried under [PIP-337 removal impact](#pip-337-removal-impact). + +**v5 client builder.** Gains `tlsFactory(PulsarTlsFactory)` to supply a custom factory instance (adopted — closed with the client), and `tlsPolicy(TlsPolicy)` / `tlsPolicy(TlsPurpose, TlsPolicy)` to configure one or more `TlsPolicy` values — each covering cert/key/trust paths or a keystore, ciphers/protocols, an optional JSSE (SSLContext) provider (`jsseProvider`), an optional JCA (material) provider (`jcaProvider`), and hostname-verification / insecure-connection flags — bound to a `TlsPurpose` (the single-argument form binds to `CLIENT_DEFAULT`, the purpose all Pulsar-cluster client traffic resolves against directly; an unconfigured client purpose resolves terminally to the system default). The simple file-based case is a single builder-level value, subsuming the PIP-466-era experimental `TlsPolicy` (see the Detailed Design). + +**v4 configuration path.** The v4 string-config path gains a **by-name factory selector**: `ClientConfigurationData` (and the admin path, which shares it) gain `tlsFactoryClassName` / `tlsFactoryConfig`, letting a v4 application name a custom `PulsarTlsFactory` by class and parameterize it with a JSON config — the **same instantiate-by-name model the server uses** (`tlsFactoryClassName` is instantiated reflectively; `tlsFactoryConfig` JSON is parsed to the factory's init params, mirroring the server-side `tlsFactoryConfig`). Until this PIP the v4 config carried only a *transient* factory instance adopted through the v5 builder, so a v4 application could select a custom factory only by routing through the v5 builder — there was no by-name path. v4 applications that don't select a factory keep their existing `tls*` file/keystore settings, which the internal migration maps onto the default factory (see the v4 mapping under [PIP-337 removal impact](#pip-337-removal-impact)). + +**The two provider axes (`jsseProvider` / `jcaProvider`).** Two new config keys map directly onto `TlsPolicy.jsseProvider` and `TlsPolicy.jcaProvider`: the **JSSE** key names the `java.security.Provider` supplying the `SSLContext` (`TLS`) and the key/trust manager factories built from the material; the **JCA** key names the provider supplying the `KeyStore` / `CertificateFactory` / `KeyFactory` engines that parse it. Both are **distinct keys on a different axis** from the engine-only `sslProvider` (client) / `tlsProvider` (broker) settings, which stay a JDK-vs-OpenSSL **engine** switch and are unchanged; a set `jsseProvider` pins the JDK engine, overriding the engine choice. The FIPS pair to configure, why both axes are required, and how a name is resolved are in [the provider-axis design](#redesigned-pip-337-ssl-provider-pulsartlsfactory). Operators normally set both keys in `broker.conf` / `client.conf`. One operator-visible consequence of that resolution order is worth stating here: because a name is matched through `ServiceLoader` **before** `Security.getProvider(name)`, and the `ServiceLoader` step constructs the provider through its no-arg constructor, a FIPS-mode provider an operator statically registered in `java.security` (`new BouncyCastleJsseProvider("fips:BCFIPS")`, which registers under the plain name `BCJSSE`) is silently shadowed by a non-FIPS no-arg instance of the same name whenever that provider is also `ServiceLoader`-discoverable on the class path — the deployment then runs non-FIPS with no error. The two axes share one surface list: + +| Surface | JSSE axis | JCA axis | +|---|---|---| +| `broker.conf` / `standalone.conf` (`ServiceConfiguration`) | `jsseProvider` (listener/web), `brokerClientJsseProvider` | `jcaProvider` (listener/web), `brokerClientJcaProvider` | +| `proxy.conf` (`ProxyConfiguration`) | `jsseProvider` (front-end), `brokerClientJsseProvider` | `jcaProvider` (front-end), `brokerClientJcaProvider` | +| `websocket.conf` (`WebSocketProxyConfiguration`) | `jsseProvider` (web), `brokerClientJsseProvider` | `jcaProvider` (web), `brokerClientJcaProvider` | +| `functions_worker.yml` (`WorkerConfig`) | `jsseProvider` (web), `brokerClientJsseProvider` | `jcaProvider` (web), `brokerClientJcaProvider` | +| `client.conf` / `ClientConfigurationData` | `jsseProvider` | `jcaProvider` | +| OAuth2 auth parameters (`AuthenticationOAuth2`) | `jsseProvider` (client→IdP token fetch) | `jcaProvider` (client→IdP token fetch) | +| `pulsar-perf` / `pulsar-testclient` | `--jsse-provider` | `--jca-provider` | + +Notes on the table: + +- **Defaults.** Both provider keys are unset on every surface, and unset means today's behaviour. The one + non-empty legacy default *routed onto the JSSE axis* is the WebSocket proxy's web-listener + `tlsProvider=Conscrypt`, which the v4 routing below carries there whenever `jsseProvider` itself is left + unset. (`webServiceTlsProvider` on `ServiceConfiguration` and `ProxyConfiguration` also defaults to + `Conscrypt`, but no code reads it — the broker and proxy web listeners take their provider from + `tlsProvider` / `jsseProvider` — so it reaches neither axis.) +- **v4 auto-routing, JSSE axis only.** Existing deployments need no config change: a non-engine-literal + `sslProvider` (client, on the v4 configuration path — `ClientConfigurationData` / `loadConf` / the v4 + client and admin builders; `bin/pulsar-client` reads only `jsseProvider` / `jcaProvider` from + `client.conf`) / `tlsProvider` (listener/web) / `brokerClientSslProvider` (outbound) value — i.e. a + JSSE provider *name* such as `Conscrypt` — is routed automatically to `jsseProvider` and keeps working; the + engine literals stay on the engine axis, and a typo fails loudly at provider resolution rather than being + silently ignored (the full mapping is under [v4 client `tls*` settings](#pip-337-removal-impact)). **No** + legacy value is ever routed onto the JCA axis. +- **Outbound legs carry both axes**, closing the paths where a process could otherwise be pinned on its broker + connection and unpinned elsewhere (the standalone OAuth2 path and pre-parsed plugin material, both described + above, remain the bounded exceptions): the `brokerClient*` rows additionally carry the engine-axis `brokerClientSslProvider`; the + functions worker propagates its three `brokerClient*` provider keys onto its own outbound `PulsarClient` + **and** `PulsarAdmin`, independently of `brokerClientTlsFactoryClassName`, and an embedded (broker-hosted) + worker inherits them — plus the web-listener keys — from the broker configuration when its own file leaves + them unset; the WebSocket proxy's outbound client maps its keys onto the `ServiceConfiguration` the service + reads; and the `pulsar-perf` flags reach the `PulsarAdmin` leg as well as the binary leg. +- **OAuth2 IdP TLS.** The token fetch builds its own `TlsPurpose.CLIENT_OAUTH2` policy from the flow's IdP + material (`tlsCertFile`, `tlsKeyFile`, `trustCertsFilePath`), which is **PEM-only** — no + store-type key applies to it. Per axis, an explicit OAuth2 parameter wins; otherwise the framework-bound + path inherits the composed `CLIENT_DEFAULT` value, so pinning the Pulsar connection also pins the IdP + connection. The standalone (pre-binding) path — the proxy and the CLIs, which create `AuthenticationOAuth2` + through `AuthenticationFactory.create(...)` — has no owning client and therefore **no inheritance step**: + only the explicit parameters apply there. Either way the providers travel on the flow's *own* policy, which + exists only when the flow carries IdP material: a flow with none contributes no policy, so `CLIENT_OAUTH2` + resolves terminally to the system default and stays unpinned. A provider-only policy for that case is a + deliberate follow-up rather than part of this PIP. +- **Neither axis is cross-validated against the other**, on any surface: a configuration pinning only one is + accepted without a warning and yields the FIPS-*shaped* deployment described under + [Security Considerations](#security-considerations), so both keys must be audited everywhere. +- **Store types are a separate audit for FIPS.** A pinned `jcaProvider` that does not register the configured + store type fails the load loudly, and every keystore-TLS surface defaults its store types to `JKS`, which + BCFIPS does not register: `tlsKeyStoreType` / `tlsTrustStoreType` on `ServiceConfiguration`, + `ProxyConfiguration`, `WebSocketProxyConfiguration`, `WorkerConfig` and `ClientConfigurationData` (the last + being also the `client.conf` keys `bin/pulsar-client` reads), plus the `brokerClientTls{Key,Trust}StoreType` + counterparts — which exist on `ServiceConfiguration` and `ProxyConfiguration` only, and, for per-cluster + geo-replication, on `ClusterData` / `ClusterDataImpl`. A keystore deployment pinning `jcaProvider=BCFIPS` + sets each applicable key to `BCFKS` or `PKCS12`; a PEM deployment consults none of them. + +### CLI + +No new CLI commands. The `pulsar-testclient` flags `--ssl-factory-plugin` / `--ssl-factory-plugin-params` are removed with PIP-337 (the perf tools follow the client). The perf tools gain `--jsse-provider` / `--jca-provider` (also readable from `client.conf` through the existing default-value provider), closing a pre-existing gap where neither provider axis was reachable from `pulsar-perf`. The other two CLIs need no flags: `bin/pulsar-client` reads the `jsseProvider` / `jcaProvider` keys straight from `client.conf` onto its `CLIENT_DEFAULT` policy, for both the PEM and keystore forms — but only where TLS is enabled for the connection (`pulsar+ssl://` or `useTls=true`), since TLS *modifiers* never enable TLS — and `bin/pulsar-admin` picks the same two keys up through the v4 `loadConf` mapping onto `ClientConfigurationData`. + +### Metrics + +New OpenTelemetry instruments: + +| Instrument | Type | Attributes | Description | +|---|---|---|---| +| `pulsar.tls.reload` | counter | `purpose`, `result` (`success`/`failure`) | TLS material (re)load events per purpose, client and server side — the initial load of a purpose, and each refresh that found changed material (whether the background poll or a one-shot acquisition observed it). A refresh that finds no change (a steady poll, or a repeat one-shot acquisition on the acquire-per-connection paths) is **not** counted, so the counter tracks real rotations rather than connection volume. The counter is not an *attempt* counter | +| `pulsar.tls.last_reload_success` | gauge (unix time) | `purpose` | Time of the last successful (re)load — advanced only by a counted event above, so it stops moving exactly when rotation silently fails or silently stops happening, letting alerts catch it long before certificates expire | +| `pulsar.client.auth.credential.duration` | histogram | `auth_method` | Latency of async credential acquisition (`getAuthDataAsync` / `getHttpHeadersAsync`) | +| `pulsar.client.auth.failure` | counter | `auth_method`, `error` (`terminal`/`transient`) | Authentication failures by error class (matches the error model) | + +`PulsarHttpClient` request metrics ride on the OpenTelemetry handle already exposed through `AuthenticationInitContext.openTelemetry()`; no bespoke HTTP-metrics API is introduced. + +### PIP-337 removal impact + +`PulsarSslFactory`, `PulsarSslConfiguration`, `DefaultPulsarSslFactory`, and `KeyStoreSSLContext` are removed (see *Removal* in the Detailed Design), and so are the PIP-337 `sslFactoryPlugin` / `sslFactoryPluginParams` / `brokerClientSslFactoryPlugin` / `brokerClientSslFactoryPluginParams` configuration keys themselves — **nothing PIP-337 is retained** (no `@Deprecated` fields, no config-only vestige). Be explicit about what this means operationally: deployment automation that sets the removed server config keys or testclient flags **will fail at startup, loudly and by design** — a 5.0 major-release break chosen over silently ignoring security-relevant settings, and over carrying a PIP-337 adapter for another release (rejected: the adapter would preserve exactly the maintenance burden and kitchen-sink surface this PIP removes, for a plugin population assumed small — an assumption to confirm on `dev@` before the vote, and one the adapter fallback above hedges). **The fail-loud detection replaces, rather than retains, the old field.** The prior design kept the `sslFactoryPlugin` keys as `@Deprecated` non-functional fields *solely* so a stale configuration would be detected rather than silently ignored; this design removes the fields outright and preserves that detection with a **removed-key validation** at config-file load — broker.conf / proxy.conf / client `loadConf` reject any removed PIP-337 key set to a **non-default value** with an actionable migration error. *Non-default value* has a precise meaning: a `*Plugin` key is tolerated (treated as unset) when its value is blank or the old default factory FQCN `org.apache.pulsar.common.util.DefaultPulsarSslFactory`, and rejected for any other value; a `*PluginParams` key has no default, so any non-blank value is rejected. Builder-method removal is inherently loud (a compile error), so no runtime guard is needed there. The one place a stale value *cannot* fail loud is `ClusterData` metadata read from the store (rejecting it would make a cluster unloadable) — the removed field is lenient-dropped, and a stale per-cluster *custom* factory value is the single silently-dropped case (see the row below and the [geo-replication note](#pulsar-geo-replication-upgrade--downgraderollback-considerations)). The public surface that exposes these today, and each item's disposition: + +| Existing surface | Where | Disposition in 5.0 | +|---|---|---| +| `sslFactoryPlugin`, `sslFactoryPluginParams` | broker `ServiceConfiguration` | **Removed** (field and all); replaced by `tlsFactoryClassName` / `tlsFactoryConfig` (see Configuration). A stale, non-default value left in broker.conf is rejected loudly at startup by the **removed-key validation** — not by a retained field | +| `brokerClientSslFactoryPlugin`, `brokerClientSslFactoryPluginParams` | broker `ServiceConfiguration` | **Removed**; replaced by `brokerClientTlsFactoryClassName` / `brokerClientTlsFactoryConfig`; a stale, non-default value is rejected by the same removed-key validation | +| the same four keys | `ProxyConfiguration` | Same treatment as the broker (proxy.conf removed-key validation) | +| hard-instantiated `DefaultPulsarSslFactory` | websocket `ProxyServer`, functions-worker `WorkerServer` | Replaced by the default factory wiring; both components gain the new factory keys | +| `ClientBuilder.sslFactoryPlugin(String)` / `sslFactoryPluginParams(String)` | **v4 public client API** | **Removed**; successor is the new `tlsFactoryClassName(String)` / `tlsFactoryConfig(String)` on the same builder (name a custom `PulsarTlsFactory` by class + JSON config). Deleting the methods is a v4 **source-compatibility break** — a deliberate 5.0 major-release break; the methods were already deprecated and non-functional, and the removal is inherently loud (a compile error), so no runtime guard is needed | +| `PulsarAdminBuilder.sslFactoryPlugin(...)` / `sslFactoryPluginParams(...)` | v4 public admin API | Same treatment as `ClientBuilder` — **removed**, with the new `tlsFactoryClassName(String)` / `tlsFactoryConfig(String)` successors | +| `ClientConfigurationData.sslFactoryPlugin` / `sslFactoryPluginParams` | v4 client config | **Removed**; replaced by new `tlsFactoryClassName` / `tlsFactoryConfig` string fields (today the v4 config carries only a *transient* factory instance set via the v5 builder, no by-name path). A stale, non-default key in a `loadConf` map is rejected by the removed-key validation | +| `ClusterData.brokerClientSslFactoryPlugin` / `...Params` | admin API + **serialized cluster metadata** | **Removed** (field, builder method, and `ClusterData` API accessor). Broker-client factory-class selection is **broker-level** (`ServiceConfiguration.brokerClientTlsFactoryClassName`), not per-cluster. Per-cluster TLS *material* (the existing `brokerClientTls*` fields on `ClusterData`) keeps working — each outbound replication client owns a factory built from its own configuration, which already carries that cluster's material, so no shared-factory purpose minting is needed. Cluster metadata written by 4.x still deserializes: the removed field is **lenient-dropped** on read (Jackson ignores the unknown property), so there is no metadata-format break. This is the one place a stale value **cannot** fail loud — a stale per-cluster *custom* factory value is silently dropped on upgrade (rare; migrate to the broker-level factory). Documented as a breaking change in the [geo-replication note](#pulsar-geo-replication-upgrade--downgraderollback-considerations) | +| `pulsar-admin clusters` `--tls-factory-plugin` / `--tls-factory-plugin-params` options | `CmdClusters` CLI | **Removed** — they wrote the now-removed `ClusterData` fields; per-cluster factory selection no longer exists (set the factory broker-level in broker.conf) | +| `--ssl-factory-plugin`, `--ssl-factory-plugin-params` | `pulsar-testclient` CLI | Removed | + +**Component → purpose mapping.** Every current `PulsarSslFactory` consumer and the `TlsPurpose` its TLS resolves to after migration: + +| Component / connection | Purpose | +|---|---| +| Broker binary listener(s) | `BROKER` | +| Broker web service (Jetty) | `WEB` | +| Broker outbound: geo-replication clients, cluster-internal admin/lookup clients | `BROKER_CLIENT` (each outbound client owns a factory built from its own configuration, which carries that cluster's per-cluster material — no per-cluster purpose is minted) | +| Proxy binary front-end | `PROXY` | +| Proxy web service / `AdminProxyHandler` outbound Jetty client | `WEB` / `BROKER_CLIENT` | +| Proxy → broker binary connections | `BROKER_CLIENT` | +| WebSocket service listener / its internal Pulsar client | `WEB` / `BROKER_CLIENT` | +| Functions-worker web server / its Pulsar + admin clients | `WEB` / `BROKER_CLIENT` | +| Application client: binary protocol, HTTP lookup, admin | `CLIENT_DEFAULT` | +| OAuth2 / IdP HTTP calls | `CLIENT_OAUTH2` | +| `pulsar-perf` / testclient tools | `CLIENT_DEFAULT` | + +**v4 client `tls*` settings.** The ~15 `tls*` fields on `ClientConfigurationData` (`tlsKeyFilePath`, `tlsCertificateFilePath`, `tlsTrustCertsFilePath`, the `tlsKeyStore*` / `tlsTrustStore*` family, `tlsCiphers`, `tlsProtocols`, `tlsAllowInsecureConnection`, `tlsHostnameVerificationEnable`, `useKeyStoreTls`) all continue to work: the internal migration maps them onto a `TlsPolicy` bound to `CLIENT_DEFAULT`. The v4 `tlsKeyStoreType` and `tlsTrustStoreType` are mapped onto **separate** `TlsPolicy.keyStoreType()` / `trustStoreType()` fields (not collapsed into one), preserving v4 parity for a mixed setup such as a PKCS12 keystore with a JKS truststore; the same holds for the server-side `brokerClientTls{Key,Trust}StoreType` mapping. `sslProvider` (JDK vs OpenSSL engine selection) is deliberately **not** a `TlsPolicy` field — the engine is a factory concern, configured on the default `FileBasedTlsFactory` and mapped from the v4 field by the migration. That mapping splits the v4 `sslProvider` value along **two axes**. The Netty `SslProvider` **engine literals** — `JDK`, `OPENSSL`, `OPENSSL_REFCNT` (case-insensitive) — stay on the **engine** axis (mapped to `FileBasedTlsFactorySettings.engineProvider` — a factory concern, unchanged): `OPENSSL` / `OPENSSL_REFCNT` select the native Netty engine and `JDK` selects the JDK engine, and none of them set `jsseProvider`. Only a value that is **not** an engine literal is treated as a **JSSE provider name** (`Conscrypt`, `SunJSSE`, … — the values 4.x accepted) and routed to **`TlsPolicy.jsseProvider`**, which pins the JDK engine with that `java.security.Provider` building the `SSLContext` (resolved via `ServiceLoader` first, then `Security.getProvider`). (A valid `sslProvider=JDK` therefore stays on the engine axis rather than being misrouted to a non-existent JSSE provider named "JDK".) The named provider is therefore honored on the binary/client path, **restoring the v4 behavior** rather than silently dropping it on upgrade. `jsseProvider` is a first-class `TlsPolicy` field on the *JSSE-provider* axis — distinct from, and orthogonal to, the *engine* axis; it is **not** deferred to `FileBasedTlsFactorySettings`. The one residual delta is fail-fast: a provider name that resolves to no installed `java.security.Provider` now fails loudly at provider resolution — client build / server start for the eagerly probed purposes — where 4.x silently fell back to the JDK default. Operators who prefer to set the JSSE provider explicitly use the dedicated `jsseProvider` config key (see [Configuration](#configuration)) — it is the same field (see [the `jsseProvider` design](#redesigned-pip-337-ssl-provider-pulsartlsfactory)). + +**TLS-utility decomposition.** Beyond the SPI-level removals, the rebuild-not-mutate rotation model (Appendix B) and the new instance-factory SPI make several delegate-swap / kitchen-sink TLS helpers obsolete: the `KeyManagerProxy` / `TrustManagerProxy` auto-refresh delegates and the `keystoretls.SSLContextValidatorEngine` are deleted, and the grab-bag `org.apache.pulsar.common.util.SecurityUtility` is dissolved. Its still-needed primitives are re-homed into small single-concern containers under `org.apache.pulsar.common.util.tls` — `PemReader` (PEM cert/key loading), `JcaProviders` (BouncyCastle / Conscrypt provider resolution), and `JdkSslContexts` (JDK `SSLContext` assembly, consumed by `TlsContexts` in `org.apache.pulsar.common.tls.impl`). The obsolete auto-refresh Netty-context builders are dropped entirely (rotation is now the factory's concern). All of these are internal utilities (no `@InterfaceStability` contract, no client-API surface), so the decomposition is a source-internal cleanup, not a public-API break. + +# Monitoring + +- **Certificate rotation health.** Alert when `pulsar.tls.last_reload_success` for a purpose is older than the deployment's rotation interval, or on any increase of `pulsar.tls.reload{result=failure}`. This matters because the reload failure semantics deliberately keep serving last-good material after a failed rotation — the system stays up, so a rotation problem is visible only through these metrics until the certificate actually expires. +- **Credential acquisition.** `pulsar.client.auth.credential.duration` exposes slow IdP/ZTS token endpoints that in v4 stalled the Netty event loop invisibly (Motivation #1); sustained latency growth indicates an unhealthy identity provider. +- Existing connection/producer/consumer monitoring is unchanged. + +# Security Considerations + +- **TLS is secure by default in 5.0.** Hostname verification is **ON by default** for both the v4 and v5 client *and* for every broker / proxy / geo-replication **outbound** (broker-as-client) connection — a peer whose certificate does not match its hostname/SAN is rejected. Concretely, PIP-478 flips these defaults from `false` to `true`: `tlsHostnameVerificationEnable` on the v4 `ClientConfigurationData` (which also flows into the `CLIENT_DEFAULT` `TlsPolicy`), `tlsHostnameVerificationEnabled` on the broker `ServiceConfiguration` (geo-replication / broker-to-broker lookup) and the proxy `ProxyConfiguration` (proxy→broker), the websocket `WebSocketProxyConfiguration`, and `tlsEnableHostnameVerification` on the functions-worker `WorkerConfig` (their broker-facing clients). The v5 `TlsPolicy` already defaulted secure (`enableHostnameVerification()` true). `allowInsecureConnection` stays **false** everywhere — untrusted server certificates are rejected — unchanged from 4.x. (The dev-only `TlsPolicy.insecure()` preset and the separate BookKeeper-client passthrough `bookkeeper_tlsHostnameVerificationEnabled` are intentionally left as-is.) Enabling any insecure setting is logged once at WARN level, on first use. +- **SAN-only hostname matching (RFC 6125).** Deprecated CN-field hostname matching is **removed**: the custom `TlsHostnameVerifier` and its CN machinery (`SubjectName`, `PublicSuffixMatcher` / `PublicSuffixList`, `DomainType`) are deleted, and both the JDK path (endpoint-identification algorithm `"HTTPS"`) and the Conscrypt / OpenSSL path now perform standard SAN-based (RFC 2818) verification only. A server certificate must carry the hostname in its SubjectAltName extension; a certificate that matches only via its CN field is no longer accepted. This affects **server-hostname** matching only — a *client* certificate's CN is still read as the role token by the broker's `AuthenticationProviderTls`, unchanged. +- **A configured truststore never silently widens trust.** On the **keystore** axis a configured `trustStorePath` whose store holds zero X.509 certificates **fails the load** instead of resolving to an empty trust list, which the context builders would treat as "not configured" and satisfy with the *platform default* trust store — silently trusting every public CA. This restores the 4.x `KeyStoreSSLContext` behavior (the `TrustManagerFactory` was initialized with the explicit store, so a zero-entry store rejected every peer), and because the rotation baseline is committed only after a successful load, an accidental rotation to an empty store keeps the last-good material, WARNs, and retries. The **PEM** axis deliberately keeps the 4.x behavior (an empty/zero-certificate `trustCertsFilePath` falls back to the platform trust store, as `SecurityUtility.setupTrustCerts` did) so existing deployments are not broken — but it now WARNs naming the file. +- **A half-configured identity fails loudly — on both axes.** On the **PEM** axis a `certificateFilePath` without a `keyFilePath` yields no usable identity and was silently dropped, surfacing only as a handshake/authentication failure; the default factory now rejects it at load, naming both fields. The converse is asymmetric on purpose: a `keyFilePath` without a certificate is what 4.x tolerated, so it stays a WARN rather than a new startup failure. On the **keystore** axis the counterpart is a *configured* `keyStorePath` whose store yields no usable key entry (a private key with an X.509 chain) — that fails the load too. The 4.x bound is worth stating precisely, because the two sub-cases differ: a wrong `keyStorePassword` already threw `UnrecoverableKeyException` in 4.x and still does (parity, raised before this check), whereas a store holding *no* key entry initialized 4.x's `KeyManagerFactory` fine but with no aliases — a certain, undiagnosed handshake failure for a server purpose, and a silently identity-less client against a peer that does not request a certificate. Failing the load is therefore a deliberate tightening whose only casualties are deployments that already presented no identity, and setting `keyStorePath` is an unambiguous statement of intent to present one. Keep-last-good applies on both axes: the rotation baseline is committed only after a successful load, so an accidental rotation to a keyless store retains the last-good material, WARNs, and retries. All these checks live in the default factory's material load, not in `TlsPolicy.Builder`, so a custom `PulsarTlsFactory` building its own policies is not constrained by them. +- **Default enabled protocol set.** When no protocols are configured, the framework pins **`{TLSv1.3, TLSv1.2}`** on the built contexts — the native Netty, the JDK-synthesis, and the Jetty include-protocols paths alike — preserving the floor the removed `DefaultPulsarSslFactory` forced rather than silently deferring to the JVM/provider default on upgrade. +- **Jetty web listener client-cert trust is scoped like Netty's.** With optional client auth (`tlsRequireTrustedClientCertOnConnect=false`), an *untrusted* client certificate is accepted at the web listener's handshake **only when `tlsAllowInsecureConnection=true`** — `wantClientAuth` no longer implies trust-all. This diverges from the pre-5.0 Jetty behavior (which trusted any presented client cert whenever client auth was optional) and is a deliberate security fix aligning the web listener with the binary listener's semantics. +- **Provider pinning fails loudly rather than reverting to the JVM default.** The two provider axes are security controls: a `jsseProvider`/`jcaProvider` name that resolves to no provider (neither via `ServiceLoader` on the thread-context class loader — falling back to `JcaProviders`' own loader when the thread has none — nor via `Security.getProvider`) is a hard failure, raised where the provider is first needed — at material-source construction and context build, which for the eagerly probed purposes (the fail-fast `CLIENT_DEFAULT` probe, a server listener at startup) means client build / server start, and at first use for a lazily resolved purpose. A pinned `jcaProvider` that does not register the requested store type likewise fails the material load — at start and on every rotation reload — naming the types it does register. The one deliberate degradation is on the `KeyManagerFactory`/`TrustManagerFactory` axis, and it is broader than algorithm negotiation: the factory algorithm is negotiated against the pinned JSSE provider (platform default, else `PKIX`), and when that provider registers **no** such service at all the framework falls back to the **platform default factory** — the `SSLContext` pin still holds and consumes standard `X509KeyManager`/`X509TrustManager` instances, but the manager factories then come from outside the pinned provider. A FIPS deployment never reaches that last step, because BCJSSE registers both factories (see the [Detailed Design](#redesigned-pip-337-ssl-provider-pulsartlsfactory)). What pinning does **not** claim: Pulsar verifies that a named provider exists and uses it, not that it is FIPS-validated or in approved-only mode, and it does not cross-validate the two axes — pinning only `jsseProvider=BCJSSE` is accepted silently and yields a deployment whose private key was manufactured by whatever provider the JVM search order reached first, outside the validated module. The FIPS configuration is the pair `jsseProvider=BCJSSE` + `jcaProvider=BCFIPS`, and its one bounded gap — an authentication plugin that exposes only pre-parsed key material — is reported with a WARN rather than silently accepted (see the [Detailed Design](#detailed-design)). +- **A statically registered provider can be silently shadowed by a no-arg instance of the same name.** Name resolution tries `ServiceLoader` before `Security.getProvider(name)`, and the `ServiceLoader` step constructs the provider through its no-arg constructor — so where a provider's *mode* is a constructor argument, the registered instance is not necessarily the one used. BouncyCastle's JSSE provider registers under the name `BCJSSE` whether or not it was built as `new BouncyCastleJsseProvider("fips:BCFIPS")`, so an operator who statically registers the FIPS-mode instance in `java.security` gets the non-FIPS no-arg instance instead whenever that provider is also `ServiceLoader`-discoverable on the class path, and the deployment runs non-FIPS with no error. A FIPS audit must therefore confirm which instance answered the name, not merely that static registration was configured (mechanism: [provider-name resolution](#redesigned-pip-337-ssl-provider-pulsartlsfactory)). +- **Private keys need never touch the filesystem — or any Pulsar API.** A custom `PulsarTlsFactory` keeps keys in its HSM/KMS or workload-identity system and returns ready-built TLS objects; key material never crosses a Pulsar interface and is never logged, serialized, or copied by the framework. For JCA-provider-backed keys (PKCS#11, KMS JCA providers) the key never leaves the HSM/KMS at all. + +# Backward & Forward Compatibility + +- **BREAKING CHANGE — TLS hostname verification on by default + SAN required (5.0).** A v4 or v5 client — and any broker / proxy / geo-replication **outbound** connection — that connects to a server whose certificate lacks a SubjectAltName matching the target hostname now **fails the TLS handshake**, where 4.x (hostname verification off by default) silently accepted it. Deprecated CN-field matching is also removed, so a CN-only certificate no longer suffices even when its CN names the host. This is a deliberate secure-by-default hardening — it closes the long-standing "hostname verification off by default is MITM-prone" gap. **Remediation:** regenerate the affected server certificates so each carries a SAN matching the hostname clients connect to (see the pulsar-site *Transport Encryption (TLS)* guide, `security-tls-transport.md`); or, only where a certificate cannot yet be reissued and the risk is understood, restore the prior per-connection behavior by setting `tlsHostnameVerificationEnable=false` (v4/v5 client), `tlsHostnameVerificationEnabled=false` (broker / proxy / websocket), or `tlsEnableHostnameVerification=false` (functions-worker) — **not recommended**, as it re-opens the connection to man-in-the-middle attacks. `allowInsecureConnection` is independent of this and still defaults to `false` (untrusted server certificates are rejected). +- The v5 client is published as a standalone artifact in 5.0.0-M1 (the existing `pulsar-client-v5` and `pulsar-client-api-v5` modules from PIP-466), **unshaded**. The v5 client will not ship a shaded distribution; applications that need shading to resolve a dependency conflict relocate it themselves in their own build, as they would any other dependency (see the plugin-packaging note above). +- **BREAKING CHANGE — PIP-337 removed (5.0).** PIP-337 is removed from Pulsar in Pulsar 5.0 since retaining it in the code base causes an additional maintenance burden and most users don't use it. Concretely: the `sslFactoryPlugin` / `sslFactoryPluginParams` / `brokerClientSslFactoryPlugin` / `brokerClientSslFactoryPluginParams` **config-file keys are removed** and a stale non-default value is **rejected at broker/proxy startup** (and at client `loadConf`) by a removed-key validation; the v4 `ClientBuilder` / `PulsarAdminBuilder` `sslFactoryPlugin(...)` / `sslFactoryPluginParams(...)` methods are **removed** (a source-compatibility break — a compile error on upgrade); the `ClusterData.brokerClientSslFactoryPlugin` / `...Params` fields are **removed** from the metadata model, and with them their `ClusterData` API accessors and builder methods (`brokerClientSslFactoryPlugin(...)` / `brokerClientSslFactoryPluginParams(...)` — another v4 public-API source break); and the `pulsar-admin clusters` `--tls-factory-plugin` / `--tls-factory-plugin-params` CLI options (`CmdClusters`) that wrote those fields are **removed** (per-cluster factory selection no longer exists — set the factory broker-level). Existing PIP-337 users migrate to the new `PulsarTlsFactory` SPI, selected by `tlsFactoryClassName` / `tlsFactoryConfig` — on server config **and**, new in this PIP, on the v4 client and admin builders — the affected public keys/methods/CLI options and each one's disposition are inventoried in [PIP-337 removal impact](#pip-337-removal-impact), and a migration sketch is under Upgrade below. +- **Security-relevant behavior changes (5.0), surfaced here so they aren't buried in the removal inventory:** + - **`sslProvider` JSSE-provider-name now honored via `jsseProvider`.** A v4 deployment that set `sslProvider` to a JSSE *provider name* (e.g. `Conscrypt`, `SunJSSE`) — which the old code accepted — is now routed to `TlsPolicy.jsseProvider`, so the named provider builds the `SSLContext` on the JDK engine on the binary/client path (**restoring v4 parity**, not dropping the provider); the `OPENSSL` / `OPENSSL_REFCNT` literals still select the native engine. The one residual delta is fail-fast: a provider name resolving to no installed `java.security.Provider` now fails loudly at build rather than being silently ignored (see [v4 client `tls*` settings](#pip-337-removal-impact)). + - **`ClusterData` per-cluster factory field removed — the one silently-dropped case.** The `brokerClientSslFactoryPlugin` / `...Params` fields are **removed** from the `ClusterData` metadata model; broker-client factory-class selection is now broker-level (`brokerClientTlsFactoryClassName`). Cluster metadata written by 4.x still deserializes — the removed field is **lenient-dropped** on read (Jackson ignores the unknown property), so there is no metadata-format break — which means a stale per-cluster *custom* factory value is **silently dropped** on upgrade rather than rejected. This is the single removed PIP-337 key whose stale value **cannot** fail loud: a metadata read can't reject without making the cluster unloadable, so no removed-key validation applies there. It is rare; the remedy is to set the factory broker-level. Per-cluster TLS *material* (`brokerClientTls*`) keeps working. See [PIP-337 removal impact](#pip-337-removal-impact) and the [geo-replication note](#pulsar-geo-replication-upgrade--downgraderollback-considerations). + - **Jetty web-listener client-cert trust is scoped like Netty's.** With optional client auth (`tlsRequireTrustedClientCertOnConnect=false`), an *untrusted* client certificate is now accepted at the web listener only when `tlsAllowInsecureConnection=true` — `wantClientAuth` no longer implies trust-all, aligning the web listener with the binary listener (see [Security Considerations](#security-considerations)). + +## Upgrade + +- Existing applications using `pulsar-client-api` (v4) need **no source changes** — except the rare application that called the removed PIP-337 `sslFactoryPlugin(...)` / `sslFactoryPluginParams(...)` builder methods, which no longer compile and must move to `tlsFactoryClassName(...)` / `tlsFactoryConfig(...)` (see below) — but 5.0 carries several **runtime behavior deltas** they should review before upgrading: + - **Hostname verification is on by default** (`tlsHostnameVerificationEnable` now defaults `true`) and **CN-based matching is removed** (a matching SAN is required) — a server certificate without a matching SAN now fails the TLS handshake where 4.x silently accepted it. See [Security Considerations](#security-considerations) and the breaking-change note under [Backward & Forward Compatibility](#backward--forward-compatibility). + - **TLS material loads eagerly at client build** (the fail-fast contract), not lazily at first connection as in v4 — a misconfigured cert/key/trust path now surfaces at `PulsarClient` build instead of on first connect. + - **`sslProvider` provider-name now routes to `jsseProvider`**: the `OPENSSL` / `OPENSSL_REFCNT` literals still select the native Netty engine; a JSSE *provider name* (e.g. `Conscrypt`, `SunJSSE`) that the old code accepted is now carried to `TlsPolicy.jsseProvider` and builds the `SSLContext` on the JDK engine (**v4 parity restored**), failing loudly if the name resolves to no installed provider (see [v4 client `tls*` settings](#pip-337-removal-impact)). + - **The new `jcaProvider` key is a no-op on upgrade.** `jcaProvider` (and `brokerClientJcaProvider`) is unset by default on every surface, and unset means every material `getInstance` call keeps its one-argument form — the JVM provider search order, i.e. the behaviour of every release before this PIP. No config file needs editing, and unlike `jsseProvider` **no legacy v4 value is ever routed onto this axis**, so nothing can start landing there by surprise. Only deployments that opt in see a change; for them the store-type interaction under [Configuration](#configuration) applies. + - **A custom PIP-337 factory via `sslFactoryPlugin`** (rare) must be ported to `PulsarTlsFactory` and selected by `tlsFactoryClassName` / `tlsFactoryConfig` — now available on the v4 client and admin builders (and `ClientConfigurationData`) as well as server config. The old v4 `sslFactoryPlugin(...)` builder methods are **removed** (a compile error on upgrade, not a silent ignore), and a stale, non-default `sslFactoryPlugin` key left in broker.conf / proxy.conf / a client `loadConf` map is **rejected at startup / build** by the removed-key validation (see [PIP-337 removal impact](#pip-337-removal-impact)). + + The v4 *API* surface is otherwise untouched apart from the PIP-337 changes above; the deprecated v4 methods (`getAuthData()` no-arg, `configure(Map)`) are retained per PIP-466's stability promise. Beyond the PIP-337 method swap (removing `sslFactoryPlugin(...)`, adding `tlsFactoryClassName(...)` / `tlsFactoryConfig(...)`), the v4 module gains one new public interface `org.apache.pulsar.client.api.internal.AsyncAuthenticationDriver` (in a `.internal.` subpackage) — application code should not implement it; it is observed by `ClientCnx` to opt into the async path. +- **PIP-337 → PIP-478 migration sketch.** A PIP-337 factory implements `initialize(PulsarSslConfiguration)` / `createInternalSslContext()` / `getInternalSslContext()` / `getInternalNettySslContext()` / `needsUpdate()` / `update()`, with consumers driving its rebuild choreography. The same integration under PIP-478 implements `PulsarTlsFactory`: read parameters in `initialize(TlsFactoryInitContext)`, answer `createInstance(purpose, SSLContext.class)` with a built context per supported purpose (the `TlsContexts` helper covers assembly), deliver rebuilt contexts through the subscribing overload's callback when material rotates, and return `Optional.empty()` for any richer class not built natively — the framework synthesizes the Netty/Jetty objects. Rotation timing, caching, and rebuild ordering move from the consumers into the factory; nothing else needs to be implemented. +- Applications using `pulsar-client-api-v5` (v5) gain the new SPI. Existing `authPluginClassName` strings continue to work across both the v4 and v5 client APIs. +- Mixed v4 + v5 Client API usage in the same JVM is supported. +- The `Serializable` contract on the v4 `Authentication` interface remains, supporting Pulsar Functions and connector frameworks that serialize auth instances. The v5 `Authentication` interface deliberately does **not** extend `Serializable`; `V5ToV4AuthenticationAdapter.writeObject` throws `NotSerializableException` with an actionable message pointing to the `authPluginClassName` + `authParams` migration path. Connectors that serialize auth across class loaders must continue to use the v4 interface or the configuration-based approach. This is the stance for now; specific configuration-serialization needs for frameworks such as Apache Flink will be addressed when Flink support is added to Pulsar 5.0. + +## Downgrade / Rollback + +- Removing the v5 SPI types is a source-incompat break for any application that compiled against them, but the v4 surface is always a rollback target — applications can pin to the v4 `pulsar-client` and the new types are simply unused. +- **Rolling back a provider-pinned deployment drops the pins silently.** `jsseProvider` / `jcaProvider` (and their `brokerClient*` counterparts) are new in 5.0, so a 4.x binary does not bind them: the config file stays valid while the `SSLContext` reverts to the platform JSSE provider and the key material is parsed by whatever the JVM search order reaches first. For a FIPS deployment that is a compliance-relevant regression rather than a performance one, so roll back only with the equivalent 4.x provider configuration (a JSSE provider name on `sslProvider`/`tlsProvider` plus JVM-level registration) in place. + +## Pulsar Geo-Replication Upgrade & Downgrade/Rollback Considerations + +Authentication is a per-client concern: each cluster authenticates the replicator client independently using whichever auth plugin is configured, and no auth-related metadata changes. TLS *is* affected, because `ClusterData` carried per-cluster `brokerClientSslFactoryPlugin`/`...Params` fields in 4.x. Those fields are **removed** from the metadata model in 5.0: broker-client factory-class selection becomes **broker-level** (`ServiceConfiguration.brokerClientTlsFactoryClassName`), not per-cluster, while the per-cluster TLS *material* fields (`brokerClientTls*`) keep working: each outbound replication client owns a factory built from its own configuration (which already carries that cluster's material), resolved under the fixed `BROKER_CLIENT` purpose — no per-cluster purpose is minted. The removed field is **not** a metadata-format break: it is an ordinary optional property, and Jackson **lenient-drops the unknown field on read**, so a mixed-version fleet (4.x and 5.0 brokers reading the same cluster metadata) deserializes cleanly in both directions — a 4.x broker ignores the absent field, and a 5.0 broker ignores a value a 4.x admin still writes. The one consequence to call out: a cluster that had a **custom** `brokerClientSslFactoryPlugin` value set is **silently dropped** on upgrade (the removed-key validation can't run on a metadata read without making the cluster unloadable — this is the single removed PIP-337 key whose stale value cannot fail loud). Such deployments are rare; the remedy is to configure the factory broker-level via `brokerClientTlsFactoryClassName` before or during upgrade. + +# Alternatives + +- **Pure additive `AuthenticationAsync` alongside v4.** Add async methods to a new sibling interface but keep the v4 surface as the data model. Rejected: it extends the kitchen-sink `AuthenticationDataProvider` problem rather than fixing it. Implementations would still mix TLS, HTTP, and command-data concerns in one type. + +- **Reusing AsyncHttpClient directly without an interface.** Skip `PulsarHttpClient` and just expose AsyncHttpClient on `AuthenticationInitContext`. Rejected: it ties Pulsar's public API to the AsyncHttpClient transitive dependency and freezes the framework's ability to ever change its internal backend. The `PulsarHttpClient` interface boundary provides that freedom by itself — which is also why the *opposite* extreme, a publicly pluggable backend SPI, was cut (see the design decision in the Detailed Design). + +- **Splitting the PIP-337 cleanup into a separate sibling PIP.** Rejected. Folding the relocation into this PIP keeps the auth-to-TLS decoupling in one document, avoids interleaved release ordering between two related PIPs, and gives reviewers a single place to evaluate the overall design. + +- **Shipping the `PulsarHttpClient` SPI as its own PIP.** Rejected. The HTTP client SPI exists to serve the auth SPI (Motivation #3: OAuth2's private HTTP clients) and shares its lifecycle contexts and TLS-purpose model; splitting it out would force the two to be co-designed across two documents and two votes, with the auth SPI depending on a not-yet-accepted sibling. It already lives in its own *module* (`pulsar-http-client-api`) so the sibling broker-side PIP can reuse it, but its *design* belongs with the auth SPI it was shaped for (see [Resolved design decisions](#resolved-design-decisions)). + +- **`@Deprecated` and remove the v4 `Authentication` interface outright.** Rejected: PIP-466 explicitly committed v4 to remain unchanged, and the v4 `Authentication` interface carries `@InterfaceStability.Stable`. Removing it would break v4 API usage and every third-party auth plugin in Pulsar 5.0. + +- **A material/configuration TLS SPI instead of the instance factory.** In this alternative, providers return raw *material* — a `TlsEndpointSpec` splitting key/cert-chain/trust-certs (or a `KeyManagerFactory`/`TrustManagerFactory` pair) from optional behavioural configuration — with the framework owning all engine assembly. Its appeal was shading-neutrality: JDK/JCA types are never relocated, so one plugin artifact would serve the shaded and unshaded clients alike. Rejected because it is harder to implement on both sides: every stack the framework integrates must be expressible from the material model, the model must grow whenever a factory needs a capability it cannot express (engine-level options, BoringSSL keyless HSM signing, SNI keystores), and the framework re-owns exactly the build/cache/rotate machinery the factory shape keeps inside the plugin. The shading concern is handled by packaging instead — a relocated plugin artifact for the shaded client, the plain artifact for `pulsar-client-original` / `pulsar-client-admin-original` and the server side (see the shading note in the Detailed Design). PIP-337 silently has the same shading constraint today; documenting the packaging requirement is strictly better than contorting the SPI to avoid it. + +# Resolved design decisions + +Design questions considered during this proposal, now resolved. Each records the options considered, the tradeoffs, and the rationale for the choice, so reviewers can challenge the reasoning rather than re-derive it. The authors consider these settled for this PIP and are not aware of open design questions blocking a vote; anything intentionally left for later is captured under [Out of Scope](#out-of-scope), and reviewers are of course invited to reopen any of the choices below. + +## 1. HTTP multi-round driver design and the SASL-vs-standard interface split + +**Decision.** Ship only the SASL-style `HttpAuthChallengeHandler`. The framework implements **one** `401`→resubmit→`200` state machine, shared behind a thin request/response adapter per HTTP client API (both APIs — JAX-RS/Jersey and raw AsyncHttpClient — run over the same AsyncHttpClient transport). The driver selects a plugin's challenge handling by capability lookup (`capability(HttpAuthChallengeHandler.class)`). A standard `WWW-Authenticate` (e.g. digest) interface is a documented future extension, added as a sibling capability when a real mechanism needs it. + +**Options and tradeoffs.** (a) *(chosen)* SASL-style only — no speculative API; the capability model makes the later addition non-breaking by construction. (b) Define both interfaces now — a complete surface from day one, but the second interface would ship unvalidated by any implementation, and speculative SPI tends to be wrong SPI. (c) One interface with a style discriminator — avoids the split but makes the capability non-self-describing at the type level. + +**Rationale.** Only SASL exercises HTTP multi-round today. Designing the standard-style interface now would be speculation squared: no in-tree implementation to validate it, on top of a capability model whose whole point is that new interfaces can be added later without disturbing existing plugins. + +## 2. Config-file (de)serialization of the v5 client configuration + +**Decision.** Deferred to a follow-up. This PIP only shapes the types so a loader is additive: `TlsPolicy` is a flat value with a `format` discriminator (not a polymorphic hierarchy), `TlsPurpose` is a plain named key usable as a config key (e.g. `tls.default.trustCertsFilePath=…`, `tls.oauth2.trustStorePath=…`), and HTTP-client plugins select TLS by purpose key, never inline material — so nothing in the auth/TLS surface resists a flat properties/JSON binding. + +**Options and tradeoffs.** (a) *(chosen)* defer, keep the config-friendly shapes; (b) define the schema and binding now — extra scope with no 5.0 consumer; (c) declare programmatic-only forever — forecloses a real deployment need. + +**Rationale.** The type shapes were the only decision that had to be made now; the loader itself has no 5.0 dependency and deserves its own focused proposal. + +## 3. Naming of the four capability interfaces + +**Decision.** Uniform `