From aedae272f51edc9e05ba405ae9a869262f2eb9d4 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:14:19 +0100 Subject: [PATCH 001/296] fix(sockets): enforce backlog overload semantics --- ZTSharp/Sockets/OverlayTcpListener.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/Sockets/OverlayTcpListener.cs b/ZTSharp/Sockets/OverlayTcpListener.cs index 8e5c6b2..2a58bf1 100644 --- a/ZTSharp/Sockets/OverlayTcpListener.cs +++ b/ZTSharp/Sockets/OverlayTcpListener.cs @@ -32,7 +32,7 @@ public OverlayTcpListener(Node node, ulong networkId, int localPort) _localPort = localPort; _acceptQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 128) { - FullMode = BoundedChannelFullMode.DropWrite, + FullMode = BoundedChannelFullMode.Wait, SingleWriter = false, SingleReader = true }); From 007c7da58db567b0a5398f6de25671971c01ab56 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:16:02 +0100 Subject: [PATCH 002/296] fix(sockets): throw receive buffer faults deterministically --- ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs | 33 ++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs index d9fa117..d89bcf9 100644 --- a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs +++ b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Threading; using System.Threading.Channels; namespace ZTSharp.Sockets; @@ -16,14 +17,14 @@ internal sealed class OverlayTcpIncomingBuffer }); private ReadOnlyMemory _currentSegment; private int _currentSegmentOffset; - private bool _remoteClosed; + private int _remoteClosed; private IOException? _fault; - public bool RemoteClosed => _remoteClosed; + public bool RemoteClosed => Volatile.Read(ref _remoteClosed) != 0; public bool TryWrite(ReadOnlyMemory segment) { - if (_fault is not null) + if (Volatile.Read(ref _fault) is not null) { return false; } @@ -50,7 +51,7 @@ public bool TryWrite(ReadOnlyMemory segment) public void MarkRemoteClosed() { - _remoteClosed = true; + Volatile.Write(ref _remoteClosed, 1); _incoming.Writer.TryComplete(); } @@ -60,9 +61,10 @@ public void Complete() public async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (_fault is not null) + var fault = Volatile.Read(ref _fault); + if (fault is not null) { - throw _fault; + throw fault; } if (_currentSegment.Length == 0 || _currentSegmentOffset >= _currentSegment.Length) @@ -76,8 +78,20 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can break; } - if (_remoteClosed) + if (Volatile.Read(ref _remoteClosed) != 0) { + fault = Volatile.Read(ref _fault); + if (fault is not null) + { + throw fault; + } + + if (_incoming.Reader.Completion.IsFaulted && + _incoming.Reader.Completion.Exception?.InnerException is IOException ioException) + { + throw ioException; + } + return 0; } @@ -109,13 +123,12 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can private void Fault(IOException exception) { - if (_fault is not null) + if (Interlocked.CompareExchange(ref _fault, exception, null) is not null) { return; } - _fault = exception; - _remoteClosed = true; + Volatile.Write(ref _remoteClosed, 1); _incoming.Writer.TryComplete(exception); } } From 48dabfdb8a32efbb9049ab492901b3ceb66e3216 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:21:23 +0100 Subject: [PATCH 003/296] fix(transport): handle NAT rebinding and rate-limit discovery responses --- ZTSharp.Tests/OsUdpSpoofingTests.cs | 18 +++++-- .../Transport/Internal/OsUdpReceiveLoop.cs | 53 ++++++++++++++++--- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/ZTSharp.Tests/OsUdpSpoofingTests.cs b/ZTSharp.Tests/OsUdpSpoofingTests.cs index 721fa9f..71030f2 100644 --- a/ZTSharp.Tests/OsUdpSpoofingTests.cs +++ b/ZTSharp.Tests/OsUdpSpoofingTests.cs @@ -1,4 +1,5 @@ using System.Net.Sockets; +using System.Net; using ZTSharp.Sockets; using ZTSharp.Transport; @@ -15,14 +16,16 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForUdpHandlers() { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await using var node2 = new Node(new NodeOptions { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await node1.StartAsync(); @@ -69,14 +72,16 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForOverlayTcpListeners { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await using var node2 = new Node(new NodeOptions { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await node1.StartAsync(); @@ -124,7 +129,10 @@ private static UdpClient CreateSpoofUdp(System.Net.IPEndPoint destination) } var v4 = new UdpClient(System.Net.Sockets.AddressFamily.InterNetwork); - v4.Client.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Any, 0)); + var spoofAddress = destination.Address.Equals(IPAddress.Parse("127.0.0.2")) + ? IPAddress.Parse("127.0.0.3") + : IPAddress.Parse("127.0.0.2"); + v4.Client.Bind(new System.Net.IPEndPoint(spoofAddress, 0)); return v4; } } diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index 6d0a29f..7975469 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Net; using System.Net.Sockets; @@ -5,6 +6,9 @@ namespace ZTSharp.Transport.Internal; internal sealed class OsUdpReceiveLoop { + private const int MaxHelloResponseCacheEntries = 4096; + private const long PeerHelloResponseMinIntervalMs = 1000; + private readonly UdpClient _udp; private readonly Func> _receiveAsync; private readonly bool _enablePeerDiscovery; @@ -12,6 +16,7 @@ internal sealed class OsUdpReceiveLoop private readonly Func, CancellationToken, Task> _dispatchFrameAsync; private readonly Func _sendDiscoveryFrameAsync; private readonly Action? _log; + private readonly Dictionary<(ulong NetworkId, ulong NodeId), long> _helloResponseLastSentMs = new(); public OsUdpReceiveLoop( UdpClient udp, @@ -83,12 +88,15 @@ public async Task RunAsync(CancellationToken cancellationToken) { if (controlFrameType == OsUdpPeerDiscoveryProtocol.FrameType.PeerHello) { - _ = SendDiscoveryFrameSafeAsync( + if (ShouldSendHelloResponse(networkId, discoveredNodeId)) + { + _ = SendDiscoveryFrameSafeAsync( networkId, localNodeId, normalizedRemoteEndpoint, OsUdpPeerDiscoveryProtocol.FrameType.PeerHelloResponse, cancellationToken); + } } } } @@ -96,16 +104,24 @@ public async Task RunAsync(CancellationToken cancellationToken) continue; } - if (sourceNodeId != 0 && - _peers.TryGetPeers(networkId, out var peers) && - peers.TryGetValue(sourceNodeId, out var expectedEndpoint) && - expectedEndpoint.Equals(normalizedRemoteEndpoint)) + if (sourceNodeId == 0 || !_peers.TryGetPeers(networkId, out var peers) || !peers.TryGetValue(sourceNodeId, out var expectedEndpoint)) { - // Valid authenticated-by-endpoint peer. + continue; } - else + + if (!expectedEndpoint.Equals(normalizedRemoteEndpoint)) { - continue; + // Allow NAT rebinding / port changes without opening too wide a spoofing hole: + // - same IP, different port: accept and update endpoint + // - different IP: require a discovery control frame (handled above) before accepting + if (expectedEndpoint.Address.Equals(normalizedRemoteEndpoint.Address)) + { + _peers.AddOrUpdatePeer(networkId, sourceNodeId, normalizedRemoteEndpoint); + } + else + { + continue; + } } try @@ -124,6 +140,27 @@ public async Task RunAsync(CancellationToken cancellationToken) } } + private bool ShouldSendHelloResponse(ulong networkId, ulong remoteNodeId) + { + if (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) + { + _helloResponseLastSentMs.Clear(); + } + + var key = (networkId, remoteNodeId); + var nowMs = Environment.TickCount64; + if (_helloResponseLastSentMs.TryGetValue(key, out var lastMs)) + { + if (unchecked(nowMs - lastMs) < PeerHelloResponseMinIntervalMs) + { + return false; + } + } + + _helloResponseLastSentMs[key] = nowMs; + return true; + } + private async Task SendDiscoveryFrameSafeAsync( ulong networkId, ulong localNodeId, From 6f72bf1eb364cc002565a18e4e652a0a0a44db51 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:23:10 +0100 Subject: [PATCH 004/296] fix(transport): refresh OS UDP peer directory entries --- .../Transport/Internal/OsUdpPeerRegistry.cs | 19 +++++++++ ZTSharp/Transport/OsUdpNodeTransport.cs | 40 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 3af5f7a..5d527fd 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -98,6 +98,25 @@ public IEnumerable> RegisterLocalAndGetKnownPeer .ToArray(); } + public void RefreshLocalRegistration(ulong networkId, ulong localNodeId, IPEndPoint advertisedEndpoint) + { + if (!_enablePeerDiscovery) + { + return; + } + + if (!_localNodeIds.TryGetValue(networkId, out var registeredNodeId) || registeredNodeId != localNodeId) + { + return; + } + + var nowTicks = GetNowTicks(); + var normalizedAdvertisedEndpoint = _normalizeEndpoint(advertisedEndpoint); + var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + discoveredPeers[localNodeId] = new PeerDirectoryEntry(normalizedAdvertisedEndpoint, nowTicks); + SweepDirectory(nowTicks); + } + public void RegisterDiscoveredPeer(ulong networkId, ulong sourceNodeId, IPEndPoint remoteEndpoint) { if (!_enablePeerDiscovery) diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index 5030596..a76efbb 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -11,6 +11,8 @@ namespace ZTSharp.Transport; /// internal sealed class OsUdpNodeTransport : INodeTransport, IAsyncDisposable { + private static readonly TimeSpan PeerDiscoveryRefreshInterval = TimeSpan.FromSeconds(90); + private sealed record Subscriber( ulong NodeId, Func, CancellationToken, Task> OnFrameReceived); @@ -18,11 +20,14 @@ private sealed record Subscriber( private readonly UdpClient _udp; private readonly SemaphoreSlim _gate = new(1, 1); private readonly ConcurrentDictionary> _networkSubscribers = new(); + private readonly ConcurrentDictionary _advertisedEndpoints = new(); private readonly OsUdpPeerRegistry _peers; private readonly OsUdpReceiveLoop _receiver; private readonly CancellationTokenSource _receiverCts = new(); private readonly Task _receiverLoop; private readonly bool _enablePeerDiscovery; + private readonly CancellationTokenSource _peerRefreshCts = new(); + private readonly Task? _peerRefreshLoop; public OsUdpNodeTransport(int localPort = 0, bool enableIpv6 = true, bool enablePeerDiscovery = true) { @@ -38,6 +43,11 @@ public OsUdpNodeTransport(int localPort = 0, bool enableIpv6 = true, bool enable SendDiscoveryFrameAsync); _receiverLoop = Task.Run(() => _receiver.RunAsync(_receiverCts.Token)); + + if (enablePeerDiscovery) + { + _peerRefreshLoop = Task.Run(() => RefreshLocalDiscoveryEntriesAsync(_peerRefreshCts.Token)); + } } public IPEndPoint LocalEndpoint @@ -63,6 +73,7 @@ public async Task JoinNetworkAsync( var advertisedEndpoint = localEndpoint is null ? UdpEndpointNormalization.NormalizeForAdvertisement(LocalEndpoint) : UdpEndpointNormalization.NormalizeForAdvertisement(localEndpoint); + _advertisedEndpoints[networkId] = advertisedEndpoint; var subscribers = _networkSubscribers.GetOrAdd( networkId, _ => new ConcurrentDictionary()); @@ -91,6 +102,7 @@ public async Task LeaveNetworkAsync(ulong networkId, Guid registrationId, Cancel } _peers.RemoveNetworkPeers(networkId); + _advertisedEndpoints.TryRemove(networkId, out _); if (!_networkSubscribers.TryGetValue(networkId, out var networkSubscribers)) { return; @@ -209,6 +221,7 @@ await SendDiscoveryFrameAsync( public async ValueTask DisposeAsync() { await _receiverCts.CancelAsync().ConfigureAwait(false); + await _peerRefreshCts.CancelAsync().ConfigureAwait(false); try { await _receiverLoop.ConfigureAwait(false); @@ -217,12 +230,39 @@ public async ValueTask DisposeAsync() { } + if (_peerRefreshLoop is not null) + { + try + { + await _peerRefreshLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) when (_peerRefreshCts.IsCancellationRequested) + { + } + } + _udp.Dispose(); _peers.Cleanup(); _receiverCts.Dispose(); + _peerRefreshCts.Dispose(); _gate.Dispose(); } + private async Task RefreshLocalDiscoveryEntriesAsync(CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(PeerDiscoveryRefreshInterval); + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var entry in _advertisedEndpoints) + { + if (_peers.TryGetLocalNodeId(entry.Key, out var localNodeId) && localNodeId != 0) + { + _peers.RefreshLocalRegistration(entry.Key, localNodeId, entry.Value); + } + } + } + } + private async Task DispatchFrameAsync(ulong sourceNodeId, ulong networkId, ReadOnlyMemory payload, CancellationToken cancellationToken) { if (!_networkSubscribers.TryGetValue(networkId, out var subscribers)) From 0eb0aadad4230eecffab8660484a8cfd9fa973ad Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:24:39 +0100 Subject: [PATCH 005/296] fix(transport): dispose UDP sockets on bind failures --- .../Transport/Internal/OsUdpSocketFactory.cs | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs index 58c2f69..b04de76 100644 --- a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs +++ b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs @@ -63,23 +63,47 @@ internal static UdpClient CreateSocketCore( private static UdpClient CreateUdp4Bound(int localPort) { var udp4 = new UdpClient(AddressFamily.InterNetwork); - udp4.Client.Bind(new IPEndPoint(IPAddress.Any, localPort)); - return udp4; + try + { + udp4.Client.Bind(new IPEndPoint(IPAddress.Any, localPort)); + return udp4; + } + catch + { + udp4.Dispose(); + throw; + } } private static UdpClient CreateUdp6DualModeBound(int localPort) { var udp6 = new UdpClient(AddressFamily.InterNetworkV6); - udp6.Client.DualMode = true; - udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); - return udp6; + try + { + udp6.Client.DualMode = true; + udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); + return udp6; + } + catch + { + udp6.Dispose(); + throw; + } } private static UdpClient CreateUdp6OnlyBound(int localPort) { var udp6 = new UdpClient(AddressFamily.InterNetworkV6); - udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); - return udp6; + try + { + udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); + return udp6; + } + catch + { + udp6.Dispose(); + throw; + } } private static void TryDisableWindowsUdpConnReset(UdpClient udp, Action? log) From 3a42a253a45f2cbfcb2e26cc9d30fda7fac4a565 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:30:00 +0100 Subject: [PATCH 006/296] fix(state): mitigate reparse traversal and bound reads --- ZTSharp/FileStateStore.cs | 51 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index ef0967c..8fc6aca 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -6,6 +6,8 @@ namespace ZTSharp; /// public sealed class FileStateStore : IStateStore { + private const long MaxReadBytes = 64L * 1024 * 1024; + private readonly string _rootPath; private readonly StringComparison _pathComparison; private readonly string _rootPathPrefix; @@ -70,6 +72,7 @@ public async Task WriteAsync(string key, ReadOnlyMemory value, Cancellatio } var path = GetPhysicalPathForNormalizedKey(normalized, key); + EnsureParentDirectoryExistsNoReparse(path); await Internal.AtomicFile.WriteAllBytesAsync(path, value, cancellationToken).ConfigureAwait(false); if (string.Equals(normalized, Internal.NodeStoreKeys.IdentitySecretKey, StringComparison.OrdinalIgnoreCase)) @@ -248,6 +251,11 @@ private static async Task ReadAllBytesWithSharingAsync(string path, Canc bufferSize: 16 * 1024, options: FileOptions.Asynchronous | FileOptions.SequentialScan); + if (stream.Length > MaxReadBytes) + { + throw new IOException($"State file exceeds maximum supported size of {MaxReadBytes} bytes."); + } + using var memory = stream.Length <= int.MaxValue ? new MemoryStream((int)stream.Length) : new MemoryStream(); await stream.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); return memory.ToArray(); @@ -330,6 +338,49 @@ private void ThrowIfPathTraversesReparsePoint(string fullPath) } } + private void EnsureParentDirectoryExistsNoReparse(string fullPath) + { + var directory = Path.GetDirectoryName(fullPath); + if (string.IsNullOrWhiteSpace(directory) || string.Equals(directory, _rootPathTrimmed, _pathComparison)) + { + return; + } + + directory = Path.GetFullPath(directory); + if (!IsUnderRoot(directory)) + { + throw new ArgumentException("Path is not under state root.", nameof(fullPath)); + } + + var relative = Path.GetRelativePath(_rootPathTrimmed, directory); + if (relative == "." || relative.Length == 0) + { + return; + } + + var current = _rootPathTrimmed; + var parts = relative.Split(new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < parts.Length; i++) + { + current = Path.Combine(current, parts[i]); + if (Directory.Exists(current)) + { + if (IsReparsePoint(current)) + { + throw new InvalidOperationException("State path traversal via symlink/junction/reparse point is not allowed."); + } + + continue; + } + + Directory.CreateDirectory(current); + if (IsReparsePoint(current)) + { + throw new InvalidOperationException("State path traversal via symlink/junction/reparse point is not allowed."); + } + } + } + private static bool IsReparsePoint(string path) { try From fc8979be31561bd6f5dbf1bb7543c09f621d0339 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:30:56 +0100 Subject: [PATCH 007/296] fix(state): expand reserved Windows device names --- ZTSharp.Tests/StateStoreKeyNormalizationTests.cs | 3 +++ ZTSharp/StateStoreKeySegmentValidation.cs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs b/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs index d5bb0fb..de4f8e7 100644 --- a/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs +++ b/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs @@ -6,6 +6,9 @@ public sealed class StateStoreKeyNormalizationTests [InlineData("con")] [InlineData("CON")] [InlineData("con.txt")] + [InlineData("conin$")] + [InlineData("conout$")] + [InlineData("clock$")] [InlineData("nul")] [InlineData("lpt1")] [InlineData("peers.d/con/peer")] diff --git a/ZTSharp/StateStoreKeySegmentValidation.cs b/ZTSharp/StateStoreKeySegmentValidation.cs index 2b62e8a..d7e1e02 100644 --- a/ZTSharp/StateStoreKeySegmentValidation.cs +++ b/ZTSharp/StateStoreKeySegmentValidation.cs @@ -5,9 +5,12 @@ internal static class StateStoreKeySegmentValidation private static readonly string[] ReservedDeviceNames = [ "CON", + "CONIN$", + "CONOUT$", "PRN", "AUX", "NUL", + "CLOCK$", "COM1", "COM2", "COM3", From 7acac16c3a2bdc2f9f772727bd7c67a929954429 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:33:42 +0100 Subject: [PATCH 008/296] fix(state): harden secret writes via atomic temp perms --- ZTSharp/FileStateStore.cs | 7 +- ZTSharp/Internal/AtomicFile.cs | 241 ++++++++++++++++++ .../Internal/ZeroTierIdentityStore.cs | 3 +- 3 files changed, 246 insertions(+), 5 deletions(-) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index 8fc6aca..f5b7231 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -73,12 +73,13 @@ public async Task WriteAsync(string key, ReadOnlyMemory value, Cancellatio var path = GetPhysicalPathForNormalizedKey(normalized, key); EnsureParentDirectoryExistsNoReparse(path); - await Internal.AtomicFile.WriteAllBytesAsync(path, value, cancellationToken).ConfigureAwait(false); - if (string.Equals(normalized, Internal.NodeStoreKeys.IdentitySecretKey, StringComparison.OrdinalIgnoreCase)) { - Internal.SecretFilePermissions.TryHardenSecretFile(path); + await Internal.AtomicFile.WriteSecretBytesAsync(path, value, cancellationToken).ConfigureAwait(false); + return; } + + await Internal.AtomicFile.WriteAllBytesAsync(path, value, cancellationToken).ConfigureAwait(false); } public Task DeleteAsync(string key, CancellationToken cancellationToken = default) diff --git a/ZTSharp/Internal/AtomicFile.cs b/ZTSharp/Internal/AtomicFile.cs index f1f9feb..91991c1 100644 --- a/ZTSharp/Internal/AtomicFile.cs +++ b/ZTSharp/Internal/AtomicFile.cs @@ -2,6 +2,8 @@ namespace ZTSharp.Internal; internal static class AtomicFile { + private const UnixFileMode SecretFileMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + public static void WriteAllBytes(string path, ReadOnlySpan bytes) { ArgumentException.ThrowIfNullOrWhiteSpace(path); @@ -83,6 +85,17 @@ public static void WriteAllBytes(string path, ReadOnlySpan bytes) } } + public static void WriteSecretBytes(string path, ReadOnlySpan bytes) + { + if (OperatingSystem.IsWindows()) + { + WriteAllBytes(path, bytes); + return; + } + + WriteAllBytesCore(path, bytes, SecretFileMode); + } + public static async Task WriteAllBytesAsync( string path, ReadOnlyMemory bytes, @@ -169,4 +182,232 @@ public static async Task WriteAllBytesAsync( } } } + + public static Task WriteSecretBytesAsync( + string path, + ReadOnlyMemory bytes, + CancellationToken cancellationToken = default) + { + if (OperatingSystem.IsWindows()) + { + return WriteAllBytesAsync(path, bytes, cancellationToken); + } + + return WriteAllBytesCoreAsync(path, bytes, SecretFileMode, cancellationToken); + } + + private static void WriteAllBytesCore(string path, ReadOnlySpan bytes, UnixFileMode unixCreateMode) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(directory)) + { + Directory.CreateDirectory(directory); + } + + var tmpPath = $"{path}.tmp.{Guid.NewGuid():N}"; + try + { + using (var stream = CreateSecretWriteStream(tmpPath, unixCreateMode, asynchronous: false)) + { + stream.Write(bytes); + stream.Flush(flushToDisk: true); + } + + const int maxAttempts = 10; + Exception? lastException = null; + var moved = false; + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + try + { + File.Move(tmpPath, path, overwrite: true); + moved = true; + break; + } + catch (IOException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + Thread.Sleep(TimeSpan.FromMilliseconds(10 * (attempt + 1))); + } + catch (UnauthorizedAccessException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + Thread.Sleep(TimeSpan.FromMilliseconds(10 * (attempt + 1))); + } + catch (IOException ex) + { + lastException = ex; + break; + } + catch (UnauthorizedAccessException ex) + { + lastException = ex; + break; + } + } + + if (!moved) + { + throw new IOException($"Atomic replace failed after {maxAttempts} attempts: '{path}'.", lastException); + } + } + finally + { + try + { + if (File.Exists(tmpPath)) + { + File.Delete(tmpPath); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + } + + private static async Task WriteAllBytesCoreAsync( + string path, + ReadOnlyMemory bytes, + UnixFileMode unixCreateMode, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(directory)) + { + Directory.CreateDirectory(directory); + } + + var tmpPath = $"{path}.tmp.{Guid.NewGuid():N}"; + try + { + using (var stream = CreateSecretWriteStream(tmpPath, unixCreateMode, asynchronous: true)) + { + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); +#pragma warning disable CA1849 + stream.Flush(flushToDisk: true); +#pragma warning restore CA1849 + } + + const int maxAttempts = 10; + Exception? lastException = null; + var moved = false; + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + try + { + File.Move(tmpPath, path, overwrite: true); + moved = true; + break; + } + catch (IOException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + await Task.Delay(TimeSpan.FromMilliseconds(10 * (attempt + 1)), cancellationToken).ConfigureAwait(false); + } + catch (UnauthorizedAccessException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + await Task.Delay(TimeSpan.FromMilliseconds(10 * (attempt + 1)), cancellationToken).ConfigureAwait(false); + } + catch (IOException ex) + { + lastException = ex; + break; + } + catch (UnauthorizedAccessException ex) + { + lastException = ex; + break; + } + } + + if (!moved) + { + throw new IOException($"Atomic replace failed after {maxAttempts} attempts: '{path}'.", lastException); + } + } + finally + { + try + { + if (File.Exists(tmpPath)) + { + File.Delete(tmpPath); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + } + + private static FileStream CreateSecretWriteStream(string path, UnixFileMode unixCreateMode, bool asynchronous) + { + if (!OperatingSystem.IsWindows()) + { + try + { + var options = new FileStreamOptions + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = 16 * 1024, + Options = asynchronous ? FileOptions.Asynchronous : FileOptions.None, + UnixCreateMode = unixCreateMode + }; + + return new FileStream(path, options); + } + catch (PlatformNotSupportedException) + { + } + catch (NotSupportedException) + { + } + } + + var fallback = new FileStream( + path, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 16 * 1024, + options: asynchronous ? FileOptions.Asynchronous : FileOptions.None); + + try + { + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(path, unixCreateMode); + } + } + catch (PlatformNotSupportedException) + { + } + catch (IOException ex) + { + fallback.Dispose(); + throw new IOException("Failed to set secret file permissions.", ex); + } + catch (UnauthorizedAccessException ex) + { + fallback.Dispose(); + throw new IOException("Failed to set secret file permissions.", ex); + } + + return fallback; + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs b/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs index 3c6f7f3..9aceee5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs @@ -94,7 +94,6 @@ public static void Save(string path, ZeroTierIdentity identity) identity.PublicKey.CopyTo(bytes.AsSpan(5 + 8, ZeroTierIdentity.PublicKeyLength)); identity.PrivateKey.CopyTo(bytes.AsSpan(5 + 8 + ZeroTierIdentity.PublicKeyLength, ZeroTierIdentity.PrivateKeyLength)); - AtomicFile.WriteAllBytes(path, bytes); - SecretFilePermissions.TryHardenSecretFile(path); + AtomicFile.WriteSecretBytes(path, bytes); } } From 4f7a5bc1ae61f50b62e3bb76b68a8d42f232f6f7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:35:40 +0100 Subject: [PATCH 009/296] fix(tcp): prevent FIN/data race and unify terminal errors --- .../ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs | 13 +++++++++++-- ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs | 10 +++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs index df1ff76..93a50a1 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs @@ -28,9 +28,11 @@ public void Update(ushort windowSize) public void SignalWaiters(Exception exception) { TaskCompletionSource? toRelease = null; + Exception terminal; lock (_lock) { _terminalException ??= exception; + terminal = _terminalException; if (_windowTcs is not null) { toRelease = _windowTcs; @@ -38,7 +40,7 @@ public void SignalWaiters(Exception exception) } } - toRelease?.TrySetException(exception); + toRelease?.TrySetException(terminal); } public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) @@ -73,7 +75,14 @@ public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) tcs = _windowTcs ??= new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); } - await tcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await tcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (_terminalException is not null && ex is not OperationCanceledException) + { + throw _terminalException; + } } } } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs index d7e823a..0f9bbb1 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs @@ -113,14 +113,14 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, Func getAck return; } - if (_finSeq is not null) - { - throw new IOException("Local has closed the connection."); - } - await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { + if (_finSeq is not null) + { + throw new IOException("Local has closed the connection."); + } + var remaining = buffer; while (!remaining.IsEmpty) { From 24b36744207f1efdb5754bb79122dabab5c5a173 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:38:26 +0100 Subject: [PATCH 010/296] fix(ipv6): skip extension headers when locating transport --- ZTSharp/ZeroTier/Net/Ipv6Codec.cs | 108 ++++++++++++++++++ .../ZeroTier/Net/UserSpaceTcpReceiveLoop.cs | 2 +- .../Net/UserSpaceTcpServerReceiveLoop.cs | 2 +- ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs | 2 +- 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs index f38d4e7..a489dce 100644 --- a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs +++ b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs @@ -7,6 +7,7 @@ internal static class Ipv6Codec { public const byte Version = 6; public const int HeaderLength = 40; + private const int MaxExtensionHeaderChain = 8; public static bool IsExtensionHeader(byte nextHeader) => nextHeader is 0 or 43 or 44 or 50 or 51 or 60; @@ -92,4 +93,111 @@ public static bool TryParse( payload = packet.Slice(HeaderLength, payloadLength); return true; } + + public static bool TryParseTransportPayload( + ReadOnlySpan packet, + out IPAddress source, + out IPAddress destination, + out byte protocol, + out byte hopLimit, + out ReadOnlySpan transportPayload) + { + transportPayload = default; + protocol = 0; + + if (!TryParse(packet, out source, out destination, out var nextHeader, out hopLimit, out var payload)) + { + return false; + } + + return TryWalkExtensionHeaders(payload, nextHeader, out protocol, out transportPayload); + } + + private static bool TryWalkExtensionHeaders( + ReadOnlySpan payload, + byte nextHeader, + out byte protocol, + out ReadOnlySpan transportPayload) + { + protocol = nextHeader; + transportPayload = payload; + + var offset = 0; + for (var i = 0; i < MaxExtensionHeaderChain && IsExtensionHeader(protocol); i++) + { + var remaining = payload.Length - offset; + if (remaining < 2) + { + return false; + } + + if (protocol is 0 or 43 or 60) + { + if (remaining < 8) + { + return false; + } + + var headerNext = payload[offset]; + var hdrExtLen = payload[offset + 1]; + var headerLength = (hdrExtLen + 1) * 8; + if (headerLength > remaining) + { + return false; + } + + offset += headerLength; + protocol = headerNext; + continue; + } + + if (protocol == 44) + { + if (remaining < 8) + { + return false; + } + + var headerNext = payload[offset]; + var fragmentOffsetAndFlags = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(offset + 2, 2)); + var fragmentOffset = (fragmentOffsetAndFlags >> 3) & 0x1FFF; + if (fragmentOffset != 0) + { + return false; + } + + offset += 8; + protocol = headerNext; + continue; + } + + if (protocol == 51) + { + if (remaining < 8) + { + return false; + } + + var headerNext = payload[offset]; + var payloadLen32 = payload[offset + 1]; + var headerLength = (payloadLen32 + 2) * 4; + if (headerLength > remaining) + { + return false; + } + + offset += headerLength; + protocol = headerNext; + continue; + } + + if (protocol == 50) + { + return false; + } + } + + transportPayload = payload.Slice(offset); + return !IsExtensionHeader(protocol); + } } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs index e2f52f9..3152a03 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs @@ -170,7 +170,7 @@ private bool TryParseAndFilterTcpPacket( } else { - if (!Ipv6Codec.TryParse(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) { return false; } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs index 8c933f4..f0e155e 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs @@ -273,7 +273,7 @@ private bool TryParseAndFilterTcpPacket( } else { - if (!Ipv6Codec.TryParse(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) { return false; } diff --git a/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs b/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs index e910786..590e013 100644 --- a/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs +++ b/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs @@ -11,7 +11,7 @@ public static uint Derive(ReadOnlySpan ipPacket) return DeriveFromTransportTuple(src.GetAddressBytes(), dst.GetAddressBytes(), protocol, payload); } - if (Ipv6Codec.TryParse(ipPacket, out src, out dst, out var nextHeader, out _, out payload)) + if (Ipv6Codec.TryParseTransportPayload(ipPacket, out src, out dst, out var nextHeader, out _, out payload)) { return DeriveFromTransportTuple(src.GetAddressBytes(), dst.GetAddressBytes(), nextHeader, payload); } From 7064a431e978a7a271ab9daf18f98f3c659468d7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:39:38 +0100 Subject: [PATCH 011/296] fix(compression): clear pooled buffers before return --- ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs index 206f101..8a70262 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs @@ -42,7 +42,7 @@ public static bool TryUncompress(ReadOnlySpan packet, out byte[] uncompres } finally { - System.Buffers.ArrayPool.Shared.Return(payload, clearArray: false); + System.Buffers.ArrayPool.Shared.Return(payload, clearArray: true); } } } From 311d686180c89015e1d3d9099a595fc6d5a782bf Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:43:15 +0100 Subject: [PATCH 012/296] fix(multipath): stabilize path ordering and active-backup selection --- .../ZeroTierPeerBondPolicyEngineTests.cs | 40 +++++++ .../Internal/ZeroTierPeerBondPolicyEngine.cs | 102 ++++++++++++++++-- 2 files changed, 136 insertions(+), 6 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs b/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs index 98450d8..1512fa6 100644 --- a/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs @@ -82,6 +82,46 @@ public void ActiveBackup_PrefersLowestLatency() Assert.Equal(epFast, selected.RemoteEndPoint); } + [Fact] + public void ActiveBackup_IsSticky_AndSwitchesAfterHold() + { + var peer = new NodeId(0x1111111111); + var epA = new IPEndPoint(IPAddress.Parse("203.0.113.1"), 1001); + var epB = new IPEndPoint(IPAddress.Parse("203.0.113.2"), 1002); + + var paths = new[] + { + new ZeroTierPeerPhysicalPath(LocalSocketId: 1, epA, LastSeenUnixMs: 1), + new ZeroTierPeerPhysicalPath(LocalSocketId: 2, epB, LastSeenUnixMs: 1), + }; + + var now = 1_000L; + var latency = new Dictionary + { + [epA] = 10, + [epB] = 100 + }; + + var engine = new ZeroTierPeerBondPolicyEngine( + getLatencyMs: (_, _, ep) => latency.TryGetValue(ep, out var l) ? l : null, + getRemoteUtility: (_, _, _) => 0, + nowMs: () => now); + + Assert.True(engine.TrySelectSinglePath(peer, (ZeroTierPeerPhysicalPath[])paths.Clone(), flowId: 0, ZeroTierBondPolicy.ActiveBackup, out var s0)); + Assert.Equal(epA, s0.RemoteEndPoint); + + latency[epA] = 200; + latency[epB] = 10; + now += 1_000; + + Assert.True(engine.TrySelectSinglePath(peer, (ZeroTierPeerPhysicalPath[])paths.Clone(), flowId: 0, ZeroTierBondPolicy.ActiveBackup, out var s1)); + Assert.Equal(epA, s1.RemoteEndPoint); + + now += 20_000; + Assert.True(engine.TrySelectSinglePath(peer, (ZeroTierPeerPhysicalPath[])paths.Clone(), flowId: 0, ZeroTierBondPolicy.ActiveBackup, out var s2)); + Assert.Equal(epB, s2.RemoteEndPoint); + } + [Fact] public void BalanceAware_ChoosesOnlyWithinSlack() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs index c61e838..9fcf49e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs @@ -9,6 +9,8 @@ internal sealed class ZeroTierPeerBondPolicyEngine { private const long AwareFlowTtlMs = 120_000; private const int AwareLatencySlackMs = 25; + private const long ActiveBackupMinHoldMs = 10_000; + private const int ActiveBackupLatencySlackMs = 25; private readonly Func _getLatencyMs; private readonly Func _getRemoteUtility; @@ -62,6 +64,9 @@ public bool TrySelectSinglePath( return SelectBalanceAware(peerNodeId, observedPaths, flowId, out selected); case ZeroTierBondPolicy.ActiveBackup: + StableSort(observedPaths); + return SelectActiveBackup(peerNodeId, observedPaths, out selected); + case ZeroTierBondPolicy.Off: default: return SelectBest(peerNodeId, observedPaths, out selected); @@ -115,7 +120,6 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa var bestHasLatency = false; var bestLatency = int.MaxValue; var bestUtility = short.MinValue; - var bestLastSeen = long.MinValue; for (var i = 0; i < observedPaths.Length; i++) { @@ -129,7 +133,7 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa (!bestHasLatency) || (latencyMs < bestLatency) || (latencyMs == bestLatency && remoteUtility > bestUtility) || - (latencyMs == bestLatency && remoteUtility == bestUtility && path.LastSeenUnixMs > bestLastSeen); + (latencyMs == bestLatency && remoteUtility == bestUtility && (bestIndex < 0 || StableComparer.Instance.Compare(path, observedPaths[bestIndex]) < 0)); if (better) { @@ -137,7 +141,6 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa bestHasLatency = true; bestLatency = latencyMs; bestUtility = remoteUtility; - bestLastSeen = path.LastSeenUnixMs; } } else if (!bestHasLatency) @@ -145,13 +148,12 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa var better = (bestIndex < 0) || (remoteUtility > bestUtility) || - (remoteUtility == bestUtility && path.LastSeenUnixMs > bestLastSeen); + (remoteUtility == bestUtility && (bestIndex < 0 || StableComparer.Instance.Compare(path, observedPaths[bestIndex]) < 0)); if (better) { bestIndex = i; bestUtility = remoteUtility; - bestLastSeen = path.LastSeenUnixMs; } } } @@ -166,6 +168,91 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa return true; } + private bool SelectActiveBackup(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths, out ZeroTierSelectedPeerPath selected) + { + var now = _nowMs(); + var state = _peerStates.GetOrAdd(peerNodeId, static _ => new PeerState()); + + lock (state.Gate) + { + if (state.ActiveBackupPath is { } existing && TryFindPath(observedPaths, existing, out var existingPath)) + { + selected = new ZeroTierSelectedPeerPath(existingPath.LocalSocketId, existingPath.RemoteEndPoint); + + if (state.ActiveBackupSelectedAtMs != 0 && unchecked(now - state.ActiveBackupSelectedAtMs) < ActiveBackupMinHoldMs) + { + return true; + } + + if (!SelectBest(peerNodeId, observedPaths, out var best) || best == selected) + { + return true; + } + + if (!ShouldSwitchActiveBackup(peerNodeId, existingPath, best)) + { + return true; + } + + state.ActiveBackupPath = new ZeroTierPeerPhysicalPathKey(best.LocalSocketId, best.RemoteEndPoint); + state.ActiveBackupSelectedAtMs = now; + selected = best; + return true; + } + + if (!SelectBest(peerNodeId, observedPaths, out selected)) + { + return false; + } + + state.ActiveBackupPath = new ZeroTierPeerPhysicalPathKey(selected.LocalSocketId, selected.RemoteEndPoint); + state.ActiveBackupSelectedAtMs = now; + return true; + } + } + + private bool ShouldSwitchActiveBackup(NodeId peerNodeId, ZeroTierPeerPhysicalPath current, ZeroTierSelectedPeerPath best) + { + var bestLatency = _getLatencyMs(peerNodeId, best.LocalSocketId, best.RemoteEndPoint); + var currentLatency = _getLatencyMs(peerNodeId, current.LocalSocketId, current.RemoteEndPoint); + + if (bestLatency is int bestLatencyMs) + { + if (currentLatency is not int currentLatencyMs) + { + return true; + } + + if (bestLatencyMs + ActiveBackupLatencySlackMs < currentLatencyMs) + { + return true; + } + } + else if (currentLatency is int) + { + return false; + } + + var bestUtility = _getRemoteUtility(peerNodeId, best.LocalSocketId, best.RemoteEndPoint); + var currentUtility = _getRemoteUtility(peerNodeId, current.LocalSocketId, current.RemoteEndPoint); + return bestUtility > currentUtility; + } + + private static bool TryFindPath(ZeroTierPeerPhysicalPath[] observedPaths, ZeroTierPeerPhysicalPathKey key, out ZeroTierPeerPhysicalPath path) + { + for (var i = 0; i < observedPaths.Length; i++) + { + if (observedPaths[i].LocalSocketId == key.LocalSocketId && observedPaths[i].RemoteEndPoint.Equals(key.RemoteEndPoint)) + { + path = observedPaths[i]; + return true; + } + } + + path = default; + return false; + } + private bool SelectBalanceAware( NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths, @@ -264,9 +351,12 @@ private static void StableSort(ZeroTierPeerPhysicalPath[] observedPaths) private sealed class PeerState { + public object Gate { get; } = new(); public int RoundRobinCounter; public ConcurrentDictionary Flows { get; } = new(); public long LastFlowCleanupMs; + public ZeroTierPeerPhysicalPathKey? ActiveBackupPath; + public long ActiveBackupSelectedAtMs; } private readonly record struct FlowAssignment(ZeroTierPeerPhysicalPathKey Path, long LastUsedMs); @@ -289,7 +379,7 @@ public int Compare(ZeroTierPeerPhysicalPath x, ZeroTierPeerPhysicalPath y) return c; } - return x.LastSeenUnixMs.CompareTo(y.LastSeenUnixMs); + return 0; } private static int CompareEndpoints(IPEndPoint x, IPEndPoint y) From a31a0df633c237f41a2a7ada38dd2c3f0957edd5 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:51:01 +0100 Subject: [PATCH 013/296] fix(multipath): use all UDP sockets for direct hints and QoS tracking --- .../Internal/ZeroTierDataplaneRuntime.cs | 79 ++++++++++++------- .../Internal/ZeroTierDirectEndpointManager.cs | 78 +++++++++++++++--- samples/ZTSharp.Cli/CliParsing.cs | 2 +- 3 files changed, 119 insertions(+), 40 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 21f6071..c7936a1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -335,36 +335,50 @@ private async ValueTask SendToPeerAsync( return; } + var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); + var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; + var confirmed = _peerEcho.TryGetLastRttMs(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, out _); if (_multipath.WarmupDuplicateToRoot && !confirmed) { - var (packetId, verb) = TryGetPacketIdAndVerb(packet, out var parsed) ? parsed : default; - if (verb != ZeroTierVerb.QosMeasurement) + if (shouldRecord) { - _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId); + _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); } + Task directSend = Task.CompletedTask; + Task rootSend = Task.CompletedTask; try { - await Task - .WhenAll( - _udp.SendAsync(direct.LocalSocketId, direct.RemoteEndPoint, packet, cancellationToken), - _udp.SendAsync(_rootEndpoint, packet, cancellationToken)) - .ConfigureAwait(false); + directSend = _udp.SendAsync(direct.LocalSocketId, direct.RemoteEndPoint, packet, cancellationToken); + rootSend = _udp.SendAsync(_rootEndpoint, packet, cancellationToken); + await Task.WhenAll(directSend, rootSend).ConfigureAwait(false); } catch (SocketException) { - _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId); - await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + // Warm-up duplicates to root. If the direct send fails, forget any QoS state for it and + // ensure a root send happens (if it hasn't already). + // If only the root send fails, keep QoS state intact since the direct send may still succeed. + var directOk = directSend.IsCompletedSuccessfully; + var rootOk = rootSend.IsCompletedSuccessfully; + + if (!directOk && shouldRecord) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); + } + + if (!directOk && !rootOk) + { + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } } return; } - var (packetId2, verb2) = TryGetPacketIdAndVerb(packet, out var parsed2) ? parsed2 : default; - if (verb2 != ZeroTierVerb.QosMeasurement) + if (shouldRecord) { - _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId2); + _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); } try @@ -373,7 +387,11 @@ await Task } catch (SocketException) { - _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId2); + if (shouldRecord) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); + } + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); } } @@ -391,8 +409,8 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo return; } - var parsed = TryGetPacketIdAndVerb(packet, out var pv) ? pv : default; - var shouldRecord = parsed.Verb != ZeroTierVerb.QosMeasurement; + var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); + var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; var anyConfirmed = false; var directSuccess = 0; @@ -427,17 +445,19 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo } } } - else - { - for (var i = 0; i < hinted.Length; i++) + else { - try - { - await _udp.SendAsync(localSocketId: 0, hinted[i], packet, cancellationToken).ConfigureAwait(false); - directSuccess++; - } - catch (SocketException) + var localSockets = _udp.LocalSockets; + for (var i = 0; i < hinted.Length; i++) { + try + { + var localSocketId = localSockets.Count == 0 ? 0 : localSockets[i % localSockets.Count].Id; + await _udp.SendAsync(localSocketId, hinted[i], packet, cancellationToken).ConfigureAwait(false); + directSuccess++; + } + catch (SocketException) + { } } } @@ -465,8 +485,13 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; if (hinted.Length > 0) { - var index = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); - selected = new ZeroTierSelectedPeerPath(LocalSocketId: 0, hinted[index]); + var localSockets = _udp.LocalSockets; + var endpointIndex = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); + var socketIndex = localSockets.Count <= 1 + ? 0 + : (int)((flowId / (uint)hinted.Length) % (uint)localSockets.Count); + var localSocketId = localSockets.Count == 0 ? 0 : localSockets[socketIndex].Id; + selected = new ZeroTierSelectedPeerPath(localSocketId, hinted[endpointIndex]); return true; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index b8a07a1..4aabd40 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -12,6 +12,8 @@ internal sealed class ZeroTierDirectEndpointManager { private const int MaxEndpoints = 8; private const long HolePunchMinIntervalMs = 5_000; + private const long HolePunchCacheTtlMs = 60_000; + private const int HolePunchCacheMaxEntries = 2048; private const long PushDirectPathsCutoffTimeMs = 30_000; private const int PushDirectPathsCutoffLimit = 8; @@ -25,6 +27,7 @@ internal sealed class ZeroTierDirectEndpointManager private IPEndPoint[] _directEndpoints = Array.Empty(); private readonly ConcurrentDictionary _holePunchLastSentMs = new(StringComparer.Ordinal); + private long _lastHolePunchCleanupMs; private long _lastDirectPathPushReceiveMs; private int _directPathPushCutoffCount; @@ -166,19 +169,43 @@ private bool RateGatePushDirectPaths(long nowMs) private void TrySendHolePunch(IPEndPoint endpoint) { - if (!ShouldSendHolePunch(endpoint)) + var localSockets = _udp.LocalSockets; + var now = Environment.TickCount64; + CleanupHolePunchCacheIfNeeded(now); + + var junk = new byte[4]; + RandomNumberGenerator.Fill(junk); + + if (localSockets.Count == 0) { + if (!ShouldSendHolePunch(localSocketId: 0, endpoint, now)) + { + return; + } + + TrySendHolePunchCore(localSocketId: 0, endpoint, junk); return; } - var junk = new byte[4]; - RandomNumberGenerator.Fill(junk); + for (var i = 0; i < localSockets.Count; i++) + { + var socketId = localSockets[i].Id; + if (!ShouldSendHolePunch(socketId, endpoint, now)) + { + continue; + } + TrySendHolePunchCore(socketId, endpoint, junk); + } + } + + private void TrySendHolePunchCore(int localSocketId, IPEndPoint endpoint, byte[] junk) + { Task sendTask; try { - ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint}."); - sendTask = _udp.SendAsync(endpoint, junk, CancellationToken.None); + ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint} (socket={localSocketId})."); + sendTask = _udp.SendAsync(localSocketId, endpoint, junk, CancellationToken.None); } catch (Exception ex) when (ex is ObjectDisposedException or SocketException or OperationCanceledException) { @@ -192,43 +219,70 @@ private void TrySendHolePunch(IPEndPoint endpoint) TaskScheduler.Default); } - private bool ShouldSendHolePunch(IPEndPoint endpoint) + private bool ShouldSendHolePunch(int localSocketId, IPEndPoint endpoint, long nowMs) { - var now = Environment.TickCount64; var keyAddress = endpoint.Address; if (keyAddress.AddressFamily == AddressFamily.InterNetworkV6 && keyAddress.IsIPv4MappedToIPv6) { keyAddress = keyAddress.MapToIPv4(); } - var key = $"{keyAddress}:{endpoint.Port}"; + var key = $"{localSocketId}|{keyAddress}:{endpoint.Port}"; while (true) { if (_holePunchLastSentMs.TryGetValue(key, out var lastSent) && - unchecked(now - lastSent) < HolePunchMinIntervalMs) + unchecked(nowMs - lastSent) < HolePunchMinIntervalMs) { return false; } - if (_holePunchLastSentMs.TryAdd(key, now)) + if (_holePunchLastSentMs.TryAdd(key, nowMs)) { return true; } _holePunchLastSentMs.TryGetValue(key, out lastSent); - if (unchecked(now - lastSent) < HolePunchMinIntervalMs) + if (unchecked(nowMs - lastSent) < HolePunchMinIntervalMs) { return false; } - if (_holePunchLastSentMs.TryUpdate(key, now, lastSent)) + if (_holePunchLastSentMs.TryUpdate(key, nowMs, lastSent)) { return true; } } } + private void CleanupHolePunchCacheIfNeeded(long nowMs) + { + if (_holePunchLastSentMs.Count <= HolePunchCacheMaxEntries) + { + return; + } + + var last = Volatile.Read(ref _lastHolePunchCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 10_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastHolePunchCleanupMs, nowMs, last) != last) + { + return; + } + + var cutoff = nowMs - HolePunchCacheTtlMs; + foreach (var entry in _holePunchLastSentMs) + { + if (entry.Value <= cutoff) + { + _holePunchLastSentMs.TryRemove(entry.Key, out _); + } + } + } + private static string FormatEndpointKey(IPEndPoint endpoint) { var address = endpoint.Address; diff --git a/samples/ZTSharp.Cli/CliParsing.cs b/samples/ZTSharp.Cli/CliParsing.cs index c1aef46..1694b59 100644 --- a/samples/ZTSharp.Cli/CliParsing.cs +++ b/samples/ZTSharp.Cli/CliParsing.cs @@ -187,7 +187,7 @@ public static ZeroTierBondPolicy ParseBondPolicy(string value) throw new InvalidOperationException("Invalid bond policy."); } - value = value.Trim(); + value = value.Trim().ToLowerInvariant(); return value switch { "off" => ZeroTierBondPolicy.Off, From df1fa05dbc35d47a16640206179217c7d131d318 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:02:20 +0100 Subject: [PATCH 014/296] fix(sockets): dispose during connect throws ObjectDisposedException --- ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs index 8abd97f..b2468ad 100644 --- a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs +++ b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs @@ -206,9 +206,14 @@ public override async ValueTask ConnectAsync(EndPoint remoteEndPoint, Cancellati InitLock.Release(); } } - catch (OperationCanceledException) when (ShutdownToken.IsCancellationRequested) + catch (OperationCanceledException ex) { - throw new ObjectDisposedException(GetType().FullName); + if (ShutdownToken.IsCancellationRequested || Disposed) + { + throw new ObjectDisposedException(GetType().FullName, ex); + } + + throw; } finally { From 5ca505734237c1312f30ebaa9578ccee9b3e3688 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:02:30 +0100 Subject: [PATCH 015/296] fix(multipath): use monotonic time for RTT and TTL --- .../Internal/ZeroTierDataplanePeerSecurity.cs | 12 ++-- .../ZeroTierExternalSurfaceAddressTracker.cs | 2 +- .../ZeroTier/Internal/ZeroTierHelloClient.cs | 4 +- .../Internal/ZeroTierPeerEchoManager.cs | 55 +++++++++++++++++-- .../ZeroTierPeerPhysicalPathTracker.cs | 2 +- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs index 4566238..40a4b52 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs @@ -62,7 +62,7 @@ public bool TryGetPeerKey(NodeId peerNodeId, out byte[] key) { key = Array.Empty(); - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; if (!TryGetPeerKeyEntry(peerNodeId, nowMs, out var entry)) { return false; @@ -79,7 +79,7 @@ public bool TryGetPeerKey(NodeId peerNodeId, out byte[] key) public void EnsurePeerKeyAsync(NodeId peerNodeId) { - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; if (TryGetPeerKeyEntry(peerNodeId, nowMs, out _)) { return; @@ -92,7 +92,7 @@ public async Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken c { cancellationToken.ThrowIfCancellationRequested(); - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; if (TryGetPeerKeyEntry(peerNodeId, nowMs, out var existing)) { if (existing.Key is not null) @@ -174,7 +174,7 @@ public async ValueTask HandleHelloAsync( return; } - CachePeerKey(peerNodeId, sharedKey, nowMs: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + CachePeerKey(peerNodeId, sharedKey, nowMs: Environment.TickCount64); var peerProtocolVersion = payload[0]; ObservePeerProtocolVersion(peerNodeId, peerProtocolVersion); @@ -211,7 +211,7 @@ private Task StartPeerKeyFetch(NodeId peerNodeId) private async Task FetchAndCachePeerKeyAsync(NodeId peerNodeId) { - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; try { var identity = await _rootClient @@ -323,6 +323,7 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) if (entry.ExpiresAtUnixMs <= nowMs) { _peerKeys.TryRemove(peerNodeId, out _); + _peerProtocolVersions.TryRemove(peerNodeId, out _); } } @@ -341,6 +342,7 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) } _peerKeys.TryRemove(peerNodeId, out _); + _peerProtocolVersions.TryRemove(peerNodeId, out _); toRemove--; } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs b/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs index 25589a7..4803846 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs @@ -18,7 +18,7 @@ public ZeroTierExternalSurfaceAddressTracker(TimeSpan ttl, Func? nowUnixMs } _ttl = ttl; - _nowUnixMs = nowUnixMs ?? (() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } public void Observe(NodeId reportingPeerNodeId, int localSocketId, IPEndPoint surfaceAddress) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs index 6b2722a..f8700d9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs @@ -44,7 +44,7 @@ public static async Task HelloRootsAsync( var rootKeys = ZeroTierRootKeyDerivation.BuildRootKeys(localIdentity, planet); - var helloTimestamp = (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var helloTimestamp = (ulong)Environment.TickCount64; var pending = new Dictionary(capacity: planet.Roots.Count); foreach (var root in planet.Roots) @@ -162,7 +162,7 @@ public static async Task HelloAsync( throw new InvalidOperationException("Local identity must contain a private key."); } - var helloTimestamp = (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var helloTimestamp = (ulong)Environment.TickCount64; var packet = ZeroTierHelloPacketBuilder.BuildPacket( localIdentity, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs index ea952f5..7c1e618 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs @@ -10,6 +10,8 @@ internal sealed class ZeroTierPeerEchoManager { private const long EchoMinIntervalMs = 5_000; private const long PendingEchoTtlMs = 30_000; + private const long EchoPathCacheTtlMs = 120_000; + private const long RttCacheTtlMs = 120_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _localNodeId; @@ -18,8 +20,9 @@ internal sealed class ZeroTierPeerEchoManager private readonly ConcurrentDictionary _lastEchoSentUnixMs = new(); private readonly ConcurrentDictionary _pendingByPacketId = new(); - private readonly ConcurrentDictionary _lastRttMsByPath = new(); + private readonly ConcurrentDictionary _lastRttMsByPath = new(); private long _lastPendingCleanupUnixMs; + private long _lastCacheCleanupMs; public ZeroTierPeerEchoManager( IZeroTierUdpTransport udp, @@ -33,14 +36,21 @@ public ZeroTierPeerEchoManager( _udp = udp; _localNodeId = localNodeId; _getPeerProtocolVersion = getPeerProtocolVersion; - _nowUnixMs = nowUnixMs ?? (() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } public bool TryGetLastRttMs(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, out int rttMs) { ArgumentNullException.ThrowIfNull(remoteEndPoint); var key = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); - return _lastRttMsByPath.TryGetValue(key, out rttMs); + if (_lastRttMsByPath.TryGetValue(key, out var entry)) + { + rttMs = entry.RttMs; + return true; + } + + rttMs = 0; + return false; } public void ObserveHelloOkRtt(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, ulong helloTimestampEcho) @@ -55,7 +65,7 @@ public void ObserveHelloOkRtt(NodeId peerNodeId, int localSocketId, IPEndPoint r } var key = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); - _lastRttMsByPath[key] = (int)rtt; + _lastRttMsByPath[key] = new RttEntry((int)rtt, LastUpdatedMs: now); } public async ValueTask TrySendEchoProbeAsync( @@ -71,6 +81,7 @@ public async ValueTask TrySendEchoProbeAsync( var now = _nowUnixMs(); CleanupPendingIfNeeded(now); + CleanupCachesIfNeeded(now); var pathKey = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); if (_lastEchoSentUnixMs.TryGetValue(pathKey, out var lastSent) && unchecked(now - lastSent) < EchoMinIntervalMs) @@ -184,7 +195,7 @@ public void HandleEchoOk( return; } - _lastRttMsByPath[pending.PathKey] = (int)rtt; + _lastRttMsByPath[pending.PathKey] = new RttEntry((int)rtt, LastUpdatedMs: now); } private void CleanupPendingIfNeeded(long nowUnixMs) @@ -210,7 +221,41 @@ private void CleanupPendingIfNeeded(long nowUnixMs) } } + private void CleanupCachesIfNeeded(long nowMs) + { + var last = Volatile.Read(ref _lastCacheCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 10_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastCacheCleanupMs, nowMs, last) != last) + { + return; + } + + var echoCutoff = nowMs - EchoPathCacheTtlMs; + foreach (var pair in _lastEchoSentUnixMs) + { + if (pair.Value <= echoCutoff) + { + _lastEchoSentUnixMs.TryRemove(pair.Key, out _); + } + } + + var rttCutoff = nowMs - RttCacheTtlMs; + foreach (var pair in _lastRttMsByPath) + { + if (pair.Value.LastUpdatedMs <= rttCutoff) + { + _lastRttMsByPath.TryRemove(pair.Key, out _); + } + } + } + private readonly record struct PendingEcho(ZeroTierPeerEchoPathKey PathKey, long TimestampUnixMs); + + private readonly record struct RttEntry(int RttMs, long LastUpdatedMs); } internal readonly record struct ZeroTierPeerEchoPathKey(NodeId PeerNodeId, ZeroTierPeerPhysicalPathKey Path); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs index 414c54b..13d0803 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs @@ -18,7 +18,7 @@ public ZeroTierPeerPhysicalPathTracker(TimeSpan ttl, Func? nowUnixMs = nul } _ttl = ttl; - _nowUnixMs = nowUnixMs ?? (() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } public void ObserveHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) From 2b494ca78b0861e59eff5d52f945c705e867af80 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:02:38 +0100 Subject: [PATCH 016/296] fix(multipath): expire per-peer path state and managers --- .../Internal/ZeroTierDataplaneRuntime.cs | 34 ++++++++++++++++- .../Internal/ZeroTierPeerBondPolicyEngine.cs | 17 ++++++++- .../ZeroTierPeerPathNegotiationManager.cs | 29 +++++++++++++++ .../Internal/ZeroTierPeerQosManager.cs | 37 +++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c7936a1..a667986 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -13,6 +13,8 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable { + private const long DirectEndpointManagerTtlMs = 600_000; + private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; private readonly IPEndPoint _rootEndpoint; @@ -26,6 +28,8 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly IPAddress[] _localManagedIpsV6; private readonly ZeroTierMac _localMac; private readonly ConcurrentDictionary _directEndpoints = new(); + private readonly ConcurrentDictionary _directEndpointLastUsedMs = new(); + private long _lastDirectEndpointCleanupMs; private readonly ZeroTierPeerPhysicalPathTracker _peerPaths; private readonly ZeroTierPeerEchoManager _peerEcho; private readonly ZeroTierExternalSurfaceAddressTracker _surfaceAddresses; @@ -541,7 +545,9 @@ private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationT private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellationToken) { + var nowMs = Environment.TickCount64; _bondEngine.MaintenanceTick(); + CleanupDirectEndpointManagers(nowMs); var peers = _peerPaths.GetPeersSnapshot(); if (peers.Length == 0) @@ -790,6 +796,32 @@ private ValueTask HandlePeerControlPacketAsync( } private ZeroTierDirectEndpointManager GetOrCreateDirectEndpointManager(NodeId peerNodeId) - => _directEndpoints.GetOrAdd(peerNodeId, id => new ZeroTierDirectEndpointManager(_udp, _rootEndpoint, id)); + { + _directEndpointLastUsedMs[peerNodeId] = Environment.TickCount64; + return _directEndpoints.GetOrAdd(peerNodeId, id => new ZeroTierDirectEndpointManager(_udp, _rootEndpoint, id)); + } + + private void CleanupDirectEndpointManagers(long nowMs) + { + var last = Volatile.Read(ref _lastDirectEndpointCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 10_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastDirectEndpointCleanupMs, nowMs, last) != last) + { + return; + } + + foreach (var pair in _directEndpointLastUsedMs) + { + if (unchecked(nowMs - pair.Value) > DirectEndpointManagerTtlMs) + { + _directEndpointLastUsedMs.TryRemove(pair.Key, out _); + _directEndpoints.TryRemove(pair.Key, out _); + } + } + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs index 9fcf49e..6ab257b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs @@ -11,6 +11,7 @@ internal sealed class ZeroTierPeerBondPolicyEngine private const int AwareLatencySlackMs = 25; private const long ActiveBackupMinHoldMs = 10_000; private const int ActiveBackupLatencySlackMs = 25; + private const long PeerStateTtlMs = 600_000; private readonly Func _getLatencyMs; private readonly Func _getRemoteUtility; @@ -56,6 +57,7 @@ public bool TrySelectSinglePath( case ZeroTierBondPolicy.BalanceRoundRobin: StableSort(observedPaths); var state = _peerStates.GetOrAdd(peerNodeId, static _ => new PeerState()); + Volatile.Write(ref state.LastUsedMs, _nowMs()); var rr = Interlocked.Increment(ref state.RoundRobinCounter); return SelectByIndex(observedPaths, index: (int)((uint)rr % (uint)observedPaths.Length), out selected); @@ -76,9 +78,19 @@ public bool TrySelectSinglePath( public void MaintenanceTick() { var now = _nowMs(); - foreach (var state in _peerStates.Values) + foreach (var pair in _peerStates) { + var state = pair.Value; CleanupFlowsIfNeeded(state, now); + + var lastUsed = Volatile.Read(ref state.LastUsedMs); + if (lastUsed != 0 && + unchecked(now - lastUsed) > PeerStateTtlMs && + state.Flows.IsEmpty && + state.ActiveBackupPath is null) + { + _peerStates.TryRemove(pair.Key, out _); + } } } @@ -172,6 +184,7 @@ private bool SelectActiveBackup(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] ob { var now = _nowMs(); var state = _peerStates.GetOrAdd(peerNodeId, static _ => new PeerState()); + Volatile.Write(ref state.LastUsedMs, now); lock (state.Gate) { @@ -261,6 +274,7 @@ private bool SelectBalanceAware( { var now = _nowMs(); var state = _peerStates.GetOrAdd(peerNodeId, static _ => new PeerState()); + Volatile.Write(ref state.LastUsedMs, now); CleanupFlowsIfNeeded(state, now); @@ -357,6 +371,7 @@ private sealed class PeerState public long LastFlowCleanupMs; public ZeroTierPeerPhysicalPathKey? ActiveBackupPath; public long ActiveBackupSelectedAtMs; + public long LastUsedMs; } private readonly record struct FlowAssignment(ZeroTierPeerPhysicalPathKey Path, long LastUsedMs); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs index 63d9760..9b8f7be 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs @@ -7,9 +7,12 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierPeerPathNegotiationManager { private const long MinSendIntervalMs = 10_000; + private const long NegotiationStateTtlMs = 300_000; + private const int MaxNegotiationStates = 4096; private readonly Func _nowMs; private readonly ConcurrentDictionary _state = new(); + private long _lastCleanupMs; public ZeroTierPeerPathNegotiationManager(Func? nowMs = null) { @@ -21,6 +24,7 @@ public void HandleInboundRequest(NodeId peerNodeId, int localSocketId, IPEndPoin ArgumentNullException.ThrowIfNull(remoteEndPoint); var now = _nowMs(); + CleanupIfNeeded(now); var key = new ZeroTierPeerNegotiationPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); _state.AddOrUpdate( key, @@ -48,6 +52,7 @@ public bool TryMarkSent(NodeId peerNodeId, int localSocketId, IPEndPoint remoteE ArgumentNullException.ThrowIfNull(remoteEndPoint); var now = _nowMs(); + CleanupIfNeeded(now); var key = new ZeroTierPeerNegotiationPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); while (true) @@ -74,6 +79,30 @@ public bool TryMarkSent(NodeId peerNodeId, int localSocketId, IPEndPoint remoteE } } + private void CleanupIfNeeded(long now) + { + var last = Volatile.Read(ref _lastCleanupMs); + if (last != 0 && unchecked(now - last) < 10_000 && _state.Count <= MaxNegotiationStates) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastCleanupMs, now, last) != last) + { + return; + } + + foreach (var pair in _state) + { + var state = pair.Value; + var touched = Math.Max(state.LastReceivedMs, state.LastSentMs); + if (touched != 0 && unchecked(now - touched) > NegotiationStateTtlMs) + { + _state.TryRemove(pair.Key, out _); + } + } + } + private readonly record struct NegotiationState(short RemoteUtility, long LastReceivedMs, long LastSentMs); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs index 3a00cd9..f60a60d 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs @@ -20,9 +20,12 @@ internal sealed class ZeroTierPeerQosManager // See: ZeroTierOne node/Bond.cpp (qosStatsOut timeout = _qosSendInterval * 3). private const long DefaultRecordTimeoutMs = 30_000; + private const long PathStateTtlMs = 300_000; + private const int MaxPathStates = 4096; private readonly Func _nowMs; private readonly ConcurrentDictionary _paths = new(); + private long _lastPathCleanupMs; public ZeroTierPeerQosManager(Func? nowMs = null) { @@ -41,6 +44,8 @@ public void RecordIncomingPacket(NodeId peerNodeId, int localSocketId, IPEndPoin var now = _nowMs(); var key = new ZeroTierPeerQosPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); var state = _paths.GetOrAdd(key, static _ => new PathState()); + Volatile.Write(ref state.LastTouchedMs, now); + CleanupPathsIfNeeded(now); if (Volatile.Read(ref state.InboundCount) >= MaxPendingRecords) { @@ -63,6 +68,8 @@ public void RecordOutgoingPacket(NodeId peerNodeId, int localSocketId, IPEndPoin var now = _nowMs(); var key = new ZeroTierPeerQosPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); var state = _paths.GetOrAdd(key, static _ => new PathState()); + Volatile.Write(ref state.LastTouchedMs, now); + CleanupPathsIfNeeded(now); CleanupOutboundIfNeeded(state, now); @@ -169,6 +176,9 @@ public void HandleInboundMeasurement( return; } + Volatile.Write(ref state.LastTouchedMs, now); + CleanupPathsIfNeeded(now); + CleanupOutboundIfNeeded(state, now); var count = 0; @@ -248,11 +258,38 @@ private static void CleanupOutboundIfNeeded(PathState state, long now) } } + private void CleanupPathsIfNeeded(long now) + { + var last = Volatile.Read(ref _lastPathCleanupMs); + if (last != 0 && unchecked(now - last) < 10_000 && _paths.Count <= MaxPathStates) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastPathCleanupMs, now, last) != last) + { + return; + } + + foreach (var pair in _paths) + { + var state = pair.Value; + var touched = Volatile.Read(ref state.LastTouchedMs); + if (touched != 0 && unchecked(now - touched) > PathStateTtlMs && + Volatile.Read(ref state.InboundCount) <= 0 && + state.OutboundSentMs.IsEmpty) + { + _paths.TryRemove(pair.Key, out _); + } + } + } + private sealed class PathState { public ConcurrentQueue Inbound { get; } = new(); public int InboundCount; public long LastSentMs; + public long LastTouchedMs; public ConcurrentDictionary OutboundSentMs { get; } = new(); public long LastOutboundCleanupMs; From fc814bf4ecfd8da88498f7a88fa41993a1887ea9 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:14:46 +0100 Subject: [PATCH 017/296] fix(tests): skip junction traversal test explicitly --- ZTSharp.Tests/FileStateStoreSecurityTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/FileStateStoreSecurityTests.cs b/ZTSharp.Tests/FileStateStoreSecurityTests.cs index 3ad7aaa..d197606 100644 --- a/ZTSharp.Tests/FileStateStoreSecurityTests.cs +++ b/ZTSharp.Tests/FileStateStoreSecurityTests.cs @@ -10,7 +10,7 @@ public async Task ReadAsync_Throws_WhenPathTraversesJunction() { if (!OperatingSystem.IsWindows()) { - return; + throw Xunit.Sdk.SkipException.ForSkip("Junction traversal tests require Windows."); } var root = TestTempPaths.CreateGuidSuffixed("zt-state-root-"); @@ -25,7 +25,7 @@ public async Task ReadAsync_Throws_WhenPathTraversesJunction() var junction = Path.Combine(root, "escape"); if (!TryCreateJunction(junction, outside)) { - return; + throw Xunit.Sdk.SkipException.ForSkip("Failed to create junction (insufficient privileges or mklink unavailable)."); } var store = new FileStateStore(root); From ac38b3ad70954037cde13d845b60db0e96b84bec Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:19:13 +0100 Subject: [PATCH 018/296] fix(transport): avoid IPv4 regression on dual-mode failure --- ZTSharp.Tests/OsUdpSocketFactoryTests.cs | 8 ++++---- ZTSharp/Transport/Internal/OsUdpSocketFactory.cs | 14 +++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs index f837c2a..6082603 100644 --- a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs +++ b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs @@ -10,7 +10,7 @@ public void WindowsSioUdpConnResetInput_IsDword() { if (!OperatingSystem.IsWindows()) { - return; + throw Xunit.Sdk.SkipException.ForSkip("Windows-only IOControl buffer test."); } var buffer = OsUdpSocketFactory.CreateWindowsSioUdpConnResetInputBuffer(disableConnReset: true); @@ -19,7 +19,7 @@ public void WindowsSioUdpConnResetInput_IsDword() } [Fact] - public void CreateSocketCore_WhenDualModeFails_TriesIpv6OnlyBeforeIpv4() + public void CreateSocketCore_WhenDualModeFails_TriesIpv4BeforeIpv6Only() { var calls = new List(); @@ -36,8 +36,8 @@ public void CreateSocketCore_WhenDualModeFails_TriesIpv6OnlyBeforeIpv4() try { - Assert.Equal(new[] { "dual", "v6only" }, calls); - Assert.Equal(AddressFamily.InterNetworkV6, socket.Client.AddressFamily); + Assert.Equal(new[] { "dual", "v4" }, calls); + Assert.Equal(AddressFamily.InterNetwork, socket.Client.AddressFamily); } finally { diff --git a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs index b04de76..332b435 100644 --- a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs +++ b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs @@ -41,12 +41,23 @@ internal static UdpClient CreateSocketCore( return createUdp4Bound(localPort); } + Exception? lastException = null; try { return createUdp6DualModeBound(localPort); } catch (Exception ex) when (ex is SocketException or PlatformNotSupportedException or NotSupportedException) { + lastException = ex; + } + + try + { + return createUdp4Bound(localPort); + } + catch (Exception ex) when (ex is SocketException or PlatformNotSupportedException or NotSupportedException) + { + lastException = ex; } try @@ -55,9 +66,10 @@ internal static UdpClient CreateSocketCore( } catch (Exception ex) when (ex is SocketException or PlatformNotSupportedException or NotSupportedException) { + lastException = ex; } - return createUdp4Bound(localPort); + throw lastException!; } private static UdpClient CreateUdp4Bound(int localPort) From 58b2925ae9ff11189541d9f78d6647215484854e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:52:04 +0100 Subject: [PATCH 019/296] Fix bond policy parsing without ToLowerInvariant --- samples/ZTSharp.Cli/CliParsing.cs | 48 ++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/samples/ZTSharp.Cli/CliParsing.cs b/samples/ZTSharp.Cli/CliParsing.cs index 1694b59..611446e 100644 --- a/samples/ZTSharp.Cli/CliParsing.cs +++ b/samples/ZTSharp.Cli/CliParsing.cs @@ -187,17 +187,45 @@ public static ZeroTierBondPolicy ParseBondPolicy(string value) throw new InvalidOperationException("Invalid bond policy."); } - value = value.Trim().ToLowerInvariant(); - return value switch + var normalized = value.Trim(); + if (normalized.Equals("off", StringComparison.OrdinalIgnoreCase)) { - "off" => ZeroTierBondPolicy.Off, - "active-backup" or "activebackup" => ZeroTierBondPolicy.ActiveBackup, - "broadcast" => ZeroTierBondPolicy.Broadcast, - "balance-rr" or "balance-roundrobin" or "balance-round-robin" or "roundrobin" or "rr" => ZeroTierBondPolicy.BalanceRoundRobin, - "balance-xor" or "xor" => ZeroTierBondPolicy.BalanceXor, - "balance-aware" or "aware" => ZeroTierBondPolicy.BalanceAware, - _ => throw new InvalidOperationException("Invalid bond policy (expected off|active-backup|broadcast|balance-rr|balance-xor|balance-aware).") - }; + return ZeroTierBondPolicy.Off; + } + + if (normalized.Equals("active-backup", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("activebackup", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.ActiveBackup; + } + + if (normalized.Equals("broadcast", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.Broadcast; + } + + if (normalized.Equals("balance-rr", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("balance-roundrobin", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("balance-round-robin", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("roundrobin", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("rr", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.BalanceRoundRobin; + } + + if (normalized.Equals("balance-xor", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("xor", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.BalanceXor; + } + + if (normalized.Equals("balance-aware", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("aware", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.BalanceAware; + } + + throw new InvalidOperationException("Invalid bond policy (expected off|active-backup|broadcast|balance-rr|balance-xor|balance-aware)."); } public static IReadOnlyList ParsePortList(string value, string name) From d4588849e562dccfdca1c33f370d47fee7a3d203 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:57:08 +0100 Subject: [PATCH 020/296] Harden OS UDP peer registry and discovery receive loop --- .../Transport/Internal/OsUdpPeerRegistry.cs | 95 ++++++-- .../Transport/Internal/OsUdpReceiveLoop.cs | 212 ++++++++++++------ ZTSharp/Transport/OsUdpNodeTransport.cs | 2 +- 3 files changed, 218 insertions(+), 91 deletions(-) diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 5d527fd..9de6dd5 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -7,20 +7,20 @@ namespace ZTSharp.Transport.Internal; internal sealed class OsUdpPeerRegistry { - private readonly record struct PeerDirectoryEntry(IPEndPoint Endpoint, long LastSeenTicks); + internal readonly record struct PeerEntry(IPEndPoint Endpoint, long LastSeenTicks); private const int DirectoryMaxNetworks = 256; internal const int DirectoryMaxPeersPerNetwork = 1024; internal static readonly TimeSpan DirectoryPeerTtl = TimeSpan.FromMinutes(5); private static readonly long DirectoryPeerTtlTicks = DirectoryPeerTtl.Ticks; - private static readonly ConcurrentDictionary> s_networkDirectory = new(); + private static readonly ConcurrentDictionary> s_networkDirectory = new(); private readonly bool _enablePeerDiscovery; private readonly Func _normalizeEndpoint; private readonly TimeProvider _timeProvider; - private readonly ConcurrentDictionary> _networkPeers = new(); + private readonly ConcurrentDictionary> _networkPeers = new(); private readonly ConcurrentDictionary _localNodeIds = new(); public OsUdpPeerRegistry(bool enablePeerDiscovery, Func normalizeEndpoint, TimeProvider? timeProvider = null) @@ -50,17 +50,27 @@ public bool TryRemoveLocalNodeIdIfMatch(ulong networkId, ulong expectedNodeId) return true; } - public bool TryGetPeers(ulong networkId, out ConcurrentDictionary peers) - => _networkPeers.TryGetValue(networkId, out peers!); + public bool TryGetPeers(ulong networkId, out ConcurrentDictionary peers) + { + if (!_networkPeers.TryGetValue(networkId, out peers!)) + { + return false; + } + + EvictExpiredAndTrimNetworkPeers(networkId, peers, GetNowTicks()); + return true; + } public void RemoveNetworkPeers(ulong networkId) => _networkPeers.TryRemove(networkId, out _); public void AddOrUpdatePeer(ulong networkId, ulong nodeId, IPEndPoint endpoint) { + var nowTicks = GetNowTicks(); var normalized = _normalizeEndpoint(endpoint); - var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - peers[nodeId] = normalized; + var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + peers[nodeId] = new PeerEntry(normalized, nowTicks); + SweepNetworkPeers(nowTicks); } public IEnumerable> RegisterLocalAndGetKnownPeers( @@ -77,11 +87,11 @@ public IEnumerable> RegisterLocalAndGetKnownPeer var nowTicks = GetNowTicks(); var normalizedAdvertisedEndpoint = _normalizeEndpoint(advertisedEndpoint); - var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - discoveredPeers[localNodeId] = new PeerDirectoryEntry(normalizedAdvertisedEndpoint, nowTicks); + var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks); SweepDirectory(nowTicks); - var localPeers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + var localPeers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); foreach (var peer in discoveredPeers) { if (peer.Key == localNodeId) @@ -89,7 +99,7 @@ public IEnumerable> RegisterLocalAndGetKnownPeer continue; } - localPeers[peer.Key] = peer.Value.Endpoint; + localPeers[peer.Key] = new PeerEntry(peer.Value.Endpoint, nowTicks); } return discoveredPeers @@ -112,8 +122,8 @@ public void RefreshLocalRegistration(ulong networkId, ulong localNodeId, IPEndPo var nowTicks = GetNowTicks(); var normalizedAdvertisedEndpoint = _normalizeEndpoint(advertisedEndpoint); - var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - discoveredPeers[localNodeId] = new PeerDirectoryEntry(normalizedAdvertisedEndpoint, nowTicks); + var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks); SweepDirectory(nowTicks); } @@ -126,11 +136,12 @@ public void RegisterDiscoveredPeer(ulong networkId, ulong sourceNodeId, IPEndPoi var nowTicks = GetNowTicks(); var endpoint = _normalizeEndpoint(remoteEndpoint); - var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - peers[sourceNodeId] = endpoint; - var directoryPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - directoryPeers[sourceNodeId] = new PeerDirectoryEntry(endpoint, nowTicks); + var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + peers[sourceNodeId] = new PeerEntry(endpoint, nowTicks); + var directoryPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + directoryPeers[sourceNodeId] = new PeerEntry(endpoint, nowTicks); SweepDirectory(nowTicks); + SweepNetworkPeers(nowTicks); } public void Cleanup() @@ -142,6 +153,7 @@ public void Cleanup() } SweepDirectory(nowTicks); + SweepNetworkPeers(nowTicks); _networkPeers.Clear(); _localNodeIds.Clear(); } @@ -161,7 +173,7 @@ private static void RemoveFromDirectory(ulong networkId, ulong nodeId) private long GetNowTicks() => _timeProvider.GetUtcNow().UtcDateTime.Ticks; - private static void EvictExpiredAndTrimNetwork(ulong networkId, ConcurrentDictionary discoveredPeers, long nowTicks) + private static void EvictExpiredAndTrimNetwork(ulong networkId, ConcurrentDictionary discoveredPeers, long nowTicks) { var cutoffTicks = nowTicks - DirectoryPeerTtlTicks; foreach (var peer in discoveredPeers) @@ -193,6 +205,53 @@ private static void EvictExpiredAndTrimNetwork(ulong networkId, ConcurrentDictio } } + private void EvictExpiredAndTrimNetworkPeers(ulong networkId, ConcurrentDictionary peers, long nowTicks) + { + EvictExpiredAndTrimNetwork(networkId, peers, nowTicks); + if (peers.IsEmpty) + { + _networkPeers.TryRemove(networkId, out _); + } + } + + private void SweepNetworkPeers(long nowTicks) + { + foreach (var network in _networkPeers) + { + EvictExpiredAndTrimNetworkPeers(network.Key, network.Value, nowTicks); + } + + if (_networkPeers.Count <= DirectoryMaxNetworks) + { + return; + } + + var overflow = _networkPeers.Count - DirectoryMaxNetworks; + var toRemove = _networkPeers + .Select(network => + { + var lastSeen = 0L; + foreach (var peer in network.Value.Values) + { + if (peer.LastSeenTicks > lastSeen) + { + lastSeen = peer.LastSeenTicks; + } + } + + return (NetworkId: network.Key, LastSeenTicks: lastSeen); + }) + .OrderBy(network => network.LastSeenTicks) + .Take(overflow) + .Select(network => network.NetworkId) + .ToArray(); + + foreach (var networkId in toRemove) + { + _networkPeers.TryRemove(networkId, out _); + } + } + private static void SweepDirectory(long nowTicks) { foreach (var network in s_networkDirectory) diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index 7975469..cf0c407 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -1,14 +1,23 @@ using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Sockets; +using System.Threading.Channels; namespace ZTSharp.Transport.Internal; internal sealed class OsUdpReceiveLoop { private const int MaxHelloResponseCacheEntries = 4096; + private const int MaxPendingDiscoverySends = 256; private const long PeerHelloResponseMinIntervalMs = 1000; + private readonly record struct DiscoverySendRequest( + ulong NetworkId, + ulong LocalNodeId, + IPEndPoint Endpoint, + OsUdpPeerDiscoveryProtocol.FrameType FrameType); + private readonly UdpClient _udp; private readonly Func> _receiveAsync; private readonly bool _enablePeerDiscovery; @@ -17,6 +26,9 @@ internal sealed class OsUdpReceiveLoop private readonly Func _sendDiscoveryFrameAsync; private readonly Action? _log; private readonly Dictionary<(ulong NetworkId, ulong NodeId), long> _helloResponseLastSentMs = new(); + private readonly Channel _discoverySendQueue; + + internal Action? DatagramReceivedForTests { get; set; } public OsUdpReceiveLoop( UdpClient udp, @@ -39,102 +51,120 @@ public OsUdpReceiveLoop( _dispatchFrameAsync = dispatchFrameAsync; _sendDiscoveryFrameAsync = sendDiscoveryFrameAsync; _log = log; + + _discoverySendQueue = Channel.CreateBounded(new BoundedChannelOptions(MaxPendingDiscoverySends) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = true, + SingleWriter = true + }); } public async Task RunAsync(CancellationToken cancellationToken) { - while (!cancellationToken.IsCancellationRequested) + var discoverySendLoop = _enablePeerDiscovery + ? RunDiscoverySendLoopAsync(cancellationToken) + : Task.CompletedTask; + + try { - UdpReceiveResult result; - try - { - result = await _receiveAsync(cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) + while (!cancellationToken.IsCancellationRequested) { - return; - } - catch (ObjectDisposedException) - { - return; - } - catch (SocketException ex) when (ex.SocketErrorCode == SocketError.ConnectionReset) - { - continue; - } - catch (SocketException ex) - { - _log?.Invoke($"OS UDP receive failed (SocketException {ex.SocketErrorCode}: {ex.Message})."); - continue; - } - catch (InvalidOperationException ex) - { - _log?.Invoke($"OS UDP receive failed (InvalidOperationException: {ex.Message})."); - continue; - } + UdpReceiveResult result; + try + { + result = await _receiveAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + catch (ObjectDisposedException) + { + return; + } + catch (SocketException ex) when (ex.SocketErrorCode == SocketError.ConnectionReset) + { + continue; + } + catch (SocketException ex) + { + _log?.Invoke($"OS UDP receive failed (SocketException {ex.SocketErrorCode}: {ex.Message})."); + continue; + } + catch (InvalidOperationException ex) + { + _log?.Invoke($"OS UDP receive failed (InvalidOperationException: {ex.Message})."); + continue; + } - if (!NodeFrameCodec.TryDecode(result.Buffer.AsMemory(), out var networkId, out var sourceNodeId, out var payload)) - { - continue; - } + var normalizedRemoteEndpoint = UdpEndpointNormalization.Normalize(result.RemoteEndPoint); + DatagramReceivedForTests?.Invoke(normalizedRemoteEndpoint); - var normalizedRemoteEndpoint = UdpEndpointNormalization.Normalize(result.RemoteEndPoint); - if (_enablePeerDiscovery && OsUdpPeerDiscoveryProtocol.TryParsePayload(payload.Span, networkId, out var controlFrameType, out var discoveredNodeId)) - { - if (discoveredNodeId != 0 && discoveredNodeId == sourceNodeId) + if (!NodeFrameCodec.TryDecode(result.Buffer.AsMemory(), out var networkId, out var sourceNodeId, out var payload)) + { + continue; + } + + if (_enablePeerDiscovery && + _peers.TryGetLocalNodeId(networkId, out var localNodeId) && + localNodeId != 0 && + OsUdpPeerDiscoveryProtocol.TryParsePayload(payload.Span, networkId, out var controlFrameType, out var discoveredNodeId)) { - _peers.RegisterDiscoveredPeer(networkId, discoveredNodeId, normalizedRemoteEndpoint); - if (_peers.TryGetLocalNodeId(networkId, out var localNodeId) && localNodeId != discoveredNodeId) + if (discoveredNodeId != 0 && discoveredNodeId == sourceNodeId) { - if (controlFrameType == OsUdpPeerDiscoveryProtocol.FrameType.PeerHello) + _peers.RegisterDiscoveredPeer(networkId, discoveredNodeId, normalizedRemoteEndpoint); + if (localNodeId != discoveredNodeId && + controlFrameType == OsUdpPeerDiscoveryProtocol.FrameType.PeerHello && + ShouldSendHelloResponse(networkId, discoveredNodeId)) { - if (ShouldSendHelloResponse(networkId, discoveredNodeId)) - { - _ = SendDiscoveryFrameSafeAsync( + QueueDiscoverySend( networkId, localNodeId, normalizedRemoteEndpoint, - OsUdpPeerDiscoveryProtocol.FrameType.PeerHelloResponse, - cancellationToken); - } + OsUdpPeerDiscoveryProtocol.FrameType.PeerHelloResponse); } } + + continue; } - continue; - } + if (sourceNodeId == 0 || + !_peers.TryGetPeers(networkId, out var peers) || + !peers.TryGetValue(sourceNodeId, out var expectedEntry)) + { + continue; + } - if (sourceNodeId == 0 || !_peers.TryGetPeers(networkId, out var peers) || !peers.TryGetValue(sourceNodeId, out var expectedEndpoint)) - { - continue; - } + var expectedEndpoint = expectedEntry.Endpoint; + if (!expectedEndpoint.Equals(normalizedRemoteEndpoint)) + { + continue; + } - if (!expectedEndpoint.Equals(normalizedRemoteEndpoint)) - { - // Allow NAT rebinding / port changes without opening too wide a spoofing hole: - // - same IP, different port: accept and update endpoint - // - different IP: require a discovery control frame (handled above) before accepting - if (expectedEndpoint.Address.Equals(normalizedRemoteEndpoint.Address)) + try { - _peers.AddOrUpdatePeer(networkId, sourceNodeId, normalizedRemoteEndpoint); + await _dispatchFrameAsync(sourceNodeId, networkId, payload, cancellationToken).ConfigureAwait(false); } - else + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } +#pragma warning disable CA1031 // Receive loop must survive dispatch failures. + catch (Exception) +#pragma warning restore CA1031 { - continue; } } - + } + finally + { + _discoverySendQueue.Writer.TryComplete(); try { - await _dispatchFrameAsync(sourceNodeId, networkId, payload, cancellationToken).ConfigureAwait(false); + await discoverySendLoop.ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - return; - } -#pragma warning disable CA1031 // Receive loop must survive dispatch failures. - catch (Exception) -#pragma warning restore CA1031 { } } @@ -142,11 +172,6 @@ public async Task RunAsync(CancellationToken cancellationToken) private bool ShouldSendHelloResponse(ulong networkId, ulong remoteNodeId) { - if (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) - { - _helloResponseLastSentMs.Clear(); - } - var key = (networkId, remoteNodeId); var nowMs = Environment.TickCount64; if (_helloResponseLastSentMs.TryGetValue(key, out var lastMs)) @@ -158,9 +183,52 @@ private bool ShouldSendHelloResponse(ulong networkId, ulong remoteNodeId) } _helloResponseLastSentMs[key] = nowMs; + + if (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) + { + var overflow = _helloResponseLastSentMs.Count - MaxHelloResponseCacheEntries; + var toRemove = _helloResponseLastSentMs + .OrderBy(pair => pair.Value) + .Take(overflow) + .Select(pair => pair.Key) + .ToArray(); + + for (var i = 0; i < toRemove.Length; i++) + { + _helloResponseLastSentMs.Remove(toRemove[i]); + } + } + return true; } + private void QueueDiscoverySend( + ulong networkId, + ulong localNodeId, + IPEndPoint endpoint, + OsUdpPeerDiscoveryProtocol.FrameType frameType) + => _discoverySendQueue.Writer.TryWrite(new DiscoverySendRequest(networkId, localNodeId, endpoint, frameType)); + + private async Task RunDiscoverySendLoopAsync(CancellationToken cancellationToken) + { + try + { + await foreach (var request in _discoverySendQueue.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + await SendDiscoveryFrameSafeAsync( + request.NetworkId, + request.LocalNodeId, + request.Endpoint, + request.FrameType, + cancellationToken) + .ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + } + private async Task SendDiscoveryFrameSafeAsync( ulong networkId, ulong localNodeId, diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index a76efbb..af6c826 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -159,7 +159,7 @@ public async Task SendFrameAsync( try { await _udp - .SendAsync(frame, peer.Value, cancellationToken) + .SendAsync(frame, peer.Value.Endpoint, cancellationToken) .ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) From dcee95574512e22c9c6d1217d46142baf68e5a69 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:58:43 +0100 Subject: [PATCH 021/296] Harden FileStateStore reparse traversal and max read size --- ZTSharp/FileStateStore.cs | 90 +++++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 8 deletions(-) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index f5b7231..4555128 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -22,10 +22,7 @@ public FileStateStore(string rootPath) _pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; Directory.CreateDirectory(_rootPath); - if (IsReparsePoint(_rootPathTrimmed)) - { - throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); - } + ThrowIfRootPathOrAncestorsAreReparsePoints(); } public Task ExistsAsync(string key, CancellationToken cancellationToken = default) @@ -257,8 +254,36 @@ private static async Task ReadAllBytesWithSharingAsync(string path, Canc throw new IOException($"State file exceeds maximum supported size of {MaxReadBytes} bytes."); } - using var memory = stream.Length <= int.MaxValue ? new MemoryStream((int)stream.Length) : new MemoryStream(); - await stream.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); + var initialCapacity = stream.Length <= int.MaxValue + ? (int)Math.Min(stream.Length, MaxReadBytes) + : 0; + + using var memory = initialCapacity > 0 ? new MemoryStream(initialCapacity) : new MemoryStream(); + var buffer = new byte[16 * 1024]; + long totalRead = 0; + while (true) + { + var read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (read <= 0) + { + break; + } + + var remaining = MaxReadBytes - totalRead; + if (read > remaining) + { + if (remaining > 0) + { + memory.Write(buffer, 0, (int)remaining); + } + + throw new IOException($"State file exceeds maximum supported size of {MaxReadBytes} bytes."); + } + + memory.Write(buffer, 0, read); + totalRead += read; + } + return memory.ToArray(); } @@ -327,18 +352,49 @@ private void ThrowIfPathTraversesReparsePoint(string fullPath) for (var i = 0; i < parts.Length; i++) { current = Path.Combine(current, parts[i]); - if (!File.Exists(current) && !Directory.Exists(current)) + if (!TryGetAttributes(current, out var attributes)) { return; } - if (IsReparsePoint(current)) + if ((attributes & FileAttributes.ReparsePoint) != 0) { throw new InvalidOperationException("State path traversal via symlink/junction/reparse point is not allowed."); } } } + private void ThrowIfRootPathOrAncestorsAreReparsePoints() + { + var root = Path.GetPathRoot(_rootPathTrimmed); + if (string.IsNullOrWhiteSpace(root)) + { + return; + } + + if (IsReparsePoint(root) || IsReparsePoint(_rootPathTrimmed)) + { + throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); + } + + var relative = Path.GetRelativePath(root, _rootPathTrimmed); + if (relative == "." || relative.Length == 0) + { + return; + } + + var current = root; + var parts = relative.Split(new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < parts.Length; i++) + { + current = Path.Combine(current, parts[i]); + if (IsReparsePoint(current)) + { + throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); + } + } + } + private void EnsureParentDirectoryExistsNoReparse(string fullPath) { var directory = Path.GetDirectoryName(fullPath); @@ -398,4 +454,22 @@ private static bool IsReparsePoint(string path) } } + private static bool TryGetAttributes(string path, out FileAttributes attributes) + { + try + { + attributes = File.GetAttributes(path); + return true; + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + + attributes = default; + return false; + } + } From b798ef7949361c0113772884cee3f590350990bb Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 03:59:51 +0100 Subject: [PATCH 022/296] Fix async analyzer: use WriteAsync in FileStateStore read --- ZTSharp/FileStateStore.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index 4555128..bb4303e 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -274,13 +274,13 @@ private static async Task ReadAllBytesWithSharingAsync(string path, Canc { if (remaining > 0) { - memory.Write(buffer, 0, (int)remaining); + await memory.WriteAsync(buffer.AsMemory(0, (int)remaining), cancellationToken).ConfigureAwait(false); } throw new IOException($"State file exceeds maximum supported size of {MaxReadBytes} bytes."); } - memory.Write(buffer, 0, read); + await memory.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false); totalRead += read; } From bda1b197503cf75c18896a0eb3f27951650a5f0a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:00:10 +0100 Subject: [PATCH 023/296] Fix IPv6 fragment handling and flow id derivation --- ZTSharp/ZeroTier/Net/Ipv6Codec.cs | 8 +++++--- ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs | 9 +++++---- ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs | 10 ++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs index a489dce..e153974 100644 --- a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs +++ b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs @@ -10,7 +10,7 @@ internal static class Ipv6Codec private const int MaxExtensionHeaderChain = 8; public static bool IsExtensionHeader(byte nextHeader) - => nextHeader is 0 or 43 or 44 or 50 or 51 or 60; + => nextHeader is 0 or 43 or 44 or 50 or 51 or 60 or 135 or 139 or 140; public static byte[] Encode( IPAddress source, @@ -131,7 +131,7 @@ private static bool TryWalkExtensionHeaders( return false; } - if (protocol is 0 or 43 or 60) + if (protocol is 0 or 43 or 60 or 135 or 139 or 140) { if (remaining < 8) { @@ -161,7 +161,9 @@ private static bool TryWalkExtensionHeaders( var headerNext = payload[offset]; var fragmentOffsetAndFlags = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(offset + 2, 2)); var fragmentOffset = (fragmentOffsetAndFlags >> 3) & 0x1FFF; - if (fragmentOffset != 0) + var reservedBits = fragmentOffsetAndFlags & 0x6; + var moreFragments = (fragmentOffsetAndFlags & 0x1) != 0; + if (fragmentOffset != 0 || moreFragments || reservedBits != 0) { return false; } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs index 93a50a1..e0c0c4e 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs @@ -49,9 +49,10 @@ public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (_terminalException is not null) + var terminal = Volatile.Read(ref _terminalException); + if (terminal is not null) { - throw _terminalException; + throw terminal; } if (_window != 0) @@ -79,9 +80,9 @@ public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) { await tcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); } - catch (Exception ex) when (_terminalException is not null && ex is not OperationCanceledException) + catch (Exception ex) when (Volatile.Read(ref _terminalException) is not null && ex is not OperationCanceledException) { - throw _terminalException; + throw Volatile.Read(ref _terminalException)!; } } } diff --git a/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs b/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs index 590e013..0df1f85 100644 --- a/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs +++ b/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs @@ -30,11 +30,17 @@ private static uint DeriveFromTransportTuple( if (protocol == TcpCodec.ProtocolNumber) { - _ = TcpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _, out _, out _, out _, out _); + if (!TcpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _, out _, out _, out _, out _)) + { + return 0; + } } else if (protocol == UdpCodec.ProtocolNumber) { - _ = UdpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _); + if (!UdpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _)) + { + return 0; + } } else { From 9299e49bfb8dd60aab9d3ff075644b7cd2501282 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:02:19 +0100 Subject: [PATCH 024/296] Fix multipath QoS tracking and peer state cleanup --- .../Internal/ZeroTierDataplaneRuntime.cs | 27 ++++++++++++------- .../Internal/ZeroTierPeerBondPolicyEngine.cs | 3 +-- .../Internal/ZeroTierPeerQosManager.cs | 1 + 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index a667986..af105f3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -449,19 +449,28 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo } } } - else + else + { + var localSockets = _udp.LocalSockets; + for (var i = 0; i < hinted.Length; i++) { - var localSockets = _udp.LocalSockets; - for (var i = 0; i < hinted.Length; i++) + var localSocketId = localSockets.Count == 0 ? 0 : localSockets[i % localSockets.Count].Id; + if (shouldRecord) + { + _peerQos.RecordOutgoingPacket(peerNodeId, localSocketId, hinted[i], parsed.PacketId); + } + + try { - try - { - var localSocketId = localSockets.Count == 0 ? 0 : localSockets[i % localSockets.Count].Id; await _udp.SendAsync(localSocketId, hinted[i], packet, cancellationToken).ConfigureAwait(false); - directSuccess++; - } - catch (SocketException) + directSuccess++; + } + catch (SocketException) + { + if (shouldRecord) { + _peerQos.ForgetOutgoingPacket(peerNodeId, localSocketId, hinted[i], parsed.PacketId); + } } } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs index 6ab257b..5629a3c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs @@ -86,8 +86,7 @@ public void MaintenanceTick() var lastUsed = Volatile.Read(ref state.LastUsedMs); if (lastUsed != 0 && unchecked(now - lastUsed) > PeerStateTtlMs && - state.Flows.IsEmpty && - state.ActiveBackupPath is null) + state.Flows.IsEmpty) { _peerStates.TryRemove(pair.Key, out _); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs index f60a60d..a2e4f51 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs @@ -234,6 +234,7 @@ public bool TryGetLastLatencyAverageMs( } private static bool ShouldTrack(ulong packetId) + // Intentionally mirrors upstream (Bond.cpp): record/ack most packets and skip every Nth (power-of-two) packet id. => (packetId & (ulong)(QosAckDivisor - 1)) != 0; private static void CleanupOutboundIfNeeded(PathState state, long now) From bd9e17a444ffa7656c1f7a971cad40059a616d8d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:05:27 +0100 Subject: [PATCH 025/296] Overlay TCP: drop late data after FIN without fault --- ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs index d89bcf9..c6514a9 100644 --- a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs +++ b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs @@ -40,11 +40,21 @@ public bool TryWrite(ReadOnlyMemory segment) return false; } + if (Volatile.Read(ref _remoteClosed) != 0) + { + return false; + } + if (_incoming.Writer.TryWrite(segment)) { return true; } + if (Volatile.Read(ref _remoteClosed) != 0) + { + return false; + } + Fault(new IOException("Overlay TCP receive buffer overflowed; closing connection to avoid silent data loss.")); return false; } From f779c4291911d62a6763e6ad27ea68dc6002d5fb Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:05:32 +0100 Subject: [PATCH 026/296] Overlay HTTP: timeout local port reservation exhaustion --- ZTSharp/Http/OverlayHttpMessageHandler.cs | 28 +++++++++++++++++++ .../Http/OverlayHttpMessageHandlerOptions.cs | 6 ++++ 2 files changed, 34 insertions(+) diff --git a/ZTSharp/Http/OverlayHttpMessageHandler.cs b/ZTSharp/Http/OverlayHttpMessageHandler.cs index 8338939..0ff5a7d 100644 --- a/ZTSharp/Http/OverlayHttpMessageHandler.cs +++ b/ZTSharp/Http/OverlayHttpMessageHandler.cs @@ -36,6 +36,11 @@ public OverlayHttpMessageHandler(Node node, ulong networkId, OverlayHttpMessageH "LocalPortEnd must be in the range 1..65535 and greater than or equal to LocalPortStart."); } + if (_options.LocalPortAllocationTimeout != Timeout.InfiniteTimeSpan && _options.LocalPortAllocationTimeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(options), "LocalPortAllocationTimeout must be positive or Timeout.InfiniteTimeSpan."); + } + var sockets = new SocketsHttpHandler { UseProxy = false @@ -87,12 +92,23 @@ private async Task AllocateReservedLocalPortAsync(CancellationToken cancell var start = _options.LocalPortStart; var end = _options.LocalPortEnd; var range = end - start + 1; + var allocationTimeout = _options.LocalPortAllocationTimeout; + var timeoutMs = allocationTimeout == Timeout.InfiniteTimeSpan + ? long.MaxValue + : (long)Math.Min(long.MaxValue, Math.Ceiling(allocationTimeout.TotalMilliseconds)); + var startedMs = Environment.TickCount64; var backoffMs = 1; while (true) { cancellationToken.ThrowIfCancellationRequested(); + if (timeoutMs != long.MaxValue && unchecked(Environment.TickCount64 - startedMs) > timeoutMs) + { + throw new HttpRequestException( + $"Overlay TCP local port allocation exhausted (range {start}..{end}) after {allocationTimeout}."); + } + for (var i = 0; i < range; i++) { var candidate = AllocateLocalPort(); @@ -102,6 +118,18 @@ private async Task AllocateReservedLocalPortAsync(CancellationToken cancell } } + if (timeoutMs != long.MaxValue) + { + var elapsed = unchecked(Environment.TickCount64 - startedMs); + var remaining = timeoutMs - elapsed; + if (remaining <= 0) + { + continue; + } + + backoffMs = (int)Math.Min(backoffMs, remaining); + } + await Task.Delay(backoffMs, cancellationToken).ConfigureAwait(false); backoffMs = Math.Min(backoffMs * 2, 50); } diff --git a/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs b/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs index 17d0da4..52b833f 100644 --- a/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs +++ b/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs @@ -18,5 +18,11 @@ public sealed class OverlayHttpMessageHandlerOptions public int LocalPortStart { get; init; } = 49152; public int LocalPortEnd { get; init; } = 65535; + + /// + /// Maximum time to wait for a reserved local overlay TCP port within ... + /// Use to wait indefinitely (not recommended unless requests are always cancellable). + /// + public TimeSpan LocalPortAllocationTimeout { get; init; } = TimeSpan.FromSeconds(30); } From dfc4da266d0ae7762397f0c38bda1c32f413dda7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:05:40 +0100 Subject: [PATCH 027/296] Tests: skip Unix perms on Windows; add key normalization cases --- ZTSharp.Tests/SecretFilePermissionTests.cs | 2 +- .../StateStoreKeyNormalizationSecurityTests.cs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 ZTSharp.Tests/StateStoreKeyNormalizationSecurityTests.cs diff --git a/ZTSharp.Tests/SecretFilePermissionTests.cs b/ZTSharp.Tests/SecretFilePermissionTests.cs index 5fc3841..f153532 100644 --- a/ZTSharp.Tests/SecretFilePermissionTests.cs +++ b/ZTSharp.Tests/SecretFilePermissionTests.cs @@ -9,7 +9,7 @@ public void ZeroTierIdentityStore_Save_SetsUnixMode600_OnUnix() { if (OperatingSystem.IsWindows()) { - return; + throw Xunit.Sdk.SkipException.ForSkip("Unix file mode tests require a Unix-like platform."); } var root = TestTempPaths.CreateGuidSuffixed("zt-secret-perms-"); diff --git a/ZTSharp.Tests/StateStoreKeyNormalizationSecurityTests.cs b/ZTSharp.Tests/StateStoreKeyNormalizationSecurityTests.cs new file mode 100644 index 0000000..6aa3be7 --- /dev/null +++ b/ZTSharp.Tests/StateStoreKeyNormalizationSecurityTests.cs @@ -0,0 +1,17 @@ +namespace ZTSharp.Tests; + +public sealed class StateStoreKeyNormalizationSecurityTests +{ + [Theory] + [InlineData("file:stream")] + [InlineData("C:\\Windows\\System32")] + [InlineData("C:Windows\\System32")] + [InlineData("\\\\?\\C:\\Windows\\System32")] + [InlineData("\\\\server\\share\\file")] + [InlineData("/absolute/path")] + [InlineData("\\absolute\\path")] + public void NormalizeKey_Rejects_Rooted_Or_Ads_Like_Keys(string key) + { + _ = Assert.Throws(() => StateStoreKeyNormalization.NormalizeKey(key)); + } +} From db9013db480f6ee65be6a7a297c01ae658e30e69 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:05:47 +0100 Subject: [PATCH 028/296] Tests: assert spoofed UDP reaches receive loop before expecting drop --- ZTSharp.Tests/OsUdpSpoofingTests.cs | 44 ++++++++++++++++++++++--- ZTSharp/Internal/NodeCore.cs | 2 ++ ZTSharp/Node.cs | 2 ++ ZTSharp/Transport/OsUdpNodeTransport.cs | 2 ++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/ZTSharp.Tests/OsUdpSpoofingTests.cs b/ZTSharp.Tests/OsUdpSpoofingTests.cs index 71030f2..e423230 100644 --- a/ZTSharp.Tests/OsUdpSpoofingTests.cs +++ b/ZTSharp.Tests/OsUdpSpoofingTests.cs @@ -38,6 +38,7 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForUdpHandlers() var node2Endpoint = node2.LocalTransportEndpoint; Assert.NotNull(node2Endpoint); + var transport2 = Assert.IsType(node2.TransportForTests); await using var udp2 = new ZtUdpClient(node2, networkId, localPort: 12002); var datagram = "spoof"u8.ToArray(); @@ -54,9 +55,26 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForUdpHandlers() var frameBuffer = new byte[NodeFrameCodec.GetEncodedLength(udpPayload.Length)]; Assert.True(NodeFrameCodec.TryEncode(networkId, node1Id, udpPayload, frameBuffer, out var frameLength)); - using (var spoofUdp = CreateSpoofUdp(node2Endpoint!)) + using var spoofUdp = CreateSpoofUdp(node2Endpoint!); + var spoofLocalEndpoint = (IPEndPoint)spoofUdp.Client.LocalEndPoint!; + var datagramSeen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport2.SetDatagramObserverForTests(remote => { - await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + if (remote.Equals(spoofLocalEndpoint)) + { + datagramSeen.TrySetResult(true); + } + }); + + try + { + var sent = await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + Assert.Equal(frameLength, sent); + _ = await datagramSeen.Task.WaitAsync(TimeSpan.FromSeconds(1)); + } + finally + { + transport2.SetDatagramObserverForTests(null); } using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(250)); @@ -109,9 +127,27 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForOverlayTcpListeners var node2Endpoint = node2.LocalTransportEndpoint; Assert.NotNull(node2Endpoint); - using (var spoofUdp = CreateSpoofUdp(node2Endpoint!)) + var transport2 = Assert.IsType(node2.TransportForTests); + using var spoofUdp = CreateSpoofUdp(node2Endpoint!); + var spoofLocalEndpoint = (IPEndPoint)spoofUdp.Client.LocalEndPoint!; + var datagramSeen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport2.SetDatagramObserverForTests(remote => + { + if (remote.Equals(spoofLocalEndpoint)) + { + datagramSeen.TrySetResult(true); + } + }); + + try + { + var sent = await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + Assert.Equal(frameLength, sent); + _ = await datagramSeen.Task.WaitAsync(TimeSpan.FromSeconds(1)); + } + finally { - await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + transport2.SetDatagramObserverForTests(null); } using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(250)); diff --git a/ZTSharp/Internal/NodeCore.cs b/ZTSharp/Internal/NodeCore.cs index 9e47a0a..5f9207a 100644 --- a/ZTSharp/Internal/NodeCore.cs +++ b/ZTSharp/Internal/NodeCore.cs @@ -66,6 +66,8 @@ public NodeCore( public IPEndPoint? LocalTransportEndpoint => _transportService.GetLocalTransportEndpoint(); + internal INodeTransport TransportForTests => _transport; + public NodeId NodeId => _runtime.NodeId; public bool IsRunning => _runtime.State == NodeState.Running; diff --git a/ZTSharp/Node.cs b/ZTSharp/Node.cs index d14e21b..f5a923b 100644 --- a/ZTSharp/Node.cs +++ b/ZTSharp/Node.cs @@ -32,6 +32,8 @@ public Node(NodeOptions options) public IPEndPoint? LocalTransportEndpoint => _core.LocalTransportEndpoint; + internal Transport.INodeTransport TransportForTests => _core.TransportForTests; + public NodeId NodeId => _core.NodeId; public bool IsRunning => _core.IsRunning; diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index af6c826..edd5739 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -318,5 +318,7 @@ await _udp } } + internal void SetDatagramObserverForTests(Action? observer) + => _receiver.DatagramReceivedForTests = observer; } From 3ce01780381317c1f72b2a2bc4b81bd81d1546d1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:06:12 +0100 Subject: [PATCH 029/296] Remove unused SecretFilePermissions helper --- ZTSharp/Internal/SecretFilePermissions.cs | 29 ----------------------- 1 file changed, 29 deletions(-) delete mode 100644 ZTSharp/Internal/SecretFilePermissions.cs diff --git a/ZTSharp/Internal/SecretFilePermissions.cs b/ZTSharp/Internal/SecretFilePermissions.cs deleted file mode 100644 index 75b6904..0000000 --- a/ZTSharp/Internal/SecretFilePermissions.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace ZTSharp.Internal; - -internal static class SecretFilePermissions -{ - public static void TryHardenSecretFile(string path) - { - ArgumentException.ThrowIfNullOrWhiteSpace(path); - - if (OperatingSystem.IsWindows()) - { - return; - } - - try - { - File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); - } - catch (PlatformNotSupportedException) - { - } - catch (IOException) - { - } - catch (UnauthorizedAccessException) - { - } - } -} - From d57fa27c686b4a467b407e53f66bf6746bebfa83 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:09:50 +0100 Subject: [PATCH 030/296] Tests: add UnixFact for platform-skipped permission test --- ZTSharp.Tests/SecretFilePermissionTests.cs | 4 ++-- ZTSharp.Tests/UnixFactAttribute.cs | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 ZTSharp.Tests/UnixFactAttribute.cs diff --git a/ZTSharp.Tests/SecretFilePermissionTests.cs b/ZTSharp.Tests/SecretFilePermissionTests.cs index f153532..37240b7 100644 --- a/ZTSharp.Tests/SecretFilePermissionTests.cs +++ b/ZTSharp.Tests/SecretFilePermissionTests.cs @@ -4,12 +4,12 @@ namespace ZTSharp.Tests; public sealed class SecretFilePermissionTests { - [Fact] + [UnixFact] public void ZeroTierIdentityStore_Save_SetsUnixMode600_OnUnix() { if (OperatingSystem.IsWindows()) { - throw Xunit.Sdk.SkipException.ForSkip("Unix file mode tests require a Unix-like platform."); + throw new InvalidOperationException("UnixFact should have skipped this test on Windows."); } var root = TestTempPaths.CreateGuidSuffixed("zt-secret-perms-"); diff --git a/ZTSharp.Tests/UnixFactAttribute.cs b/ZTSharp.Tests/UnixFactAttribute.cs new file mode 100644 index 0000000..9103e4d --- /dev/null +++ b/ZTSharp.Tests/UnixFactAttribute.cs @@ -0,0 +1,16 @@ +using Xunit; + +namespace ZTSharp.Tests; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +internal sealed class UnixFactAttribute : FactAttribute +{ + public UnixFactAttribute() + { + if (OperatingSystem.IsWindows()) + { + Skip = "Unix only."; + } + } +} + From 363dfb91fab50417175589a0125d66fb09408004 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:42:43 +0100 Subject: [PATCH 031/296] OS UDP: refresh peer last-seen on app frames --- .../OsUdpPeerRegistryLastSeenTests.cs | 46 +++++++++++++++++++ .../Transport/Internal/OsUdpPeerRegistry.cs | 23 ++++++++++ .../Transport/Internal/OsUdpReceiveLoop.cs | 1 + 3 files changed, 70 insertions(+) create mode 100644 ZTSharp.Tests/OsUdpPeerRegistryLastSeenTests.cs diff --git a/ZTSharp.Tests/OsUdpPeerRegistryLastSeenTests.cs b/ZTSharp.Tests/OsUdpPeerRegistryLastSeenTests.cs new file mode 100644 index 0000000..7fcd2cf --- /dev/null +++ b/ZTSharp.Tests/OsUdpPeerRegistryLastSeenTests.cs @@ -0,0 +1,46 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.Tests; + +[Collection("OsUdpPeerRegistry")] +public sealed class OsUdpPeerRegistryLastSeenTests +{ + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public ManualTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta); + } + + [Fact] + public void RefreshPeerLastSeen_KeepsActivePeerFromExpiring() + { + var time = new ManualTimeProvider(DateTimeOffset.UnixEpoch); + var registry = new OsUdpPeerRegistry(enablePeerDiscovery: false, UdpEndpointNormalization.Normalize, timeProvider: time); + var networkId = 0x1111UL; + var peerNodeId = 0x2222UL; + var endpoint = new IPEndPoint(IPAddress.Loopback, 9999); + + registry.AddOrUpdatePeer(networkId, peerNodeId, endpoint); + + time.Advance(TimeSpan.FromMinutes(4)); + registry.RefreshPeerLastSeen(networkId, peerNodeId); + + time.Advance(TimeSpan.FromMinutes(2)); + Assert.True(registry.TryGetPeers(networkId, out var peers)); + Assert.True(peers.ContainsKey(peerNodeId)); + + time.Advance(TimeSpan.FromMinutes(10)); + Assert.True(registry.TryGetPeers(networkId, out peers)); + Assert.False(peers.ContainsKey(peerNodeId)); + } +} + diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 9de6dd5..67df804 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -64,6 +64,29 @@ public bool TryGetPeers(ulong networkId, out ConcurrentDictionary _networkPeers.TryRemove(networkId, out _); + public void RefreshPeerLastSeen(ulong networkId, ulong nodeId) + { + if (!_networkPeers.TryGetValue(networkId, out var peers)) + { + return; + } + + var nowTicks = GetNowTicks(); + while (peers.TryGetValue(nodeId, out var existing)) + { + if (existing.LastSeenTicks >= nowTicks) + { + return; + } + + var updated = existing with { LastSeenTicks = nowTicks }; + if (peers.TryUpdate(nodeId, updated, existing)) + { + return; + } + } + } + public void AddOrUpdatePeer(ulong networkId, ulong nodeId, IPEndPoint endpoint) { var nowTicks = GetNowTicks(); diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index cf0c407..160bb4b 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -145,6 +145,7 @@ public async Task RunAsync(CancellationToken cancellationToken) try { await _dispatchFrameAsync(sourceNodeId, networkId, payload, cancellationToken).ConfigureAwait(false); + _peers.RefreshPeerLastSeen(networkId, sourceNodeId); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { From 522c88dd0c9541704320302b6e67d5d860a7ca9c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:46:48 +0100 Subject: [PATCH 032/296] OS UDP: harden discovery against peer injection --- ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs | 9 ++- .../OsUdpReceiveLoopSocketExceptionTests.cs | 1 + .../Transport/Internal/OsUdpPeerRegistry.cs | 79 +++++++++++-------- .../Transport/Internal/OsUdpReceiveLoop.cs | 2 +- 4 files changed, 55 insertions(+), 36 deletions(-) diff --git a/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs b/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs index 6662f2d..f141c2e 100644 --- a/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs +++ b/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs @@ -32,7 +32,10 @@ public void DirectoryPeers_AreCappedPerNetwork() for (var i = 0; i < OsUdpPeerRegistry.DirectoryMaxPeersPerNetwork + 50; i++) { - registry.RegisterDiscoveredPeer(networkId, sourceNodeId: (ulong)(i + 1), new IPEndPoint(IPAddress.Loopback, 9999)); + _ = registry.RegisterLocalAndGetKnownPeers( + networkId, + localNodeId: (ulong)(i + 1), + advertisedEndpoint: new IPEndPoint(IPAddress.Loopback, 9999)); } Assert.True(OsUdpPeerRegistry.GetDirectoryPeerCountForTests(networkId) <= OsUdpPeerRegistry.DirectoryMaxPeersPerNetwork); @@ -53,11 +56,11 @@ public void DirectoryPeers_ExpireOnTtlSweep() var registry = new OsUdpPeerRegistry(enablePeerDiscovery: true, UdpEndpointNormalization.Normalize, timeProvider: time); var networkId = 0x5678UL; - registry.RegisterDiscoveredPeer(networkId, sourceNodeId: 1, new IPEndPoint(IPAddress.Loopback, 9999)); + _ = registry.RegisterLocalAndGetKnownPeers(networkId, localNodeId: 1, advertisedEndpoint: new IPEndPoint(IPAddress.Loopback, 9999)); Assert.Equal(1, OsUdpPeerRegistry.GetDirectoryPeerCountForTests(networkId)); time.Advance(OsUdpPeerRegistry.DirectoryPeerTtl + TimeSpan.FromSeconds(1)); - registry.Cleanup(); + _ = registry.RegisterLocalAndGetKnownPeers(networkId: networkId + 1, localNodeId: 2, advertisedEndpoint: new IPEndPoint(IPAddress.Loopback, 9998)); Assert.Equal(0, OsUdpPeerRegistry.GetDirectoryPeerCountForTests(networkId)); } diff --git a/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs b/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs index 1cb33cc..ed3c6fe 100644 --- a/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs +++ b/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs @@ -80,6 +80,7 @@ public async Task OsUdpReceiveLoop_DoesNotBlockOnDiscoveryReplySends() var peers = new OsUdpPeerRegistry(enablePeerDiscovery: true, UdpEndpointNormalization.Normalize); peers.SetLocalNodeId(networkId, localNodeId); + peers.AddOrUpdatePeer(networkId, remoteNodeId, remoteEndpoint); var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var allowSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 67df804..9ce32e9 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -57,7 +57,9 @@ public bool TryGetPeers(ulong networkId, out ConcurrentDictionary new ConcurrentDictionary()); - peers[sourceNodeId] = new PeerEntry(endpoint, nowTicks); - var directoryPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - directoryPeers[sourceNodeId] = new PeerEntry(endpoint, nowTicks); - SweepDirectory(nowTicks); - SweepNetworkPeers(nowTicks); - } - public void Cleanup() { var nowTicks = GetNowTicks(); @@ -196,21 +181,21 @@ private static void RemoveFromDirectory(ulong networkId, ulong nodeId) private long GetNowTicks() => _timeProvider.GetUtcNow().UtcDateTime.Ticks; - private static void EvictExpiredAndTrimNetwork(ulong networkId, ConcurrentDictionary discoveredPeers, long nowTicks) + private static void EvictExpiredAndTrimPeers(ConcurrentDictionary peers, long nowTicks) { var cutoffTicks = nowTicks - DirectoryPeerTtlTicks; - foreach (var peer in discoveredPeers) + foreach (var peer in peers) { if (peer.Value.LastSeenTicks < cutoffTicks) { - discoveredPeers.TryRemove(peer.Key, out _); + peers.TryRemove(peer.Key, out _); } } - if (discoveredPeers.Count > DirectoryMaxPeersPerNetwork) + if (peers.Count > DirectoryMaxPeersPerNetwork) { - var removeCount = discoveredPeers.Count - DirectoryMaxPeersPerNetwork; - var toRemove = discoveredPeers + var removeCount = peers.Count - DirectoryMaxPeersPerNetwork; + var toRemove = peers .OrderBy(p => p.Value.LastSeenTicks) .Take(removeCount) .Select(p => p.Key) @@ -218,19 +203,14 @@ private static void EvictExpiredAndTrimNetwork(ulong networkId, ConcurrentDictio foreach (var key in toRemove) { - discoveredPeers.TryRemove(key, out _); + peers.TryRemove(key, out _); } } - - if (discoveredPeers.IsEmpty) - { - s_networkDirectory.TryRemove(networkId, out _); - } } private void EvictExpiredAndTrimNetworkPeers(ulong networkId, ConcurrentDictionary peers, long nowTicks) { - EvictExpiredAndTrimNetwork(networkId, peers, nowTicks); + EvictExpiredAndTrimPeers(peers, nowTicks); if (peers.IsEmpty) { _networkPeers.TryRemove(networkId, out _); @@ -279,7 +259,11 @@ private static void SweepDirectory(long nowTicks) { foreach (var network in s_networkDirectory) { - EvictExpiredAndTrimNetwork(network.Key, network.Value, nowTicks); + EvictExpiredAndTrimPeers(network.Value, nowTicks); + if (network.Value.IsEmpty) + { + s_networkDirectory.TryRemove(network.Key, out _); + } } if (s_networkDirectory.Count <= DirectoryMaxNetworks) @@ -313,6 +297,37 @@ private static void SweepDirectory(long nowTicks) } } + private void ImportDirectoryPeers(ulong networkId, ConcurrentDictionary peers, long nowTicks) + { + if (!_enablePeerDiscovery) + { + return; + } + + if (!s_networkDirectory.TryGetValue(networkId, out var directoryPeers)) + { + return; + } + + EvictExpiredAndTrimPeers(directoryPeers, nowTicks); + if (directoryPeers.IsEmpty) + { + s_networkDirectory.TryRemove(networkId, out _); + return; + } + + _localNodeIds.TryGetValue(networkId, out var localNodeId); + foreach (var peer in directoryPeers) + { + if (peer.Key == localNodeId) + { + continue; + } + + peers[peer.Key] = new PeerEntry(peer.Value.Endpoint, nowTicks); + } + } + internal static void ClearDirectoryForTests() => s_networkDirectory.Clear(); diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index 160bb4b..22c2bbc 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -113,9 +113,9 @@ public async Task RunAsync(CancellationToken cancellationToken) { if (discoveredNodeId != 0 && discoveredNodeId == sourceNodeId) { - _peers.RegisterDiscoveredPeer(networkId, discoveredNodeId, normalizedRemoteEndpoint); if (localNodeId != discoveredNodeId && controlFrameType == OsUdpPeerDiscoveryProtocol.FrameType.PeerHello && + IPAddress.IsLoopback(normalizedRemoteEndpoint.Address) && ShouldSendHelloResponse(networkId, discoveredNodeId)) { QueueDiscoverySend( From 8252bd4be8310234983233afd94bb09e3fce7879 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:51:09 +0100 Subject: [PATCH 033/296] IPv6: accept extension headers in ingress --- .../ZeroTierDataplaneFragmentPolicyTests.cs | 51 +++++++++++++++++++ .../Internal/ZeroTierDataplaneIpHandler.cs | 24 +++++---- ZTSharp/ZeroTier/Net/Ipv6Codec.cs | 34 ++++++++++++- ZTSharp/ZeroTier/ZeroTierTcpListener.cs | 4 +- ZTSharp/ZeroTier/ZeroTierUdpSocket.cs | 4 +- 5 files changed, 100 insertions(+), 17 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs b/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs index 67da90b..6e0652e 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs @@ -49,6 +49,40 @@ public void Ipv6ExtensionHeaders_AreRecognized() Assert.False(Ipv6Codec.IsExtensionHeader(TcpCodec.ProtocolNumber)); } + [Fact] + public async Task Ipv6HopByHopHeader_DoesNotPreventUdpHandlerDispatch() + { + var localManagedIpV6 = IPAddress.Parse("fd00::2"); + await using var runtime = CreateRuntimeV6(localManagedIpV6); + var ip = GetIpHandler(runtime); + + const ushort localPort = 12011; + var udpChannel = Channel.CreateUnbounded(); + Assert.True(runtime.TryRegisterUdpPort(AddressFamily.InterNetworkV6, localPort, udpChannel.Writer)); + + var remoteIpV6 = IPAddress.Parse("fd00::1"); + var udp = UdpCodec.Encode(remoteIpV6, localManagedIpV6, sourcePort: 1111, destinationPort: localPort, payload: new byte[] { 1, 2, 3 }); + + var hbh = new byte[8]; + hbh[0] = UdpCodec.ProtocolNumber; // next header + hbh[1] = 0; // hdr ext len (8 bytes total) + + var payload = new byte[hbh.Length + udp.Length]; + hbh.CopyTo(payload, 0); + udp.CopyTo(payload, hbh.Length); + + var ipv6 = Ipv6Codec.Encode( + source: remoteIpV6, + destination: localManagedIpV6, + nextHeader: 0, // hop-by-hop + payload: payload, + hopLimit: 64); + + await ip.HandleIpv6PacketAsync(peerNodeId: new NodeId(0x3333333333), ipv6Packet: ipv6, cancellationToken: CancellationToken.None); + + Assert.True(udpChannel.Reader.TryRead(out _)); + } + private static void RewriteIpv4HeaderChecksum(byte[] packet) { var headerLength = (packet[0] & 0x0F) * 4; @@ -112,4 +146,21 @@ private static ZeroTierDataplaneRuntime CreateRuntime(IPAddress localManagedIpV4 localManagedIpsV4: new[] { localManagedIpV4 }, localManagedIpsV6: Array.Empty(), inlineCom: new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); + + [global::System.Diagnostics.CodeAnalysis.SuppressMessage( + "Reliability", + "CA2000:Dispose objects before losing scope", + Justification = "UDP transport ownership transfers to ZeroTierDataplaneRuntime, which is disposed by the caller.")] + private static ZeroTierDataplaneRuntime CreateRuntimeV6(IPAddress localManagedIpV6) + => new( + udp: new ZeroTierUdpTransport(localPort: 0, enableIpv6: false), + rootNodeId: new NodeId(0x1111111111), + rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootKey: new byte[48], + rootProtocolVersion: 12, + localIdentity: ZeroTierTestIdentities.CreateFastIdentity(0x2222222222), + networkId: 1, + localManagedIpsV4: Array.Empty(), + localManagedIpsV6: new[] { localManagedIpV6 }, + inlineCom: new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs index d2d71c2..bf24a0c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs @@ -58,18 +58,20 @@ public async ValueTask HandleIpv6PacketAsync(NodeId peerNodeId, ReadOnlyMemory packet, + out IPAddress source, + out IPAddress destination, + out byte protocol, + out byte hopLimit, + out ReadOnlySpan transportPayload, + out int transportPayloadOffset) + { + transportPayload = default; + transportPayloadOffset = 0; + protocol = 0; + + if (!TryParse(packet, out source, out destination, out var nextHeader, out hopLimit, out var payload)) + { + return false; + } + + if (!TryWalkExtensionHeaders(payload, nextHeader, out protocol, out transportPayload, out var offsetFromPayload)) + { + return false; + } + + transportPayloadOffset = HeaderLength + offsetFromPayload; + return true; } private static bool TryWalkExtensionHeaders( ReadOnlySpan payload, byte nextHeader, out byte protocol, - out ReadOnlySpan transportPayload) + out ReadOnlySpan transportPayload, + out int transportPayloadOffsetFromPayload) { protocol = nextHeader; transportPayload = payload; + transportPayloadOffsetFromPayload = 0; var offset = 0; for (var i = 0; i < MaxExtensionHeaderChain && IsExtensionHeader(protocol); i++) @@ -199,6 +228,7 @@ private static bool TryWalkExtensionHeaders( } } + transportPayloadOffsetFromPayload = offset; transportPayload = payload.Slice(offset); return !IsExtensionHeader(protocol); } diff --git a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs index c90fd9e..0768ae3 100644 --- a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs +++ b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs @@ -151,12 +151,12 @@ private Task OnSynAsync(NodeId peerNodeId, ReadOnlyMemory ipPacket, Cancel } else { - if (!Ipv6Codec.TryParse(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(ipPacket.Span, out src, out dst, out var protocol, out _, out ipPayload)) { return Task.CompletedTask; } - if ((!_isWildcardBind && !ZeroTierIpAddressCanonicalization.EqualsForManagedIpComparison(dst, _localAddressMatch)) || nextHeader != TcpCodec.ProtocolNumber) + if ((!_isWildcardBind && !ZeroTierIpAddressCanonicalization.EqualsForManagedIpComparison(dst, _localAddressMatch)) || protocol != TcpCodec.ProtocolNumber) { return Task.CompletedTask; } diff --git a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs index b1b7975..d781bce 100644 --- a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs @@ -140,12 +140,12 @@ public async ValueTask ReceiveFromAsync( } else { - if (!Ipv6Codec.TryParse(routed.Packet.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(routed.Packet.Span, out src, out dst, out var protocol, out _, out ipPayload)) { continue; } - if (!dst.Equals(_localAddress) || nextHeader != UdpCodec.ProtocolNumber) + if (!dst.Equals(_localAddress) || protocol != UdpCodec.ProtocolNumber) { continue; } From 6b72b7672f69ef1a20b1fbe1230cb0e1d1cac87c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:52:13 +0100 Subject: [PATCH 034/296] TCP: synchronize remote send window reads --- ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs index e0c0c4e..49ca57d 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs @@ -2,12 +2,12 @@ namespace ZTSharp.ZeroTier.Net; internal sealed class UserSpaceTcpRemoteSendWindow { - private ushort _window = ushort.MaxValue; + private int _window = ushort.MaxValue; private TaskCompletionSource? _windowTcs; private Exception? _terminalException; private readonly object _lock = new(); - public ushort Window => _window; + public ushort Window => (ushort)Volatile.Read(ref _window); public void Update(ushort windowSize) { @@ -55,7 +55,7 @@ public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) throw terminal; } - if (_window != 0) + if (Volatile.Read(ref _window) != 0) { return; } From ea15668df11abcf4edc36d727128c8bc8c115046 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:54:57 +0100 Subject: [PATCH 035/296] Multipath: honor LocalUdpPorts for single socket --- ...ketRuntimeBootstrapperUdpTransportTests.cs | 35 +++++++++++++++++++ .../ZeroTierSocketRuntimeBootstrapper.cs | 19 +++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index ad3e546..3aa7e05 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -1,5 +1,7 @@ +using System.Net.Sockets; using ZTSharp.ZeroTier; using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; namespace ZTSharp.Tests; @@ -36,4 +38,37 @@ public async Task CreateUdpTransport_MultipathEnabled_UsesMultipleSockets() await transport.DisposeAsync(); } } + + [Fact] + public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() + { + const int attempts = 25; + for (var i = 0; i < attempts; i++) + { + var port = Random.Shared.Next(20_000, 60_000); + var transport = default(IZeroTierUdpTransport); + try + { + transport = ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 1, LocalUdpPorts = new[] { port } }, + enableIpv6: false); + + Assert.Single(transport.LocalSockets); + Assert.Equal(port, transport.LocalSockets[0].LocalEndpoint.Port); + return; + } + catch (SocketException) + { + } + finally + { + if (transport is not null) + { + await transport.DisposeAsync(); + } + } + } + + Assert.Fail($"Failed to bind a UDP socket to a random port after {attempts} attempts."); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 3174c44..6074e28 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -13,11 +13,28 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption { ArgumentNullException.ThrowIfNull(multipath); - if (!multipath.Enabled || multipath.UdpSocketCount == 1) + if (!multipath.Enabled) { return new ZeroTierUdpTransport(localPort: 0, enableIpv6: enableIpv6, localSocketId: 0); } + if (multipath.UdpSocketCount == 1) + { + var port = 0; + var localPorts = multipath.LocalUdpPorts; + if (localPorts is not null) + { + if (localPorts.Count != 1) + { + throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts length must match UdpSocketCount."); + } + + port = localPorts[0]; + } + + return new ZeroTierUdpTransport(localPort: port, enableIpv6: enableIpv6, localSocketId: 0); + } + var ports = multipath.LocalUdpPorts; if (ports is null) { From 8ac29fd45a2b1a76849e93c8774424f0b68f99dc Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:56:22 +0100 Subject: [PATCH 036/296] QoS: skip Ack sampling and stale latency --- ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs | 21 +++++++++++++++++++ .../ZeroTierDataplanePeerDatagramProcessor.cs | 2 +- .../Internal/ZeroTierPeerQosManager.cs | 18 +++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs index 7463ef6..8a2fae4 100644 --- a/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs @@ -71,4 +71,25 @@ public void TryBuildOutboundPayload_EncodesLittleEndianPacketIdAndHoldingTime() Assert.False(mgr.TryBuildOutboundPayload(peerNodeId, localSocketId: 1, endpoint, out _)); } + + [Fact] + public void TryGetLastLatencyAverageMs_ReturnsFalse_WhenAverageIsStale() + { + var now = 1_000L; + var mgr = new ZeroTierPeerQosManager(nowMs: () => now); + + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("203.0.113.9"), 9999); + + mgr.RecordOutgoingPacket(peerNodeId, localSocketId: 1, endpoint, packetId: 3); + + now = 1_500; + Span payload = stackalloc byte[8 + 2]; + BinaryPrimitives.WriteUInt64LittleEndian(payload.Slice(0, 8), 3UL); + BinaryPrimitives.WriteUInt16LittleEndian(payload.Slice(8, 2), 200); + mgr.HandleInboundMeasurement(peerNodeId, localSocketId: 1, endpoint, payload); + + now = 1_500 + 120_000 + 1; + Assert.False(mgr.TryGetLastLatencyAverageMs(peerNodeId, localSocketId: 1, endpoint, out _)); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index 43f3e9b..ddd7db4 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -107,7 +107,7 @@ await _peerEcho var payload = packetBytes.AsMemory(ZeroTierPacketHeader.IndexPayload); var payloadSpan = payload.Span; - if (decoded.Header.HopCount == 0 && verb != ZeroTierVerb.QosMeasurement) + if (decoded.Header.HopCount == 0 && verb is not (ZeroTierVerb.QosMeasurement or ZeroTierVerb.Ack)) { _peerQos.RecordIncomingPacket(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, decoded.Header.PacketId); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs index a2e4f51..9deb2ee 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs @@ -21,6 +21,7 @@ internal sealed class ZeroTierPeerQosManager // See: ZeroTierOne node/Bond.cpp (qosStatsOut timeout = _qosSendInterval * 3). private const long DefaultRecordTimeoutMs = 30_000; private const long PathStateTtlMs = 300_000; + private const long LatencyAverageTtlMs = 120_000; private const int MaxPathStates = 4096; private readonly Func _nowMs; @@ -223,8 +224,23 @@ public bool TryGetLastLatencyAverageMs( ArgumentNullException.ThrowIfNull(remoteEndPoint); var key = new ZeroTierPeerQosPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); - if (_paths.TryGetValue(key, out var state) && Volatile.Read(ref state.LastLatencyUpdatedMs) != 0) + if (_paths.TryGetValue(key, out var state)) { + var updated = Volatile.Read(ref state.LastLatencyUpdatedMs); + if (updated == 0) + { + averageLatencyMs = 0; + return false; + } + + var now = _nowMs(); + var age = unchecked(now - updated); + if (age < 0 || age > LatencyAverageTtlMs) + { + averageLatencyMs = 0; + return false; + } + averageLatencyMs = Volatile.Read(ref state.LastLatencyAvgMs); return true; } From 3ab2a35d0c600c3a2adad8665c4e4ecee84b1dcf Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:57:38 +0100 Subject: [PATCH 037/296] Bond: include IPv6 ScopeId in stable ordering --- .../ZeroTierPeerBondPolicyEngineTests.cs | 19 +++++++++++++++++++ .../Internal/ZeroTierPeerBondPolicyEngine.cs | 9 +++++++++ 2 files changed, 28 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs b/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs index 1512fa6..985adc1 100644 --- a/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs @@ -174,4 +174,23 @@ public void Broadcast_ReturnsStableSortedPaths() Assert.Equal(2, broadcast[1].LocalSocketId); Assert.Equal(epB, broadcast[1].RemoteEndPoint); } + + [Fact] + public void Broadcast_SortsLinkLocalEndpointsByScopeId() + { + var addrBytes = IPAddress.Parse("fe80::1").GetAddressBytes(); + var epScope2 = new IPEndPoint(new IPAddress(addrBytes, scopeid: 2), 1001); + var epScope1 = new IPEndPoint(new IPAddress(addrBytes, scopeid: 1), 1001); + + var paths = new[] + { + new ZeroTierPeerPhysicalPath(LocalSocketId: 1, epScope2, LastSeenUnixMs: 1), + new ZeroTierPeerPhysicalPath(LocalSocketId: 1, epScope1, LastSeenUnixMs: 1), + }; + + var broadcast = ZeroTierPeerBondPolicyEngine.GetBroadcastPaths((ZeroTierPeerPhysicalPath[])paths.Clone()); + Assert.Equal(2, broadcast.Length); + Assert.Equal(epScope1, broadcast[0].RemoteEndPoint); + Assert.Equal(epScope2, broadcast[1].RemoteEndPoint); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs index 5629a3c..bcbe900 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs @@ -421,6 +421,15 @@ private static int CompareEndpoints(IPEndPoint x, IPEndPoint y) return xb.Length.CompareTo(yb.Length); } + if (x.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + var scopeCompare = x.Address.ScopeId.CompareTo(y.Address.ScopeId); + if (scopeCompare != 0) + { + return scopeCompare; + } + } + return x.Port.CompareTo(y.Port); } } From f8728404ec34466aa75f589a28533ad8eb2ddc81 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:00:36 +0100 Subject: [PATCH 038/296] Overlay TCP: grace window after FIN --- .../OverlayTcpIncomingBufferTests.cs | 18 +++++++++ ZTSharp/Sockets/OverlayTcpClient.cs | 7 +++- ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs | 40 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs index e0c173e..3927d52 100644 --- a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs +++ b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs @@ -20,4 +20,22 @@ public async Task WhenBufferOverflows_ReadThrowsIOException() await Assert.ThrowsAsync(() => incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask()); } + + [Fact] + public async Task FinGrace_AllowsLateDataFramesToBeRead() + { + var incoming = new OverlayTcpIncomingBuffer(); + incoming.MarkRemoteFinReceived(); + + var payload = new byte[] { 1, 2, 3 }; + Assert.True(incoming.TryWrite(payload)); + + var buffer = new byte[3]; + var read = await incoming.ReadAsync(buffer, CancellationToken.None); + Assert.Equal(3, read); + Assert.Equal(payload, buffer); + + var eof = await incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(1)); + Assert.Equal(0, eof); + } } diff --git a/ZTSharp/Sockets/OverlayTcpClient.cs b/ZTSharp/Sockets/OverlayTcpClient.cs index c60ea0a..46add9a 100644 --- a/ZTSharp/Sockets/OverlayTcpClient.cs +++ b/ZTSharp/Sockets/OverlayTcpClient.cs @@ -125,6 +125,11 @@ internal async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationTok throw new IOException("Remote has closed the connection."); } + if (_incoming.RemoteFinReceived) + { + throw new IOException("Remote has closed the connection."); + } + await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { @@ -213,7 +218,7 @@ private void OnFrameReceived(in RawFrame frame) if (type == OverlayTcpFrameCodec.FrameType.Fin) { - _incoming.MarkRemoteClosed(); + _incoming.MarkRemoteFinReceived(); return; } diff --git a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs index c6514a9..e3f48f5 100644 --- a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs +++ b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs @@ -8,6 +8,7 @@ internal sealed class OverlayTcpIncomingBuffer { private const int MaxQueuedSegments = 1024; private const int MaxSegmentLength = 1024; + private const int FinGracePeriodMs = 200; private readonly Channel> _incoming = Channel.CreateBounded>(new BoundedChannelOptions(MaxQueuedSegments) { @@ -17,11 +18,15 @@ internal sealed class OverlayTcpIncomingBuffer }); private ReadOnlyMemory _currentSegment; private int _currentSegmentOffset; + private int _remoteFinReceived; private int _remoteClosed; + private long _lastActivityMs; private IOException? _fault; public bool RemoteClosed => Volatile.Read(ref _remoteClosed) != 0; + public bool RemoteFinReceived => Volatile.Read(ref _remoteFinReceived) != 0; + public bool TryWrite(ReadOnlyMemory segment) { if (Volatile.Read(ref _fault) is not null) @@ -47,6 +52,7 @@ public bool TryWrite(ReadOnlyMemory segment) if (_incoming.Writer.TryWrite(segment)) { + Volatile.Write(ref _lastActivityMs, Environment.TickCount64); return true; } @@ -59,6 +65,14 @@ public bool TryWrite(ReadOnlyMemory segment) return false; } + public void MarkRemoteFinReceived() + { + if (Interlocked.CompareExchange(ref _remoteFinReceived, 1, 0) == 0) + { + Volatile.Write(ref _lastActivityMs, Environment.TickCount64); + } + } + public void MarkRemoteClosed() { Volatile.Write(ref _remoteClosed, 1); @@ -107,6 +121,32 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can try { + if (Volatile.Read(ref _remoteFinReceived) != 0) + { + var now = Environment.TickCount64; + var last = Volatile.Read(ref _lastActivityMs); + var remainingGraceMs = FinGracePeriodMs - (int)unchecked(now - last); + if (remainingGraceMs <= 0) + { + MarkRemoteClosed(); + return 0; + } + + using var graceCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + graceCts.CancelAfter(TimeSpan.FromMilliseconds(remainingGraceMs)); + try + { + _currentSegment = await _incoming.Reader.ReadAsync(graceCts.Token).ConfigureAwait(false); + _currentSegmentOffset = 0; + break; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && graceCts.IsCancellationRequested) + { + MarkRemoteClosed(); + return 0; + } + } + _currentSegment = await _incoming.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); _currentSegmentOffset = 0; break; From de83fad087b02fef7bf445b68d9f89b18ccb77df Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:23:29 +0100 Subject: [PATCH 039/296] Overlay TCP: unblock pending reads on FIN --- .../OverlayTcpIncomingBufferTests.cs | 14 +++++ ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs | 58 +++++++++++++++---- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs index 3927d52..74aff6a 100644 --- a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs +++ b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs @@ -38,4 +38,18 @@ public async Task FinGrace_AllowsLateDataFramesToBeRead() var eof = await incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(1)); Assert.Equal(0, eof); } + + [Fact] + public async Task FinArrival_UnblocksReaderAwaitingReadAsync() + { + var incoming = new OverlayTcpIncomingBuffer(); + + var readTask = incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask(); + + await Task.Delay(TimeSpan.FromMilliseconds(25)); + incoming.MarkRemoteFinReceived(); + + var eof = await readTask.WaitAsync(TimeSpan.FromSeconds(1)); + Assert.Equal(0, eof); + } } diff --git a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs index e3f48f5..4fa7ab3 100644 --- a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs +++ b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs @@ -8,7 +8,8 @@ internal sealed class OverlayTcpIncomingBuffer { private const int MaxQueuedSegments = 1024; private const int MaxSegmentLength = 1024; - private const int FinGracePeriodMs = 200; + private const int FinProbeGracePeriodMs = 50; + private const int FinLateDataGracePeriodMs = 200; private readonly Channel> _incoming = Channel.CreateBounded>(new BoundedChannelOptions(MaxQueuedSegments) { @@ -16,9 +17,11 @@ internal sealed class OverlayTcpIncomingBuffer SingleWriter = false, SingleReader = true }); + private readonly TaskCompletionSource _finArrived = new(TaskCreationOptions.RunContinuationsAsynchronously); private ReadOnlyMemory _currentSegment; private int _currentSegmentOffset; private int _remoteFinReceived; + private long _remoteFinReceivedAtMs; private int _remoteClosed; private long _lastActivityMs; private IOException? _fault; @@ -69,7 +72,8 @@ public void MarkRemoteFinReceived() { if (Interlocked.CompareExchange(ref _remoteFinReceived, 1, 0) == 0) { - Volatile.Write(ref _lastActivityMs, Environment.TickCount64); + Volatile.Write(ref _remoteFinReceivedAtMs, Environment.TickCount64); + _finArrived.TrySetResult(true); } } @@ -95,9 +99,9 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can { while (true) { - if (_incoming.Reader.TryRead(out var segment)) + if (_incoming.Reader.TryRead(out var nextSegment)) { - _currentSegment = segment; + _currentSegment = nextSegment; _currentSegmentOffset = 0; break; } @@ -121,11 +125,14 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can try { + ReadOnlyMemory segment; if (Volatile.Read(ref _remoteFinReceived) != 0) { var now = Environment.TickCount64; + var finReceivedAtMs = Volatile.Read(ref _remoteFinReceivedAtMs); var last = Volatile.Read(ref _lastActivityMs); - var remainingGraceMs = FinGracePeriodMs - (int)unchecked(now - last); + var gracePeriodMs = last > finReceivedAtMs ? FinLateDataGracePeriodMs : FinProbeGracePeriodMs; + var remainingGraceMs = gracePeriodMs - (int)unchecked(now - last); if (remainingGraceMs <= 0) { MarkRemoteClosed(); @@ -136,9 +143,23 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can graceCts.CancelAfter(TimeSpan.FromMilliseconds(remainingGraceMs)); try { - _currentSegment = await _incoming.Reader.ReadAsync(graceCts.Token).ConfigureAwait(false); - _currentSegmentOffset = 0; - break; + while (await _incoming.Reader.WaitToReadAsync(graceCts.Token).ConfigureAwait(false)) + { + if (_incoming.Reader.TryRead(out segment)) + { + _currentSegment = segment; + _currentSegmentOffset = 0; + break; + } + } + + if (_currentSegment.Length != 0) + { + break; + } + + MarkRemoteClosed(); + return 0; } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && graceCts.IsCancellationRequested) { @@ -147,9 +168,24 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can } } - _currentSegment = await _incoming.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); - _currentSegmentOffset = 0; - break; + var waitToReadTask = _incoming.Reader.WaitToReadAsync(cancellationToken).AsTask(); + var completed = await Task.WhenAny(waitToReadTask, _finArrived.Task).ConfigureAwait(false); + if (completed != waitToReadTask) + { + continue; + } + + if (!await waitToReadTask.ConfigureAwait(false)) + { + throw new ChannelClosedException(); + } + + if (_incoming.Reader.TryRead(out segment)) + { + _currentSegment = segment; + _currentSegmentOffset = 0; + break; + } } catch (ChannelClosedException) { From e5219495d99370fbdb9167ffd6f9d930a4f9ea4c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:27:55 +0100 Subject: [PATCH 040/296] UDP socket: accept any local managed dst + bound channel --- .../ZeroTierUdpSocketReceiveTests.cs | 70 +++++++++++++++++++ ZTSharp/ZeroTier/ZeroTierUdpSocket.cs | 13 +++- 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs diff --git a/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs b/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs new file mode 100644 index 0000000..8575cef --- /dev/null +++ b/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs @@ -0,0 +1,70 @@ +using System.Net; +using ZTSharp.ZeroTier; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Net; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierUdpSocketReceiveTests +{ + [Fact] + public async Task ReceiveFromAsync_WildcardSemantics_AllowAnyLocalManagedIp() + { + var localIpA = IPAddress.Parse("10.0.0.2"); + var localIpB = IPAddress.Parse("10.0.0.3"); + await using var runtime = CreateRuntime(localIpA, localIpB); + var ip = GetIpHandler(runtime); + + const ushort localPort = 12010; + await using var socket = new ZeroTierUdpSocket(runtime, localIpA, localPort); + + var buffer = new byte[32]; + var receiveTask = socket.ReceiveFromAsync(buffer, TimeSpan.FromSeconds(1)).AsTask(); + + var remoteIp = IPAddress.Parse("10.0.0.1"); + var payload = new byte[] { 1, 2, 3, 4 }; + var udp = UdpCodec.Encode(remoteIp, localIpB, sourcePort: 1111, destinationPort: localPort, payload); + var ipv4 = Ipv4Codec.Encode(remoteIp, localIpB, protocol: UdpCodec.ProtocolNumber, payload: udp, identification: 1); + + await ip.HandleIpv4PacketAsync(peerNodeId: new NodeId(0x3333333333), ipv4Packet: ipv4, cancellationToken: CancellationToken.None); + + var received = await receiveTask; + Assert.Equal(payload.Length, received.ReceivedBytes); + Assert.Equal(new IPEndPoint(remoteIp, 1111), received.RemoteEndPoint); + Assert.Equal(payload, buffer.AsSpan(0, payload.Length).ToArray()); + } + + private static ZeroTierDataplaneIpHandler GetIpHandler(ZeroTierDataplaneRuntime runtime) + { + var peerPackets = GetPrivateField(runtime, "_peerPackets"); + return GetPrivateField(peerPackets, "_ip"); + } + + private static T GetPrivateField(object instance, string fieldName) + { + var field = instance.GetType().GetField(fieldName, System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + Assert.NotNull(field); + var value = field!.GetValue(instance); + Assert.NotNull(value); + return (T)value!; + } + + [global::System.Diagnostics.CodeAnalysis.SuppressMessage( + "Reliability", + "CA2000:Dispose objects before losing scope", + Justification = "UDP transport ownership transfers to ZeroTierDataplaneRuntime, which is disposed by the caller.")] + private static ZeroTierDataplaneRuntime CreateRuntime(IPAddress localManagedIpV4A, IPAddress localManagedIpV4B) + => new( + udp: new ZeroTierUdpTransport(localPort: 0, enableIpv6: false), + rootNodeId: new NodeId(0x1111111111), + rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootKey: new byte[48], + rootProtocolVersion: 12, + localIdentity: ZeroTierTestIdentities.CreateFastIdentity(0x2222222222), + networkId: 1, + localManagedIpsV4: new[] { localManagedIpV4A, localManagedIpV4B }, + localManagedIpsV6: Array.Empty(), + inlineCom: new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); +} + diff --git a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs index d781bce..77ab20d 100644 --- a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs @@ -10,8 +10,15 @@ namespace ZTSharp.ZeroTier; public sealed class ZeroTierUdpSocket : IAsyncDisposable { + private const int MaxQueuedDatagrams = 1024; + private readonly SemaphoreSlim _disposeLock = new(1, 1); - private readonly Channel _incoming = Channel.CreateUnbounded(); + private readonly Channel _incoming = Channel.CreateBounded(new BoundedChannelOptions(MaxQueuedDatagrams) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleWriter = false, + SingleReader = true + }); private readonly ZeroTierDataplaneRuntime _runtime; private readonly IPAddress _localAddress; private readonly ushort _localPort; @@ -133,7 +140,7 @@ public async ValueTask ReceiveFromAsync( continue; } - if (!dst.Equals(_localAddress) || protocol != UdpCodec.ProtocolNumber) + if (protocol != UdpCodec.ProtocolNumber) { continue; } @@ -145,7 +152,7 @@ public async ValueTask ReceiveFromAsync( continue; } - if (!dst.Equals(_localAddress) || protocol != UdpCodec.ProtocolNumber) + if (protocol != UdpCodec.ProtocolNumber) { continue; } From ddc4e2bb6907a8bdb3e28470c02e05c01baa9da9 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:28:59 +0100 Subject: [PATCH 041/296] IPv6: trace drop for non-atomic fragments --- ZTSharp/ZeroTier/Net/Ipv6Codec.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs index 25715e6..aa3f7c7 100644 --- a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs +++ b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Net; +using ZTSharp.ZeroTier.Internal; namespace ZTSharp.ZeroTier.Net; @@ -194,6 +195,11 @@ private static bool TryWalkExtensionHeaders( var moreFragments = (fragmentOffsetAndFlags & 0x1) != 0; if (fragmentOffset != 0 || moreFragments || reservedBits != 0) { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine("[zerotier] Drop: IPv6 fragments are not supported."); + } + return false; } From f3fb0819ae3ca6015ee27021f827e97a0b48d734 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:30:14 +0100 Subject: [PATCH 042/296] Multipath: validate UdpSocketCount >= 1 --- .../ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs | 8 ++++++++ .../Internal/ZeroTierSocketRuntimeBootstrapper.cs | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index 3aa7e05..86ecdd7 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -39,6 +39,14 @@ public async Task CreateUdpTransport_MultipathEnabled_UsesMultipleSockets() } } + [Fact] + public void CreateUdpTransport_MultipathEnabled_RequiresAtLeastOneSocket() + { + Assert.Throws(() => ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 0 }, + enableIpv6: false)); + } + [Fact] public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 6074e28..77ba2c6 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -18,6 +18,11 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption return new ZeroTierUdpTransport(localPort: 0, enableIpv6: enableIpv6, localSocketId: 0); } + if (multipath.UdpSocketCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(multipath), multipath.UdpSocketCount, "UdpSocketCount must be at least 1 when multipath is enabled."); + } + if (multipath.UdpSocketCount == 1) { var port = 0; From d3ff1c5aed38753b17278a252f98580693f438df Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:31:51 +0100 Subject: [PATCH 043/296] Peer negotiation: TTL-check remote utilities --- .../ZeroTierPeerPathNegotiationManagerTests.cs | 16 ++++++++++++++++ .../ZeroTierPeerPathNegotiationManager.cs | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs index 48aac34..cdb3bea 100644 --- a/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs @@ -21,6 +21,22 @@ public void HandleInboundRequest_StoresUtilityPerPath() Assert.Equal((short)123, utility); } + [Fact] + public void TryGetRemoteUtility_DoesNotReturnStaleUtilityBeyondTtl() + { + var now = 1_000L; + var mgr = new ZeroTierPeerPathNegotiationManager(nowMs: () => now); + + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("203.0.113.9"), 9999); + + mgr.HandleInboundRequest(peerNodeId, localSocketId: 2, endpoint, remoteUtility: 123); + + now += 300_001; + + Assert.False(mgr.TryGetRemoteUtility(peerNodeId, localSocketId: 2, endpoint, out _)); + } + [Fact] public void TryMarkSent_RateLimitsPerPath() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs index 9b8f7be..5165348 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs @@ -36,9 +36,18 @@ public bool TryGetRemoteUtility(NodeId peerNodeId, int localSocketId, IPEndPoint { ArgumentNullException.ThrowIfNull(remoteEndPoint); + var now = _nowMs(); + CleanupIfNeeded(now); var key = new ZeroTierPeerNegotiationPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); if (_state.TryGetValue(key, out var state)) { + if (state.LastReceivedMs != 0 && unchecked(now - state.LastReceivedMs) > NegotiationStateTtlMs) + { + _state.TryRemove(key, out _); + remoteUtility = 0; + return false; + } + remoteUtility = state.RemoteUtility; return true; } From 0d66a4f6991f0cefb6503fcabeec7a2ef04e6fa4 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:49:35 +0100 Subject: [PATCH 044/296] Overlay TCP: fix FIN grace overflow --- ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs | 30 ++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs index 4fa7ab3..43a81c3 100644 --- a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs +++ b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs @@ -72,7 +72,23 @@ public void MarkRemoteFinReceived() { if (Interlocked.CompareExchange(ref _remoteFinReceived, 1, 0) == 0) { - Volatile.Write(ref _remoteFinReceivedAtMs, Environment.TickCount64); + var finReceivedAtMs = Environment.TickCount64; + Volatile.Write(ref _remoteFinReceivedAtMs, finReceivedAtMs); + + while (true) + { + var lastActivityMs = Volatile.Read(ref _lastActivityMs); + if (lastActivityMs >= finReceivedAtMs) + { + break; + } + + if (Interlocked.CompareExchange(ref _lastActivityMs, finReceivedAtMs, lastActivityMs) == lastActivityMs) + { + break; + } + } + _finArrived.TrySetResult(true); } } @@ -130,9 +146,16 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can { var now = Environment.TickCount64; var finReceivedAtMs = Volatile.Read(ref _remoteFinReceivedAtMs); - var last = Volatile.Read(ref _lastActivityMs); + var lastActivityMs = Volatile.Read(ref _lastActivityMs); + var last = Math.Max(lastActivityMs, finReceivedAtMs); var gracePeriodMs = last > finReceivedAtMs ? FinLateDataGracePeriodMs : FinProbeGracePeriodMs; - var remainingGraceMs = gracePeriodMs - (int)unchecked(now - last); + var elapsedMs = now - last; + if (elapsedMs < 0) + { + elapsedMs = 0; + } + + var remainingGraceMs = gracePeriodMs - elapsedMs; if (remainingGraceMs <= 0) { MarkRemoteClosed(); @@ -140,6 +163,7 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can } using var graceCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + remainingGraceMs = Math.Min(remainingGraceMs, gracePeriodMs); graceCts.CancelAfter(TimeSpan.FromMilliseconds(remainingGraceMs)); try { From 0bbcd22afca8d1b8f4060f0f57d0dc1ecc9bd988 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:50:01 +0100 Subject: [PATCH 045/296] OS UDP: keep empty peer maps for joined networks --- ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 9ce32e9..44cca4d 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -211,7 +211,7 @@ private static void EvictExpiredAndTrimPeers(ConcurrentDictionary peers, long nowTicks) { EvictExpiredAndTrimPeers(peers, nowTicks); - if (peers.IsEmpty) + if (peers.IsEmpty && (!_localNodeIds.TryGetValue(networkId, out var localNodeId) || localNodeId == 0)) { _networkPeers.TryRemove(networkId, out _); } From 2768e96d8c0bf151d7829f4144304422908efbe8 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:50:14 +0100 Subject: [PATCH 046/296] OS UDP: refresh peer last-seen on send --- ZTSharp/Transport/OsUdpNodeTransport.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index edd5739..e006276 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -161,6 +161,7 @@ public async Task SendFrameAsync( await _udp .SendAsync(frame, peer.Value.Endpoint, cancellationToken) .ConfigureAwait(false); + _peers.RefreshPeerLastSeen(networkId, peer.Key); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { From 23dd10c3c01e8385721e9a0a96f9467e8613e56e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:51:01 +0100 Subject: [PATCH 047/296] OS UDP: avoid sort in hello-response eviction --- .../Transport/Internal/OsUdpReceiveLoop.cs | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index 22c2bbc..c006127 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.Linq; using System.Net; using System.Net.Sockets; using System.Threading.Channels; @@ -26,6 +25,7 @@ private readonly record struct DiscoverySendRequest( private readonly Func _sendDiscoveryFrameAsync; private readonly Action? _log; private readonly Dictionary<(ulong NetworkId, ulong NodeId), long> _helloResponseLastSentMs = new(); + private readonly Queue<((ulong NetworkId, ulong NodeId) Key, long LastSentMs)> _helloResponseEvictionQueue = new(); private readonly Channel _discoverySendQueue; internal Action? DatagramReceivedForTests { get; set; } @@ -184,23 +184,37 @@ private bool ShouldSendHelloResponse(ulong networkId, ulong remoteNodeId) } _helloResponseLastSentMs[key] = nowMs; + _helloResponseEvictionQueue.Enqueue((key, nowMs)); if (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) { - var overflow = _helloResponseLastSentMs.Count - MaxHelloResponseCacheEntries; - var toRemove = _helloResponseLastSentMs - .OrderBy(pair => pair.Value) - .Take(overflow) - .Select(pair => pair.Key) - .ToArray(); - - for (var i = 0; i < toRemove.Length; i++) + TrimHelloResponseCache(); + } + + return true; + } + + private void TrimHelloResponseCache() + { + while (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries && _helloResponseEvictionQueue.Count > 0) + { + var (key, lastSentMs) = _helloResponseEvictionQueue.Dequeue(); + if (_helloResponseLastSentMs.TryGetValue(key, out var currentLastSentMs) && currentLastSentMs == lastSentMs) { - _helloResponseLastSentMs.Remove(toRemove[i]); + _helloResponseLastSentMs.Remove(key); } } - return true; + while (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) + { + using var enumerator = _helloResponseLastSentMs.Keys.GetEnumerator(); + if (!enumerator.MoveNext()) + { + break; + } + + _helloResponseLastSentMs.Remove(enumerator.Current); + } } private void QueueDiscoverySend( From 33d5adcd356cfca4b9f4dcd3eb495e60fe0eb807 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:51:38 +0100 Subject: [PATCH 048/296] Docs: correct UDP ephemeral port selection --- docs/ZEROTIER_SOCKETS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ZEROTIER_SOCKETS.md b/docs/ZEROTIER_SOCKETS.md index fd039f2..9643a19 100644 --- a/docs/ZEROTIER_SOCKETS.md +++ b/docs/ZEROTIER_SOCKETS.md @@ -131,7 +131,7 @@ There is no virtual network interface visible to the operating system. - `ZeroTierSocket.ListenTcpAsync(IPAddress.IPv6Any, port)` / `ManagedSocket.BindAsync(IPAddress.IPv6Any, port)` + `ListenAsync(...)` create a wildcard listener that accepts connections addressed to any of the node's managed IPv6s. - TCP connect may use `IPAddress.Any` / `IPAddress.IPv6Any` as the local endpoint to mean "choose a managed IP" (the selected local endpoint is returned by APIs that surface it). - TCP listen with port `0` is **not supported** (`NotSupportedException`). -- UDP bind supports port `0` (ephemeral); the bound local port is assigned by the OS and can be read back from the socket. +- UDP bind supports port `0` (ephemeral); the bound local port is selected internally via `ZeroTierEphemeralPorts.Generate()` and can be read back from the socket. ### Socket Options From 779bc2658ff4617b2b68171b7643061ebadada12 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 05:52:52 +0100 Subject: [PATCH 049/296] Tests: reduce timing and port flakiness --- ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs | 2 +- ...ierSocketRuntimeBootstrapperUdpTransportTests.cs | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs index 74aff6a..98c474e 100644 --- a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs +++ b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs @@ -46,7 +46,7 @@ public async Task FinArrival_UnblocksReaderAwaitingReadAsync() var readTask = incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask(); - await Task.Delay(TimeSpan.FromMilliseconds(25)); + await Task.Yield(); incoming.MarkRemoteFinReceived(); var eof = await readTask.WaitAsync(TimeSpan.FromSeconds(1)); diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index 86ecdd7..58c7a53 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Net.Sockets; using ZTSharp.ZeroTier; using ZTSharp.ZeroTier.Internal; @@ -50,10 +51,10 @@ public void CreateUdpTransport_MultipathEnabled_RequiresAtLeastOneSocket() [Fact] public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() { - const int attempts = 25; + const int attempts = 10; for (var i = 0; i < attempts; i++) { - var port = Random.Shared.Next(20_000, 60_000); + var port = GetAvailableUdpPort(); var transport = default(IZeroTierUdpTransport); try { @@ -77,6 +78,12 @@ public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() } } - Assert.Fail($"Failed to bind a UDP socket to a random port after {attempts} attempts."); + Assert.Fail($"Failed to bind a UDP socket to an available port after {attempts} attempts."); + } + + private static int GetAvailableUdpPort() + { + using var udp = new UdpClient(new IPEndPoint(IPAddress.Loopback, 0)); + return ((IPEndPoint)udp.Client.LocalEndPoint!).Port; } } From 1cd6f4313a4beda7dc7094618e47d629bfc867a3 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:10:30 +0100 Subject: [PATCH 050/296] fix(transport): bound hello-response eviction queue --- ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index c006127..8242c97 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -8,6 +8,7 @@ namespace ZTSharp.Transport.Internal; internal sealed class OsUdpReceiveLoop { private const int MaxHelloResponseCacheEntries = 4096; + private const int MaxHelloResponseEvictionQueueEntries = MaxHelloResponseCacheEntries * 2; private const int MaxPendingDiscoverySends = 256; private const long PeerHelloResponseMinIntervalMs = 1000; @@ -185,6 +186,7 @@ private bool ShouldSendHelloResponse(ulong networkId, ulong remoteNodeId) _helloResponseLastSentMs[key] = nowMs; _helloResponseEvictionQueue.Enqueue((key, nowMs)); + TrimHelloResponseEvictionQueueIfNeeded(); if (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) { @@ -194,6 +196,14 @@ private bool ShouldSendHelloResponse(ulong networkId, ulong remoteNodeId) return true; } + private void TrimHelloResponseEvictionQueueIfNeeded() + { + while (_helloResponseEvictionQueue.Count > MaxHelloResponseEvictionQueueEntries) + { + _helloResponseEvictionQueue.Dequeue(); + } + } + private void TrimHelloResponseCache() { while (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries && _helloResponseEvictionQueue.Count > 0) From 702b533ce3775155cfc5bb3f7cb6531412a5b42e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:10:38 +0100 Subject: [PATCH 051/296] docs: clarify UDP bind/receive semantics --- docs/ZEROTIER_SOCKETS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/ZEROTIER_SOCKETS.md b/docs/ZEROTIER_SOCKETS.md index 9643a19..01e5770 100644 --- a/docs/ZEROTIER_SOCKETS.md +++ b/docs/ZEROTIER_SOCKETS.md @@ -132,6 +132,8 @@ There is no virtual network interface visible to the operating system. - TCP connect may use `IPAddress.Any` / `IPAddress.IPv6Any` as the local endpoint to mean "choose a managed IP" (the selected local endpoint is returned by APIs that surface it). - TCP listen with port `0` is **not supported** (`NotSupportedException`). - UDP bind supports port `0` (ephemeral); the bound local port is selected internally via `ZeroTierEphemeralPorts.Generate()` and can be read back from the socket. +- UDP bind is currently **per address family + port** (not per IP + port). A UDP socket may receive datagrams destined to any of the node's managed IPs on that port. +- `ReceiveFromAsync(...)` does not currently surface the local destination IP. ### Socket Options @@ -154,3 +156,5 @@ Accepted sockets populate `ManagedSocket.RemoteEndPoint` (peer IP/port). The user-space TCP stack is correctness-oriented. It currently lacks OS optimizations like congestion control and high-throughput sender pipelines. Large transfers may be significantly slower than OS TCP. + +UDP receive queues are bounded and may drop datagrams under load. From 70cee3aad7b35423d1facc3527c1f8f54f23f5af Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:10:43 +0100 Subject: [PATCH 052/296] test: increase UDP port bind retry budget --- .../ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index 58c7a53..1624075 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -51,7 +51,7 @@ public void CreateUdpTransport_MultipathEnabled_RequiresAtLeastOneSocket() [Fact] public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() { - const int attempts = 10; + const int attempts = 25; for (var i = 0; i < attempts; i++) { var port = GetAvailableUdpPort(); From 3bf10938793dfacb4b80d9dfa247326bb9c163ff Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:29:41 +0100 Subject: [PATCH 053/296] fix(node): serialize concurrent network leaves --- .../NodeNetworkLeaveOrderingTests.cs | 76 +++++++++++++++++++ ZTSharp/Internal/NodeNetworkService.cs | 41 +++++++--- 2 files changed, 108 insertions(+), 9 deletions(-) diff --git a/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs b/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs index b77e175..0bb0bbd 100644 --- a/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs +++ b/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs @@ -46,6 +46,38 @@ await service.JoinNetworkAsync( Assert.Equal(2, received); } + [Fact] + public async Task LeaveNetworkAsync_ConcurrentCalls_InvokeTransportLeaveOnce() + { + var store = new MemoryStateStore(); + var transport = new BlockingLeaveTransport(); + var events = new NodeEventStream(_ => { }, NullLogger.Instance); + var peers = new NodePeerService(store); + var service = new NodeNetworkService(store, transport, events, peers); + + var networkId = 0xCAFE5002UL; + + await service.JoinNetworkAsync( + networkId, + localNodeId: 1, + localEndpoint: null, + onFrameReceived: (_, _, _, _) => Task.CompletedTask, + CancellationToken.None); + + var leave1 = service.LeaveNetworkAsync(networkId, CancellationToken.None); + await transport.LeaveEntered; + + var leave2 = service.LeaveNetworkAsync(networkId, CancellationToken.None); + + await Task.Delay(25); + Assert.Equal(1, transport.LeaveCallCount); + + transport.AllowLeaveToProceed(); + await Task.WhenAll(leave1, leave2); + + Assert.Equal(1, transport.LeaveCallCount); + } + private sealed class FlakyLeaveTransport : INodeTransport, IDisposable { public InMemoryNodeTransport Inner { get; } = new(); @@ -79,4 +111,48 @@ public Task FlushAsync(CancellationToken cancellationToken = default) public void Dispose() => Inner.Dispose(); } + + private sealed class BlockingLeaveTransport : INodeTransport, IDisposable + { + private readonly TaskCompletionSource _leaveEntered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _allowLeaveToProceed = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _leaveEnteredSet; + private int _leaveCallCount; + + public InMemoryNodeTransport Inner { get; } = new(); + + public int LeaveCallCount => Volatile.Read(ref _leaveCallCount); + + public Task LeaveEntered => _leaveEntered.Task; + + public void AllowLeaveToProceed() => _allowLeaveToProceed.TrySetResult(); + + public Task JoinNetworkAsync( + ulong networkId, + ulong nodeId, + Func, CancellationToken, Task> onFrameReceived, + System.Net.IPEndPoint? localEndpoint = null, + CancellationToken cancellationToken = default) + => Inner.JoinNetworkAsync(networkId, nodeId, onFrameReceived, localEndpoint, cancellationToken); + + public async Task LeaveNetworkAsync(ulong networkId, Guid registrationId, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _leaveCallCount); + if (Interlocked.Exchange(ref _leaveEnteredSet, 1) == 0) + { + _leaveEntered.TrySetResult(); + } + + await _allowLeaveToProceed.Task.ConfigureAwait(false); + await Inner.LeaveNetworkAsync(networkId, registrationId, cancellationToken).ConfigureAwait(false); + } + + public Task SendFrameAsync(ulong networkId, ulong sourceNodeId, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Inner.SendFrameAsync(networkId, sourceNodeId, payload, cancellationToken); + + public Task FlushAsync(CancellationToken cancellationToken = default) + => Inner.FlushAsync(cancellationToken); + + public void Dispose() => Inner.Dispose(); + } } diff --git a/ZTSharp/Internal/NodeNetworkService.cs b/ZTSharp/Internal/NodeNetworkService.cs index 2f06ca1..18ec675 100644 --- a/ZTSharp/Internal/NodeNetworkService.cs +++ b/ZTSharp/Internal/NodeNetworkService.cs @@ -16,6 +16,7 @@ internal sealed class NodeNetworkService private readonly ConcurrentDictionary _joinedNetworks = new(); private readonly ConcurrentDictionary _networkRegistrations = new(); + private readonly ConcurrentDictionary _leaveGates = new(); private readonly NetworkIdReadOnlyCollection _joinedNetworkIds; public NodeNetworkService(IStateStore store, INodeTransport transport, NodeEventStream events, NodePeerService peerService) @@ -85,17 +86,26 @@ public async Task JoinNetworkAsync( public async Task LeaveNetworkAsync(ulong networkId, CancellationToken cancellationToken) { var key = BuildNetworkFileKey(networkId); - if (_networkRegistrations.TryGetValue(networkId, out var registration)) + var gate = _leaveGates.GetOrAdd(networkId, static _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - await _transport.LeaveNetworkAsync(networkId, registration, cancellationToken).ConfigureAwait(false); - _networkRegistrations.TryRemove(networkId, out _); - } + if (_networkRegistrations.TryGetValue(networkId, out var registration)) + { + await _transport.LeaveNetworkAsync(networkId, registration, cancellationToken).ConfigureAwait(false); + _networkRegistrations.TryRemove(networkId, out _); + } - var removed = _joinedNetworks.TryRemove(networkId, out _); - if (removed) + var removed = _joinedNetworks.TryRemove(networkId, out _); + if (removed) + { + await _store.DeleteAsync(key, cancellationToken).ConfigureAwait(false); + _events.Publish(EventCode.NetworkLeft, DateTimeOffset.UtcNow, networkId); + } + } + finally { - await _store.DeleteAsync(key, cancellationToken).ConfigureAwait(false); - _events.Publish(EventCode.NetworkLeft, DateTimeOffset.UtcNow, networkId); + gate.Release(); } } @@ -213,7 +223,20 @@ public async Task UnregisterAllNetworksAsync(CancellationToken cancellationToken { foreach (var kv in _networkRegistrations) { - await _transport.LeaveNetworkAsync(kv.Key, kv.Value, cancellationToken).ConfigureAwait(false); + var gate = _leaveGates.GetOrAdd(kv.Key, static _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_networkRegistrations.TryGetValue(kv.Key, out var registration)) + { + await _transport.LeaveNetworkAsync(kv.Key, registration, cancellationToken).ConfigureAwait(false); + _networkRegistrations.TryRemove(kv.Key, out _); + } + } + finally + { + gate.Release(); + } } _networkRegistrations.Clear(); From 812f67fec2b587c4ae040ffc341e7ce7ec3c8838 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:29:59 +0100 Subject: [PATCH 054/296] fix(store): validate state root before mkdir --- ZTSharp/FileStateStore.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index bb4303e..0f6fe74 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -20,9 +20,8 @@ public FileStateStore(string rootPath) _rootPathTrimmed = Path.TrimEndingDirectorySeparator(_rootPath); _rootPathPrefix = _rootPathTrimmed + Path.DirectorySeparatorChar; _pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - Directory.CreateDirectory(_rootPath); - ThrowIfRootPathOrAncestorsAreReparsePoints(); + Directory.CreateDirectory(_rootPath); } public Task ExistsAsync(string key, CancellationToken cancellationToken = default) From ad305ead61b71a97668d2348131fe8ca9bc5878c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:57:35 +0100 Subject: [PATCH 055/296] fix(transport): refresh peer last-seen even on dispatch faults --- .../OsUdpReceiveLoopLastSeenRefreshTests.cs | 82 +++++++++++++++++++ .../Transport/Internal/OsUdpReceiveLoop.cs | 2 +- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 ZTSharp.Tests/OsUdpReceiveLoopLastSeenRefreshTests.cs diff --git a/ZTSharp.Tests/OsUdpReceiveLoopLastSeenRefreshTests.cs b/ZTSharp.Tests/OsUdpReceiveLoopLastSeenRefreshTests.cs new file mode 100644 index 0000000..880bcec --- /dev/null +++ b/ZTSharp.Tests/OsUdpReceiveLoopLastSeenRefreshTests.cs @@ -0,0 +1,82 @@ +using System.Net; +using System.Net.Sockets; +using ZTSharp.Transport; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.Tests; + +public sealed class OsUdpReceiveLoopLastSeenRefreshTests +{ + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public ManualTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta); + } + + [Fact] + public async Task ReceiveLoop_RefreshesPeerLastSeen_EvenWhenDispatchThrows() + { + using var udp = new UdpClient(AddressFamily.InterNetwork); + + var time = new ManualTimeProvider(DateTimeOffset.UnixEpoch); + var peers = new OsUdpPeerRegistry(enablePeerDiscovery: false, UdpEndpointNormalization.Normalize, timeProvider: time); + var networkId = 0xABCDEF10UL; + var sourceNodeId = 0x1234UL; + var remoteEndpoint = new IPEndPoint(IPAddress.Loopback, 40100); + peers.AddOrUpdatePeer(networkId, sourceNodeId, remoteEndpoint); + + time.Advance(TimeSpan.FromMinutes(4)); + var appPayload = new byte[] { 0x42 }; + var frame = NodeFrameCodec.Encode(networkId, sourceNodeId, appPayload); + var result = new UdpReceiveResult(frame.ToArray(), remoteEndpoint); + + var dispatched = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task DispatchAsync(ulong _, ulong __, ReadOnlyMemory ___, CancellationToken ____) + { + dispatched.TrySetResult(true); + throw new InvalidOperationException("synthetic dispatch fault"); + } + + static Task SendDiscoveryAsync(ulong _, ulong __, IPEndPoint ___, OsUdpPeerDiscoveryProtocol.FrameType ____, CancellationToken _____) + => Task.CompletedTask; + + var callCount = 0; + async ValueTask ReceiveAsync(CancellationToken ct) + { + if (Interlocked.Increment(ref callCount) == 1) + { + return result; + } + + await Task.Delay(Timeout.InfiniteTimeSpan, ct); + throw new OperationCanceledException(ct); + } + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var loop = new OsUdpReceiveLoop( + udp, + enablePeerDiscovery: false, + peers, + DispatchAsync, + SendDiscoveryAsync, + receiveAsync: ReceiveAsync); + + var run = loop.RunAsync(cts.Token); + _ = await dispatched.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + time.Advance(TimeSpan.FromMinutes(2)); + Assert.True(peers.TryGetPeers(networkId, out var networkPeers)); + Assert.True(networkPeers.ContainsKey(sourceNodeId)); + + cts.Cancel(); + await run; + } +} diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index 8242c97..9d02700 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -145,8 +145,8 @@ public async Task RunAsync(CancellationToken cancellationToken) try { - await _dispatchFrameAsync(sourceNodeId, networkId, payload, cancellationToken).ConfigureAwait(false); _peers.RefreshPeerLastSeen(networkId, sourceNodeId); + await _dispatchFrameAsync(sourceNodeId, networkId, payload, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { From 4b9c04a4a302e53f0fd1f3fd17702776a2f3359d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:58:47 +0100 Subject: [PATCH 056/296] fix(transport): avoid evicting locally-registered networks --- .../OsUdpPeerRegistryNetworkTrimTests.cs | 44 +++++++++++++++++++ .../Transport/Internal/OsUdpPeerRegistry.cs | 1 + 2 files changed, 45 insertions(+) create mode 100644 ZTSharp.Tests/OsUdpPeerRegistryNetworkTrimTests.cs diff --git a/ZTSharp.Tests/OsUdpPeerRegistryNetworkTrimTests.cs b/ZTSharp.Tests/OsUdpPeerRegistryNetworkTrimTests.cs new file mode 100644 index 0000000..f06aa45 --- /dev/null +++ b/ZTSharp.Tests/OsUdpPeerRegistryNetworkTrimTests.cs @@ -0,0 +1,44 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.Tests; + +[Collection("OsUdpPeerRegistry")] +public sealed class OsUdpPeerRegistryNetworkTrimTests +{ + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public ManualTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta); + } + + [Fact] + public void SweepNetworkPeers_DoesNotEvictNetworks_WithLocalRegistration() + { + var time = new ManualTimeProvider(DateTimeOffset.UnixEpoch); + var registry = new OsUdpPeerRegistry(enablePeerDiscovery: false, UdpEndpointNormalization.Normalize, timeProvider: time); + + const int extraNetworks = 16; + var totalNetworks = 256 + extraNetworks; + + for (var i = 0; i < totalNetworks; i++) + { + var networkId = (ulong)(0x9000 + i); + registry.SetLocalNodeId(networkId, nodeId: 1); + registry.AddOrUpdatePeer(networkId, nodeId: (ulong)(0xA000 + i), endpoint: new IPEndPoint(IPAddress.Loopback, 10000 + i)); + time.Advance(TimeSpan.FromSeconds(1)); + } + + var firstNetworkId = 0x9000UL; + Assert.True(registry.TryGetPeers(firstNetworkId, out _)); + } +} + diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 44cca4d..4b00c7d 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -231,6 +231,7 @@ private void SweepNetworkPeers(long nowTicks) var overflow = _networkPeers.Count - DirectoryMaxNetworks; var toRemove = _networkPeers + .Where(network => !_localNodeIds.TryGetValue(network.Key, out var localNodeId) || localNodeId == 0) .Select(network => { var lastSeen = 0L; From 9c4ab557810ba8ab17c6157e3b5fd7952ffc6c62 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:59:24 +0100 Subject: [PATCH 057/296] fix(net): validate UDP checksum using header length --- .../ZeroTierDataplaneUdpChecksumTests.cs | 23 +++++++++++++++++++ ZTSharp/ZeroTier/Net/UdpCodec.cs | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs b/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs index a121fd0..7d72d60 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs @@ -51,6 +51,29 @@ public async Task ValidUdpChecksum_IsDelivered_ToUdpHandler() Assert.Equal(new NodeId(0x3333333333), delivered.PeerNodeId); } + [Fact] + public async Task ValidUdpChecksum_WithTrailingBytes_IsDelivered_ToUdpHandler() + { + var localManagedIpV4 = IPAddress.Parse("10.0.0.2"); + await using var runtime = CreateRuntime(localManagedIpV4); + var ip = GetIpHandler(runtime); + + const ushort localPort = 12002; + var udpChannel = Channel.CreateUnbounded(); + Assert.True(runtime.TryRegisterUdpPort(AddressFamily.InterNetwork, localPort, udpChannel.Writer)); + + var remoteIp = IPAddress.Parse("10.0.0.1"); + var udp = UdpCodec.Encode(remoteIp, localManagedIpV4, sourcePort: 1111, destinationPort: localPort, payload: new byte[] { 1, 2, 3 }); + var padded = new byte[udp.Length + 8]; + udp.CopyTo(padded, 0); + + var ipv4 = Ipv4Codec.Encode(remoteIp, localManagedIpV4, UdpCodec.ProtocolNumber, padded, identification: 1); + await ip.HandleIpv4PacketAsync(peerNodeId: new NodeId(0x3333333333), ipv4Packet: ipv4, cancellationToken: CancellationToken.None); + + Assert.True(udpChannel.Reader.TryRead(out var delivered)); + Assert.Equal(new NodeId(0x3333333333), delivered.PeerNodeId); + } + private static ZeroTierDataplaneIpHandler GetIpHandler(ZeroTierDataplaneRuntime runtime) { var peerPackets = GetPrivateField(runtime, "_peerPackets"); diff --git a/ZTSharp/ZeroTier/Net/UdpCodec.cs b/ZTSharp/ZeroTier/Net/UdpCodec.cs index 7c05aec..68b0332 100644 --- a/ZTSharp/ZeroTier/Net/UdpCodec.cs +++ b/ZTSharp/ZeroTier/Net/UdpCodec.cs @@ -112,9 +112,10 @@ public static bool TryParseWithChecksum( return false; // IPv6 UDP checksum is mandatory } + var udpLength = HeaderLength + payload.Length; try { - return ComputeChecksum(sourceIp, destinationIp, segment) == 0; + return ComputeChecksum(sourceIp, destinationIp, segment.Slice(0, udpLength)) == 0; } catch (ArgumentOutOfRangeException) { From 6fd46fd5c76f55ddff7ab4f60f832c35a0327e5c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:00:36 +0100 Subject: [PATCH 058/296] fix(store): re-check root reparse point on each access --- ZTSharp/FileStateStore.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index 0f6fe74..e3af16f 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -330,6 +330,11 @@ private bool IsUnderRoot(string fullPath) private void ThrowIfPathTraversesReparsePoint(string fullPath) { + if (IsReparsePoint(_rootPathTrimmed)) + { + throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); + } + if (string.Equals(fullPath, _rootPathTrimmed, _pathComparison)) { return; @@ -396,6 +401,11 @@ private void ThrowIfRootPathOrAncestorsAreReparsePoints() private void EnsureParentDirectoryExistsNoReparse(string fullPath) { + if (IsReparsePoint(_rootPathTrimmed)) + { + throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); + } + var directory = Path.GetDirectoryName(fullPath); if (string.IsNullOrWhiteSpace(directory) || string.Equals(directory, _rootPathTrimmed, _pathComparison)) { From 552d6b33323e056753b02f27df3467bbdaa9ce17 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:00:41 +0100 Subject: [PATCH 059/296] fix(io): strip BOMs in bounded text reads --- ZTSharp.Tests/BoundedFileIOBomTests.cs | 29 ++++++++++++++++++++++++++ ZTSharp/Internal/BoundedFileIO.cs | 4 +++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 ZTSharp.Tests/BoundedFileIOBomTests.cs diff --git a/ZTSharp.Tests/BoundedFileIOBomTests.cs b/ZTSharp.Tests/BoundedFileIOBomTests.cs new file mode 100644 index 0000000..8267c0d --- /dev/null +++ b/ZTSharp.Tests/BoundedFileIOBomTests.cs @@ -0,0 +1,29 @@ +using System.Text; +using ZTSharp.Internal; + +namespace ZTSharp.Tests; + +public sealed class BoundedFileIOBomTests +{ + [Fact] + public void TryReadAllText_StripsUtf8Bom() + { + var path = TestTempPaths.CreateGuidSuffixed("bounded-io-bom-"); + try + { + var payload = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes("hello")).ToArray(); + File.WriteAllBytes(path, payload); + + Assert.True(BoundedFileIO.TryReadAllText(path, maxBytes: 1024, Encoding.UTF8, out var text)); + Assert.Equal("hello", text); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} + diff --git a/ZTSharp/Internal/BoundedFileIO.cs b/ZTSharp/Internal/BoundedFileIO.cs index 8a16112..67a0d47 100644 --- a/ZTSharp/Internal/BoundedFileIO.cs +++ b/ZTSharp/Internal/BoundedFileIO.cs @@ -67,7 +67,9 @@ public static bool TryReadAllText(string path, int maxBytes, Encoding encoding, return false; } - text = encoding.GetString(bytes); + using var stream = new MemoryStream(bytes, writable: false); + using var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true); + text = reader.ReadToEnd(); return true; } } From 081a03bfbc9dffd4e95ac53a531350eaaee1ebe2 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:01:19 +0100 Subject: [PATCH 060/296] fix(sockets): dispose stream if connect canceled mid-init --- ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs index b2468ad..b4c74d6 100644 --- a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs +++ b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs @@ -188,7 +188,15 @@ public override async ValueTask ConnectAsync(EndPoint remoteEndPoint, Cancellati ? await Zt.ConnectTcpWithLocalEndpointAsync(remoteIp, token).ConfigureAwait(false) : await Zt.ConnectTcpWithLocalEndpointAsync(local, remoteIp, token).ConfigureAwait(false); - await InitLock.WaitAsync(token).ConfigureAwait(false); + try + { + await InitLock.WaitAsync(token).ConfigureAwait(false); + } + catch + { + await stream.DisposeAsync().ConfigureAwait(false); + throw; + } try { ObjectDisposedException.ThrowIf(Disposed, this); From a35921097fb0d4e91a9617c28d1f94e65da078a2 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:01:56 +0100 Subject: [PATCH 061/296] fix(tcp): prevent negative PendingAcceptCount race --- ZTSharp/ZeroTier/ZeroTierTcpListener.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs index 0768ae3..779847e 100644 --- a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs +++ b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs @@ -211,8 +211,10 @@ private async Task HandleAcceptedConnectionAsync( await connection.AcceptAsync(token).ConfigureAwait(false); stream = connection.GetStream(); + Interlocked.Increment(ref _pendingAcceptCount); if (!_acceptQueue.Writer.TryWrite(new AcceptedTcpConnection(stream, localEndpoint, remoteEndpoint))) { + Interlocked.Decrement(ref _pendingAcceptCount); Interlocked.Increment(ref _droppedAcceptCount); if (ZeroTierTrace.Enabled) { @@ -225,7 +227,6 @@ private async Task HandleAcceptedConnectionAsync( return; } - Interlocked.Increment(ref _pendingAcceptCount); handedOff = true; stream = null; } From c906d131f82c0651b8b5e6b55b35aae9abc93e3e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:02:17 +0100 Subject: [PATCH 062/296] fix(multipath): handle multi-send SocketExceptions consistently --- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index af105f3..bbcc690 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -358,7 +358,7 @@ private async ValueTask SendToPeerAsync( rootSend = _udp.SendAsync(_rootEndpoint, packet, cancellationToken); await Task.WhenAll(directSend, rootSend).ConfigureAwait(false); } - catch (SocketException) + catch (Exception ex) when (ex is SocketException || (ex is AggregateException ae && ae.InnerExceptions.All(static inner => inner is SocketException))) { // Warm-up duplicates to root. If the direct send fails, forget any QoS state for it and // ensure a root send happens (if it hasn't already). From 8c6b1179ad06cd92c0293e06bd9b7912e3b9b763 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:04:34 +0100 Subject: [PATCH 063/296] fix(dataplane): enforce root-relayed RX when multipath off --- ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs | 77 ++++++++++++++++++- .../ZeroTierDataplaneShutdownTests.cs | 5 +- .../Internal/ZeroTierDataplaneRuntime.cs | 1 + .../Internal/ZeroTierDataplaneRxLoops.cs | 8 ++ 4 files changed, 85 insertions(+), 6 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs index 50ae769..d6e891e 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs @@ -188,7 +188,7 @@ public async Task PeerLoopAsync_ContinuesAfterProcessorFault() } [Fact] - public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint() + public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint_WhenDirectPeerRxEnabled() { await using var rxUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var rootSender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); @@ -215,7 +215,8 @@ public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint() rootKey: new byte[48], localNodeId, rootClient, - peerDatagrams: new NoopPeerDatagrams()); + peerDatagrams: new NoopPeerDatagrams(), + acceptDirectPeerDatagrams: true); var peerChannel = Channel.CreateUnbounded(new UnboundedChannelOptions { @@ -257,16 +258,84 @@ await Assert.ThrowsAsync(async () => await dispatcher.WaitAsync(TimeSpan.FromSeconds(2)); } + [Fact] + public async Task DispatcherLoopAsync_ForwardsOnlyRootPeerDatagrams_WhenDirectPeerRxDisabled() + { + await using var rxUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var rootSender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var attackerSender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var rootEndpoint = TestUdpEndpoints.ToLoopback(rootSender.LocalEndpoint); + var rootNodeId = new NodeId(0x1111111111); + var localNodeId = new NodeId(0x2222222222); + + var rootClient = new ZeroTierDataplaneRootClient( + rxUdp, + rootNodeId, + rootEndpoint, + rootKey: new byte[48], + rootProtocolVersion: 12, + localNodeId, + networkId: 1, + inlineCom: Array.Empty()); + + var loops = new ZeroTierDataplaneRxLoops( + rxUdp, + rootNodeId, + rootEndpoint, + rootKey: new byte[48], + localNodeId, + rootClient, + peerDatagrams: new NoopPeerDatagrams(), + acceptDirectPeerDatagrams: false); + + var peerChannel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = true + }); + + using var cts = new CancellationTokenSource(); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + + var peerNodeId = new NodeId(0x3333333333); + var peerPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localNodeId, + Source: peerNodeId, + Flags: 0, + Mac: 0, + VerbRaw: 0), + ReadOnlySpan.Empty); + + var receiverEndpoint = TestUdpEndpoints.ToLoopback(rxUdp.LocalEndpoint); + await attackerSender.SendAsync(receiverEndpoint, peerPacket); + await rootSender.SendAsync(receiverEndpoint, peerPacket); + + var forwarded = await ReadAsyncWithTimeout(peerChannel.Reader, TimeSpan.FromSeconds(2)); + Assert.Equal(rootEndpoint, forwarded.RemoteEndPoint); + + await Assert.ThrowsAsync(async () => + { + _ = await ReadAsyncWithTimeout(peerChannel.Reader, TimeSpan.FromMilliseconds(100)); + }); + + cts.Cancel(); + await dispatcher.WaitAsync(TimeSpan.FromSeconds(2)); + } + [Fact] public async Task DispatcherLoopAsync_DropsPeerDatagrams_WhenPeerQueueIsFull_AndCountsDrops() { await using var rxUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var sender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + var rootEndpoint = TestUdpEndpoints.ToLoopback(sender.LocalEndpoint); var rootClient = new ZeroTierDataplaneRootClient( rxUdp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], rootProtocolVersion: 12, localNodeId: new NodeId(0x2222222222), @@ -277,7 +346,7 @@ public async Task DispatcherLoopAsync_DropsPeerDatagrams_WhenPeerQueueIsFull_And var loops = new ZeroTierDataplaneRxLoops( rxUdp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], localNodeId: new NodeId(0x2222222222), rootClient: rootClient, diff --git a/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs b/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs index 41fd306..644b406 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs @@ -14,10 +14,11 @@ public async Task DispatcherLoopAsync_ExitsCleanly_WhenUdpTransportIsDisposed() await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var sender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + var rootEndpoint = TestUdpEndpoints.ToLoopback(sender.LocalEndpoint); var rootClient = new ZeroTierDataplaneRootClient( udp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], rootProtocolVersion: 12, localNodeId: new NodeId(0x2222222222), @@ -27,7 +28,7 @@ public async Task DispatcherLoopAsync_ExitsCleanly_WhenUdpTransportIsDisposed() var loops = new ZeroTierDataplaneRxLoops( udp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], localNodeId: new NodeId(0x2222222222), rootClient: rootClient, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index bbcc690..078d349 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -196,6 +196,7 @@ public ZeroTierDataplaneRuntime( _localIdentity.NodeId, _rootClient, _peerDatagrams, + acceptDirectPeerDatagrams: multipath.Enabled, handleRootControlAsync: HandleRootControlPacketAsync, onPeerQueueDrop: () => { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index a5945c2..856b0a4 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -16,6 +16,7 @@ internal sealed class ZeroTierDataplaneRxLoops private readonly IZeroTierDataplanePeerDatagramProcessor _peerDatagrams; private readonly Func, IPEndPoint, CancellationToken, ValueTask>? _handleRootControlAsync; private readonly Action? _onPeerQueueDrop; + private readonly bool _acceptDirectPeerDatagrams; private int _traceRxRemaining = 200; @@ -27,6 +28,7 @@ public ZeroTierDataplaneRxLoops( NodeId localNodeId, ZeroTierDataplaneRootClient rootClient, IZeroTierDataplanePeerDatagramProcessor peerDatagrams, + bool acceptDirectPeerDatagrams = false, Func, IPEndPoint, CancellationToken, ValueTask>? handleRootControlAsync = null, Action? onPeerQueueDrop = null) { @@ -43,6 +45,7 @@ public ZeroTierDataplaneRxLoops( _localNodeId = localNodeId; _rootClient = rootClient; _peerDatagrams = peerDatagrams; + _acceptDirectPeerDatagrams = acceptDirectPeerDatagrams; _handleRootControlAsync = handleRootControlAsync; _onPeerQueueDrop = onPeerQueueDrop; } @@ -132,6 +135,11 @@ public async Task DispatcherLoopAsync(ChannelWriter peerWri continue; } + if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) + { + continue; + } + if (!peerWriter.TryWrite(datagram)) { if (cancellationToken.IsCancellationRequested) From f7e83de3a0a8691331b8b3f68392dea237e4d910 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:04:58 +0100 Subject: [PATCH 064/296] test: relax concurrency deadlines to reduce flakiness --- ZTSharp.Tests/ChannelWriterConcurrencyTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs index 2aee83c..69179ff 100644 --- a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs +++ b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs @@ -34,9 +34,9 @@ public async Task InMemoryZtUdpClient_ReceivesDatagrams_UnderConcurrentSends() sendTasks[i] = sender.SendToAsync(payload, receiverId, remotePort: 30000); } - await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(2)); + await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(10)); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var seen = new bool[count]; for (var i = 0; i < count; i++) { @@ -183,11 +183,11 @@ void CaptureSyn(in RawFrame frame) }); } - await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(2)); + await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(10)); var clientStream = client.GetStream(); var buffer = new byte[frames * payload.Length]; - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var read = await StreamTestHelpers.ReadExactAsync(clientStream, buffer, buffer.Length, cts.Token); Assert.Equal(buffer.Length, read); } From 90e09109499f20db49d62acc039d52dacb5d6cff Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:05:29 +0100 Subject: [PATCH 065/296] fix(udp): drop when incoming queue is full to avoid stalls --- .../ZeroTier/Transport/ZeroTierUdpTransport.cs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index cb3abb4..9b0be23 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -130,23 +130,7 @@ private async Task ProcessReceiveLoopAsync() UdpEndpointNormalization.Normalize(result.RemoteEndPoint), result.Buffer))) { - try - { - var write = _incoming.Writer.WriteAsync(new ZeroTierUdpDatagram( - _localSocketId, - UdpEndpointNormalization.Normalize(result.RemoteEndPoint), - result.Buffer), token); - if (!write.IsCompletedSuccessfully) - { - Interlocked.Increment(ref _incomingBackpressureCount); - } - - await write.ConfigureAwait(false); - } - catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException or ChannelClosedException) - { - return; - } + Interlocked.Increment(ref _incomingBackpressureCount); } } } From abf0edecbd2fc9d47c4e25966678f2eba9cda075 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:26:11 +0100 Subject: [PATCH 066/296] fix(socket): dispose created runtime on mid-creation cancellation --- ZTSharp/ZeroTier/ZeroTierSocket.cs | 59 +++++++++++++++++++----------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/ZTSharp/ZeroTier/ZeroTierSocket.cs b/ZTSharp/ZeroTier/ZeroTierSocket.cs index 5f9dcce..74f72cb 100644 --- a/ZTSharp/ZeroTier/ZeroTierSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierSocket.cs @@ -408,39 +408,54 @@ private async Task CreateRuntimeAsync(byte[] inlineCom cancellationToken) .ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); - ThrowIfDisposed(); - - ZeroTierDataplaneRuntime? toDispose = null; - ZeroTierDataplaneRuntime runtime; - await _runtimeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + var createdNeedsDispose = true; try { + cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); - if (_runtime is not null) + + ZeroTierDataplaneRuntime? toDispose = null; + ZeroTierDataplaneRuntime runtime; + await _runtimeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + if (_runtime is not null) + { + toDispose = created; + runtime = _runtime; + } + else + { + _upstreamRoot ??= helloOk; + _upstreamRootKey ??= rootKey; + _runtime = created; + runtime = created; + createdNeedsDispose = false; + } + } + finally { - toDispose = created; - runtime = _runtime; + _runtimeLock.Release(); } - else + + if (toDispose is not null) { - _upstreamRoot ??= helloOk; - _upstreamRootKey ??= rootKey; - _runtime = created; - runtime = created; + await toDispose.DisposeAsync().ConfigureAwait(false); + createdNeedsDispose = false; } + + return runtime; } - finally + catch { - _runtimeLock.Release(); - } + if (createdNeedsDispose) + { + await created.DisposeAsync().ConfigureAwait(false); + } - if (toDispose is not null) - { - await toDispose.DisposeAsync().ConfigureAwait(false); + throw; } - - return runtime; } private void ThrowIfDisposed() From a8353eea451eaaca497b67286903b34b1bdc409e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:27:01 +0100 Subject: [PATCH 067/296] fix(sockets): block bind/listen while connect is in progress --- ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs index b4c74d6..2f8a125 100644 --- a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs +++ b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs @@ -60,6 +60,11 @@ public override async ValueTask BindAsync(EndPoint localEndPoint, CancellationTo { ObjectDisposedException.ThrowIf(Disposed, this); + if (Volatile.Read(ref _connectInProgress) != 0) + { + throw new InvalidOperationException("Socket connect is in progress."); + } + if (_stream is not null || _listener is not null) { throw new InvalidOperationException("Socket is already initialized."); @@ -85,6 +90,11 @@ public override async ValueTask ListenAsync(int backlog, CancellationToken cance { ObjectDisposedException.ThrowIf(Disposed, this); + if (Volatile.Read(ref _connectInProgress) != 0) + { + throw new InvalidOperationException("Socket connect is in progress."); + } + if (_listener is not null) { throw new InvalidOperationException("Socket is already listening."); From c68af1e779d0d9b19c8a1c72ea6e427fb0ab7f41 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:27:44 +0100 Subject: [PATCH 068/296] fix(http): always release reserved overlay local ports --- ZTSharp/Http/OverlayHttpMessageHandler.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ZTSharp/Http/OverlayHttpMessageHandler.cs b/ZTSharp/Http/OverlayHttpMessageHandler.cs index 0ff5a7d..ac1297b 100644 --- a/ZTSharp/Http/OverlayHttpMessageHandler.cs +++ b/ZTSharp/Http/OverlayHttpMessageHandler.cs @@ -77,10 +77,16 @@ private async ValueTask ConnectOverlayAsync(SocketsHttpConnectionContext ReleaseLocalPort(localPort); throw; } - catch (Exception ex) when (ex is not HttpRequestException) + catch (Exception ex) { await client.DisposeAsync().ConfigureAwait(false); ReleaseLocalPort(localPort); + + if (ex is HttpRequestException) + { + throw; + } + throw new HttpRequestException( $"Overlay connect to '{host}:{endpoint.Port}' failed.", ex); From 0c083f4b5948f250f91fe5087672d9259a7a5f83 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:28:33 +0100 Subject: [PATCH 069/296] fix(store): allow symlinked ancestors; validate alias paths --- ZTSharp/FileStateStore.cs | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index e3af16f..22562e6 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -127,8 +127,8 @@ public Task> ListAsync(string prefix = "", CancellationTok if (virtualPrefix.Length != 0 && StateStorePlanetAliases.TryGetCanonicalAliasKey(virtualPrefix, out var canonicalAliasPrefix)) { - var planetPath = Path.Combine(_rootPath, StateStorePlanetAliases.PlanetKey); - var rootsPath = Path.Combine(_rootPath, StateStorePlanetAliases.RootsKey); + var planetPath = GetPhysicalPathForNormalizedKey(StateStorePlanetAliases.PlanetKey, StateStorePlanetAliases.PlanetKey); + var rootsPath = GetPhysicalPathForNormalizedKey(StateStorePlanetAliases.RootsKey, StateStorePlanetAliases.RootsKey); if (File.Exists(planetPath) || File.Exists(rootsPath)) { return Task.FromResult>([canonicalAliasPrefix]); @@ -370,33 +370,10 @@ private void ThrowIfPathTraversesReparsePoint(string fullPath) private void ThrowIfRootPathOrAncestorsAreReparsePoints() { - var root = Path.GetPathRoot(_rootPathTrimmed); - if (string.IsNullOrWhiteSpace(root)) - { - return; - } - - if (IsReparsePoint(root) || IsReparsePoint(_rootPathTrimmed)) + if (IsReparsePoint(_rootPathTrimmed)) { throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); } - - var relative = Path.GetRelativePath(root, _rootPathTrimmed); - if (relative == "." || relative.Length == 0) - { - return; - } - - var current = root; - var parts = relative.Split(new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries); - for (var i = 0; i < parts.Length; i++) - { - current = Path.Combine(current, parts[i]); - if (IsReparsePoint(current)) - { - throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); - } - } } private void EnsureParentDirectoryExistsNoReparse(string fullPath) From 0a86042b4ba45c4baac0bc89fdf7f9170f451ee6 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:29:23 +0100 Subject: [PATCH 070/296] fix(net): reject invalid IPv6 AH header lengths --- ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs | 32 +++++++++++++++++++++++++ ZTSharp/ZeroTier/Net/Ipv6Codec.cs | 5 ++++ 2 files changed, 37 insertions(+) create mode 100644 ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs diff --git a/ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs b/ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs new file mode 100644 index 0000000..ca5ccba --- /dev/null +++ b/ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs @@ -0,0 +1,32 @@ +using System.Net; +using ZTSharp.ZeroTier.Net; + +namespace ZTSharp.Tests; + +public sealed class Ipv6CodecAhHeaderTests +{ + [Fact] + public void TryParseTransportPayload_RejectsInvalidAhHeaderLength() + { + var src = IPAddress.Parse("2001:db8::1"); + var dst = IPAddress.Parse("2001:db8::2"); + + var payload = new byte[8 + 8]; + payload[0] = UdpCodec.ProtocolNumber; // next header + payload[1] = 0; // AH payload len (0 => 8 bytes total, invalid; must be >= 12 bytes) + + var packet = new byte[40 + payload.Length]; + packet[0] = 0x60; // version + packet[4] = (byte)(payload.Length >> 8); + packet[5] = (byte)(payload.Length & 0xFF); + packet[6] = 51; // AH + packet[7] = 64; // hop limit + + src.GetAddressBytes().CopyTo(packet, 8); + dst.GetAddressBytes().CopyTo(packet, 24); + payload.CopyTo(packet, 40); + + Assert.False(Ipv6Codec.TryParseTransportPayload(packet, out _, out _, out _, out _, out _)); + } +} + diff --git a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs index aa3f7c7..5df485d 100644 --- a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs +++ b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs @@ -218,6 +218,11 @@ private static bool TryWalkExtensionHeaders( var headerNext = payload[offset]; var payloadLen32 = payload[offset + 1]; var headerLength = (payloadLen32 + 2) * 4; + if (headerLength < 12) + { + return false; + } + if (headerLength > remaining) { return false; From d50e1160d31b3f2813b2d5c8dd76316fe997b78d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:32:03 +0100 Subject: [PATCH 071/296] test: harden junction helper and relax remaining timeouts --- ZTSharp.Tests/ChannelWriterConcurrencyTests.cs | 4 ++-- ZTSharp.Tests/FileStateStoreSecurityTests.cs | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs index 69179ff..c96200c 100644 --- a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs +++ b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs @@ -158,9 +158,9 @@ void CaptureSyn(in RawFrame frame) await using var client = new OverlayTcpClient(clientNode, networkId, clientPort); await client.ConnectAsync(serverNode.NodeId.Value, serverPort); - await using var _ = await acceptTask.WaitAsync(TimeSpan.FromSeconds(2)); + await using var _ = await acceptTask.WaitAsync(TimeSpan.FromSeconds(10)); - var (connectionId, sourcePort) = await synInfoTcs.Task.WaitAsync(TimeSpan.FromSeconds(2)); + var (connectionId, sourcePort) = await synInfoTcs.Task.WaitAsync(TimeSpan.FromSeconds(10)); const int frames = 500; var payload = Encoding.ASCII.GetBytes("abcdefgh"); diff --git a/ZTSharp.Tests/FileStateStoreSecurityTests.cs b/ZTSharp.Tests/FileStateStoreSecurityTests.cs index d197606..ef6f528 100644 --- a/ZTSharp.Tests/FileStateStoreSecurityTests.cs +++ b/ZTSharp.Tests/FileStateStoreSecurityTests.cs @@ -76,7 +76,19 @@ private static bool TryCreateJunction(string junctionPath, string targetPath) return false; } - process.WaitForExit(milliseconds: 5000); + if (!process.WaitForExit(milliseconds: 5000)) + { + try + { + process.Kill(entireProcessTree: true); + } + catch + { + } + + return false; + } + return process.ExitCode == 0 && Directory.Exists(junctionPath); } catch From 09910e5710ad109d236e2f8731a77315648cd165 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:35:41 +0100 Subject: [PATCH 072/296] fix(dataplane): drop oldest on queue pressure instead of newest --- ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs | 8 ++++---- ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs | 2 +- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 2 +- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs | 6 +++++- ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs | 11 +++++++---- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs index d6e891e..4968590 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs @@ -225,7 +225,7 @@ public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint_When }); using var cts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); var peerNodeId = new NodeId(0x3333333333); var peerPacket = ZeroTierPacketCodec.Encode( @@ -296,7 +296,7 @@ public async Task DispatcherLoopAsync_ForwardsOnlyRootPeerDatagrams_WhenDirectPe }); using var cts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); var peerNodeId = new NodeId(0x3333333333); var peerPacket = ZeroTierPacketCodec.Encode( @@ -362,7 +362,7 @@ public async Task DispatcherLoopAsync_DropsPeerDatagrams_WhenPeerQueueIsFull_And Assert.True(peerChannel.Writer.TryWrite(new ZeroTierUdpDatagram(LocalSocketId: 0, new IPEndPoint(IPAddress.Loopback, 1), new byte[1]))); using var cts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); var localNodeId = new NodeId(0x2222222222); var peerNodeId = new NodeId(0x3333333333); @@ -429,7 +429,7 @@ public async Task ResolveNodeIdAsync_CachesResult_WhenRootResponseComesFromAlter }); using var dispatcherCts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, dispatcherCts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, dispatcherCts.Token), CancellationToken.None); var remoteNodeId = new NodeId(0xaaaaaaaaaa); var serverTask = RunGatherOkServerOnceAsync( diff --git a/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs b/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs index 644b406..e79d847 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs @@ -41,7 +41,7 @@ public async Task DispatcherLoopAsync_ExitsCleanly_WhenUdpTransportIsDisposed() }); using var cts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); var peerPacket = ZeroTierPacketCodec.Encode( new ZeroTierPacketHeader( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 078d349..13f158d 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -207,7 +207,7 @@ public ZeroTierDataplaneRuntime( } }); - _dispatcherLoop = Task.Run(() => _rxLoops.DispatcherLoopAsync(_peerQueue.Writer, _cts.Token), CancellationToken.None); + _dispatcherLoop = Task.Run(() => _rxLoops.DispatcherLoopAsync(_peerQueue, _cts.Token), CancellationToken.None); _peerLoop = Task.Run(() => _rxLoops.PeerLoopAsync(_peerQueue.Reader, _cts.Token), CancellationToken.None); _multipathMaintenanceLoop = multipath.Enabled ? Task.Run(() => MultipathMaintenanceLoopAsync(_cts.Token), CancellationToken.None) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 856b0a4..192637c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -50,8 +50,10 @@ public ZeroTierDataplaneRxLoops( _onPeerQueueDrop = onPeerQueueDrop; } - public async Task DispatcherLoopAsync(ChannelWriter peerWriter, CancellationToken cancellationToken) + public async Task DispatcherLoopAsync(Channel peerQueue, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(peerQueue); + var peerWriter = peerQueue.Writer; while (!cancellationToken.IsCancellationRequested) { ZeroTierUdpDatagram datagram; @@ -148,6 +150,8 @@ public async Task DispatcherLoopAsync(ChannelWriter peerWri } _onPeerQueueDrop?.Invoke(); + _ = peerQueue.Reader.TryRead(out _); + peerWriter.TryWrite(datagram); continue; } } diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index 9b0be23..4bdab05 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -125,12 +125,15 @@ private async Task ProcessReceiveLoopAsync() continue; } - if (!_incoming.Writer.TryWrite(new ZeroTierUdpDatagram( - _localSocketId, - UdpEndpointNormalization.Normalize(result.RemoteEndPoint), - result.Buffer))) + var datagram = new ZeroTierUdpDatagram( + _localSocketId, + UdpEndpointNormalization.Normalize(result.RemoteEndPoint), + result.Buffer); + if (!_incoming.Writer.TryWrite(datagram)) { Interlocked.Increment(ref _incomingBackpressureCount); + _ = _incoming.Reader.TryRead(out _); + _incoming.Writer.TryWrite(datagram); } } } From 154ab52d1b9d7fb25e9b52b4202c23cb4330f8ac Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:48:15 +0100 Subject: [PATCH 073/296] fix(dataplane): allow dispatcher drop logic with multi-reader channel --- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 13f158d..c3f7a95 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -41,7 +41,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly Channel _peerQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) { FullMode = BoundedChannelFullMode.Wait, - SingleReader = true, + SingleReader = false, SingleWriter = true }); private long _peerQueueDropCount; From ca58dc50d09abc615e68a0ffe9b6d99c65ca036a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:04:44 +0100 Subject: [PATCH 074/296] fix(http): always release reserved port --- ZTSharp/Http/OverlayHttpMessageHandler.cs | 31 ++++++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/ZTSharp/Http/OverlayHttpMessageHandler.cs b/ZTSharp/Http/OverlayHttpMessageHandler.cs index ac1297b..1f6ca83 100644 --- a/ZTSharp/Http/OverlayHttpMessageHandler.cs +++ b/ZTSharp/Http/OverlayHttpMessageHandler.cs @@ -73,14 +73,37 @@ private async ValueTask ConnectOverlayAsync(SocketsHttpConnectionContext } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - await client.DisposeAsync().ConfigureAwait(false); - ReleaseLocalPort(localPort); + try + { + await client.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose is best-effort; preserve cancellation semantics and always release the reserved port. + catch +#pragma warning restore CA1031 + { + } + finally + { + ReleaseLocalPort(localPort); + } + throw; } catch (Exception ex) { - await client.DisposeAsync().ConfigureAwait(false); - ReleaseLocalPort(localPort); + try + { + await client.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose is best-effort; preserve original exception and always release the reserved port. + catch +#pragma warning restore CA1031 + { + } + finally + { + ReleaseLocalPort(localPort); + } if (ex is HttpRequestException) { From 5b2fabfd59e8277cf284fedeff37706d08597a74 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:04:49 +0100 Subject: [PATCH 075/296] fix(tcp): reset segment after full consumption --- .../OverlayTcpIncomingBufferTests.cs | 20 +++++++++++++++++++ ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs index 98c474e..af6de4b 100644 --- a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs +++ b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs @@ -39,6 +39,26 @@ public async Task FinGrace_AllowsLateDataFramesToBeRead() Assert.Equal(0, eof); } + [Fact] + public async Task FinGrace_AfterSegmentConsumed_SetsRemoteClosed_WhenNoLateDataArrives() + { + var incoming = new OverlayTcpIncomingBuffer(); + + var payload = new byte[] { 1, 2, 3 }; + Assert.True(incoming.TryWrite(payload)); + + var buffer = new byte[3]; + var read = await incoming.ReadAsync(buffer, CancellationToken.None); + Assert.Equal(3, read); + Assert.Equal(payload, buffer); + + incoming.MarkRemoteFinReceived(); + + var eof = await incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(1)); + Assert.Equal(0, eof); + Assert.True(incoming.RemoteClosed); + } + [Fact] public async Task FinArrival_UnblocksReaderAwaitingReadAsync() { diff --git a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs index 43a81c3..c1c91ca 100644 --- a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs +++ b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs @@ -113,6 +113,12 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can if (_currentSegment.Length == 0 || _currentSegmentOffset >= _currentSegment.Length) { + if (_currentSegmentOffset >= _currentSegment.Length) + { + _currentSegment = default; + _currentSegmentOffset = 0; + } + while (true) { if (_incoming.Reader.TryRead(out var nextSegment)) From a899f633b4a0060abe90f285bfd4fb4b3a285742 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:04:53 +0100 Subject: [PATCH 076/296] fix(io): fail if secret file perms unsupported --- ZTSharp/Internal/AtomicFile.cs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ZTSharp/Internal/AtomicFile.cs b/ZTSharp/Internal/AtomicFile.cs index 91991c1..90083f7 100644 --- a/ZTSharp/Internal/AtomicFile.cs +++ b/ZTSharp/Internal/AtomicFile.cs @@ -396,6 +396,15 @@ private static FileStream CreateSecretWriteStream(string path, UnixFileMode unix } catch (PlatformNotSupportedException) { + fallback.Dispose(); + TryDeleteFallback(path); + throw new IOException("Failed to set secret file permissions."); + } + catch (NotSupportedException) + { + fallback.Dispose(); + TryDeleteFallback(path); + throw new IOException("Failed to set secret file permissions."); } catch (IOException ex) { @@ -410,4 +419,21 @@ private static FileStream CreateSecretWriteStream(string path, UnixFileMode unix return fallback; } + + private static void TryDeleteFallback(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } } From 7f6c6686ca9d18cfabb5582eb9012df0d8dd47d7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:05:04 +0100 Subject: [PATCH 077/296] chore(test): remove unused using --- ZTSharp.Tests/FileStateStoreSecurityTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/ZTSharp.Tests/FileStateStoreSecurityTests.cs b/ZTSharp.Tests/FileStateStoreSecurityTests.cs index ef6f528..e0842c4 100644 --- a/ZTSharp.Tests/FileStateStoreSecurityTests.cs +++ b/ZTSharp.Tests/FileStateStoreSecurityTests.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Net; namespace ZTSharp.Tests; From 6e05adf7461468944063f381a2c88877e9cf9c21 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:24:00 +0100 Subject: [PATCH 078/296] fix(store): validate root reparse after create --- ZTSharp/FileStateStore.cs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index 22562e6..873ceb8 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -20,8 +20,9 @@ public FileStateStore(string rootPath) _rootPathTrimmed = Path.TrimEndingDirectorySeparator(_rootPath); _rootPathPrefix = _rootPathTrimmed + Path.DirectorySeparatorChar; _pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - ThrowIfRootPathOrAncestorsAreReparsePoints(); + ThrowIfRootPathIsReparsePoint(); Directory.CreateDirectory(_rootPath); + ThrowIfRootPathIsReparsePoint(); } public Task ExistsAsync(string key, CancellationToken cancellationToken = default) @@ -330,10 +331,7 @@ private bool IsUnderRoot(string fullPath) private void ThrowIfPathTraversesReparsePoint(string fullPath) { - if (IsReparsePoint(_rootPathTrimmed)) - { - throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); - } + ThrowIfRootPathIsReparsePoint(); if (string.Equals(fullPath, _rootPathTrimmed, _pathComparison)) { @@ -368,7 +366,7 @@ private void ThrowIfPathTraversesReparsePoint(string fullPath) } } - private void ThrowIfRootPathOrAncestorsAreReparsePoints() + private void ThrowIfRootPathIsReparsePoint() { if (IsReparsePoint(_rootPathTrimmed)) { @@ -378,10 +376,7 @@ private void ThrowIfRootPathOrAncestorsAreReparsePoints() private void EnsureParentDirectoryExistsNoReparse(string fullPath) { - if (IsReparsePoint(_rootPathTrimmed)) - { - throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); - } + ThrowIfRootPathIsReparsePoint(); var directory = Path.GetDirectoryName(fullPath); if (string.IsNullOrWhiteSpace(directory) || string.Equals(directory, _rootPathTrimmed, _pathComparison)) From 73e3085773450fb6a2b6181b6a74c6fd7e48ded6 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:34:11 +0100 Subject: [PATCH 079/296] fix(internal): prevent ActiveTaskSet.WaitAsync spin --- ZTSharp/Internal/ActiveTaskSet.cs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/ZTSharp/Internal/ActiveTaskSet.cs b/ZTSharp/Internal/ActiveTaskSet.cs index 21aa3e9..0150ab4 100644 --- a/ZTSharp/Internal/ActiveTaskSet.cs +++ b/ZTSharp/Internal/ActiveTaskSet.cs @@ -30,10 +30,10 @@ public async Task WaitAsync(CancellationToken cancellationToken = default) return; } - var snapshot = new List(_tasks.Count); - foreach (var task in _tasks.Values) + var snapshot = new List>(_tasks.Count); + foreach (var pair in _tasks) { - snapshot.Add(task); + snapshot.Add(pair); } if (snapshot.Count == 0) @@ -44,12 +44,27 @@ public async Task WaitAsync(CancellationToken cancellationToken = default) try { - await Task.WhenAll(snapshot).WaitAsync(cancellationToken).ConfigureAwait(false); + var tasks = new Task[snapshot.Count]; + for (var i = 0; i < snapshot.Count; i++) + { + tasks[i] = snapshot[i].Value; + } + + await Task.WhenAll(tasks).WaitAsync(cancellationToken).ConfigureAwait(false); } #pragma warning disable CA1031 // ActiveTaskSet is primarily used for shutdown coordination; faults are observed but not surfaced here. catch #pragma warning restore CA1031 { + foreach (var pair in snapshot) + { + if (pair.Value.IsCompleted) + { + _tasks.TryRemove(pair.Key, out _); + } + } + + await Task.Yield(); } } } From e007a54f61fe960b5c79e239a4a9ef94dadd54de Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:35:18 +0100 Subject: [PATCH 080/296] fix(dataplane): skip decode for non-root when direct RX off --- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 192637c..4befd89 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -74,6 +74,11 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca return; } + if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) + { + continue; + } + var packetBytes = datagram.Payload; if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var decoded)) { @@ -137,11 +142,6 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca continue; } - if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) - { - continue; - } - if (!peerWriter.TryWrite(datagram)) { if (cancellationToken.IsCancellationRequested) From 5a6d4e992b19722249b5eba71e9ce71ed484c6cf Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:35:44 +0100 Subject: [PATCH 081/296] test: reduce concurrency flakiness in channel writer tests --- .../ChannelWriterConcurrencyTests.cs | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs index c96200c..c850ef3 100644 --- a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs +++ b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs @@ -68,7 +68,7 @@ public async Task InMemoryOverlayTcpListener_AcceptsMultipleConcurrentClients() await using var listener = new OverlayTcpListener(serverNode, networkId, serverPort); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var acceptTask = Task.Run(async () => { var accepted = new List(capacity: count); @@ -94,8 +94,8 @@ public async Task InMemoryOverlayTcpListener_AcceptsMultipleConcurrentClients() }, cts.Token); } - await Task.WhenAll(connectTasks).WaitAsync(TimeSpan.FromSeconds(5), cts.Token); - var acceptedClients = await acceptTask.WaitAsync(TimeSpan.FromSeconds(5), cts.Token); + await Task.WhenAll(connectTasks).WaitAsync(TimeSpan.FromSeconds(10), cts.Token); + var acceptedClients = await acceptTask.WaitAsync(TimeSpan.FromSeconds(10), cts.Token); Assert.Equal(count, acceptedClients.Count); foreach (var accepted in acceptedClients) @@ -168,26 +168,23 @@ void CaptureSyn(in RawFrame frame) var sendTasks = new Task[frames]; for (var i = 0; i < frames; i++) { - sendTasks[i] = Task.Run(async () => - { - var dataFrame = new byte[OverlayTcpFrameCodec.HeaderLength + payload.Length]; - OverlayTcpFrameCodec.BuildHeader( - OverlayTcpFrameCodec.FrameType.Data, - sourcePort: serverPort, - destinationPort: clientPort, - destinationNodeId: clientNode.NodeId.Value, - connectionId, - dataFrame); - payload.CopyTo(dataFrame.AsSpan(OverlayTcpFrameCodec.HeaderLength)); - await serverNode.SendFrameAsync(networkId, dataFrame); - }); + var dataFrame = new byte[OverlayTcpFrameCodec.HeaderLength + payload.Length]; + OverlayTcpFrameCodec.BuildHeader( + OverlayTcpFrameCodec.FrameType.Data, + sourcePort: serverPort, + destinationPort: clientPort, + destinationNodeId: clientNode.NodeId.Value, + connectionId, + dataFrame); + payload.CopyTo(dataFrame.AsSpan(OverlayTcpFrameCodec.HeaderLength)); + sendTasks[i] = serverNode.SendFrameAsync(networkId, dataFrame); } - await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(10)); + await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(20)); var clientStream = client.GetStream(); var buffer = new byte[frames * payload.Length]; - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); var read = await StreamTestHelpers.ReadExactAsync(clientStream, buffer, buffer.Length, cts.Token); Assert.Equal(buffer.Length, read); } From efbe91d2608e4140e1c2c2dafb96cefc66b315af Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:36:47 +0100 Subject: [PATCH 082/296] fix(udp): restore destination IP filtering on receive --- ZTSharp/ZeroTier/ZeroTierUdpSocket.cs | 4 ++-- docs/ZEROTIER_SOCKETS.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs index 77ab20d..b70a671 100644 --- a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs @@ -140,7 +140,7 @@ public async ValueTask ReceiveFromAsync( continue; } - if (protocol != UdpCodec.ProtocolNumber) + if (!dst.Equals(_localAddress) || protocol != UdpCodec.ProtocolNumber) { continue; } @@ -152,7 +152,7 @@ public async ValueTask ReceiveFromAsync( continue; } - if (protocol != UdpCodec.ProtocolNumber) + if (!ZeroTierIpAddressCanonicalization.EqualsForManagedIpComparison(dst, _localAddress) || protocol != UdpCodec.ProtocolNumber) { continue; } diff --git a/docs/ZEROTIER_SOCKETS.md b/docs/ZEROTIER_SOCKETS.md index 01e5770..4fb9674 100644 --- a/docs/ZEROTIER_SOCKETS.md +++ b/docs/ZEROTIER_SOCKETS.md @@ -132,7 +132,7 @@ There is no virtual network interface visible to the operating system. - TCP connect may use `IPAddress.Any` / `IPAddress.IPv6Any` as the local endpoint to mean "choose a managed IP" (the selected local endpoint is returned by APIs that surface it). - TCP listen with port `0` is **not supported** (`NotSupportedException`). - UDP bind supports port `0` (ephemeral); the bound local port is selected internally via `ZeroTierEphemeralPorts.Generate()` and can be read back from the socket. -- UDP bind is currently **per address family + port** (not per IP + port). A UDP socket may receive datagrams destined to any of the node's managed IPs on that port. +- UDP bind is currently **per address family + port** (not per IP + port), meaning you cannot bind multiple UDP sockets to the same port on different managed IPs. - `ReceiveFromAsync(...)` does not currently surface the local destination IP. ### Socket Options From d51f3eec8ee95fba5ab78a34d2e74852bcce63df Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:37:33 +0100 Subject: [PATCH 083/296] Revert "fix(udp): restore destination IP filtering on receive" This reverts commit efbe91d2608e4140e1c2c2dafb96cefc66b315af. --- ZTSharp/ZeroTier/ZeroTierUdpSocket.cs | 4 ++-- docs/ZEROTIER_SOCKETS.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs index b70a671..77ab20d 100644 --- a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs @@ -140,7 +140,7 @@ public async ValueTask ReceiveFromAsync( continue; } - if (!dst.Equals(_localAddress) || protocol != UdpCodec.ProtocolNumber) + if (protocol != UdpCodec.ProtocolNumber) { continue; } @@ -152,7 +152,7 @@ public async ValueTask ReceiveFromAsync( continue; } - if (!ZeroTierIpAddressCanonicalization.EqualsForManagedIpComparison(dst, _localAddress) || protocol != UdpCodec.ProtocolNumber) + if (protocol != UdpCodec.ProtocolNumber) { continue; } diff --git a/docs/ZEROTIER_SOCKETS.md b/docs/ZEROTIER_SOCKETS.md index 4fb9674..01e5770 100644 --- a/docs/ZEROTIER_SOCKETS.md +++ b/docs/ZEROTIER_SOCKETS.md @@ -132,7 +132,7 @@ There is no virtual network interface visible to the operating system. - TCP connect may use `IPAddress.Any` / `IPAddress.IPv6Any` as the local endpoint to mean "choose a managed IP" (the selected local endpoint is returned by APIs that surface it). - TCP listen with port `0` is **not supported** (`NotSupportedException`). - UDP bind supports port `0` (ephemeral); the bound local port is selected internally via `ZeroTierEphemeralPorts.Generate()` and can be read back from the socket. -- UDP bind is currently **per address family + port** (not per IP + port), meaning you cannot bind multiple UDP sockets to the same port on different managed IPs. +- UDP bind is currently **per address family + port** (not per IP + port). A UDP socket may receive datagrams destined to any of the node's managed IPs on that port. - `ReceiveFromAsync(...)` does not currently surface the local destination IP. ### Socket Options From 333a356498db00aafaa3b96987719fa058ecea7d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:38:22 +0100 Subject: [PATCH 084/296] Revert "fix(dataplane): skip decode for non-root when direct RX off" This reverts commit e007a54f61fe960b5c79e239a4a9ef94dadd54de. --- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 4befd89..192637c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -74,11 +74,6 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca return; } - if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) - { - continue; - } - var packetBytes = datagram.Payload; if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var decoded)) { @@ -142,6 +137,11 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca continue; } + if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) + { + continue; + } + if (!peerWriter.TryWrite(datagram)) { if (cancellationToken.IsCancellationRequested) From 2e729c16d342a3ccecb22b91ca0ae3a4de9d14be Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:38:53 +0100 Subject: [PATCH 085/296] fix(dataplane): early-drop non-root when direct RX off --- .../Internal/ZeroTierDataplaneRxLoops.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 192637c..00e5c69 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -74,6 +74,23 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca return; } + if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) + { + var peek = datagram.Payload.AsSpan(); + if (peek.Length < ZeroTierPacketHeader.Length) + { + continue; + } + + var source = new NodeId( + ZeroTierBinaryPrimitives.ReadUInt40BigEndian( + peek.Slice(ZeroTierPacketHeader.IndexSource, 5))); + if (source != _rootNodeId) + { + continue; + } + } + var packetBytes = datagram.Payload; if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var decoded)) { From 9aa487791487029185fd73efa53dac81a29bba2c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:58:41 +0100 Subject: [PATCH 086/296] fix(udp): avoid extra drops on backpressure --- ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index 4bdab05..c72efb3 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -132,8 +132,15 @@ private async Task ProcessReceiveLoopAsync() if (!_incoming.Writer.TryWrite(datagram)) { Interlocked.Increment(ref _incomingBackpressureCount); - _ = _incoming.Reader.TryRead(out _); - _incoming.Writer.TryWrite(datagram); + if (_incoming.Writer.TryWrite(datagram)) + { + continue; + } + + if (_incoming.Reader.TryRead(out _)) + { + _incoming.Writer.TryWrite(datagram); + } } } } From 722281597d4eb53733d4f1614bfed63118ff090d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:59:05 +0100 Subject: [PATCH 087/296] fix(dataplane): avoid extra peer-queue drops --- .../Internal/ZeroTierDataplaneRxLoops.cs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 00e5c69..832ab8e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -166,10 +166,21 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca return; } + if (peerWriter.TryWrite(datagram)) + { + continue; + } + _onPeerQueueDrop?.Invoke(); - _ = peerQueue.Reader.TryRead(out _); - peerWriter.TryWrite(datagram); - continue; + if (peerQueue.Reader.TryRead(out _)) + { + if (peerWriter.TryWrite(datagram)) + { + continue; + } + } + + return; } } } From 55628c5f989abc5a34de8df75035d9dc6985a99b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:59:31 +0100 Subject: [PATCH 088/296] fix(bootstrap): avoid sync-over-async udp cleanup --- .../ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 77ba2c6..7a188f1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -9,7 +9,7 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierSocketRuntimeBootstrapper { - internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOptions multipath, bool enableIpv6) + private static async ValueTask CreateUdpTransportAsync(ZeroTierMultipathOptions multipath, bool enableIpv6) { ArgumentNullException.ThrowIfNull(multipath); @@ -72,7 +72,7 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption { try { - socket.DisposeAsync().AsTask().GetAwaiter().GetResult(); + await socket.DisposeAsync().ConfigureAwait(false); } catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException or SocketException or InvalidOperationException) { @@ -102,7 +102,7 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption ArgumentNullException.ThrowIfNull(managedIps); ArgumentNullException.ThrowIfNull(inlineCom); - var udp = CreateUdpTransport(multipath, enableIpv6: true); + var udp = await CreateUdpTransportAsync(multipath, enableIpv6: true).ConfigureAwait(false); try { var localManagedIpsV6 = managedIps From b772580f7b7f005e8280f9bd98d777914d35c42a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:59:46 +0100 Subject: [PATCH 089/296] fix(tcp): make listener dispose backlog drain best-effort --- ZTSharp/ZeroTier/ZeroTierTcpListener.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs index 779847e..c6bd735 100644 --- a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs +++ b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs @@ -103,7 +103,15 @@ public async ValueTask DisposeAsync() while (_acceptQueue.Reader.TryRead(out var accepted)) { Interlocked.Decrement(ref _pendingAcceptCount); - await accepted.Stream.DisposeAsync().ConfigureAwait(false); + try + { + await accepted.Stream.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose is best-effort during shutdown/drain. + catch +#pragma warning restore CA1031 + { + } } using var drainCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); From 0be0ece47df2766c6d7834b8acf348497b10640c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:00:51 +0100 Subject: [PATCH 090/296] test: use CreateUdpTransportAsync in bootstrapper tests --- ...rSocketRuntimeBootstrapperUdpTransportTests.cs | 15 ++++++++------- .../Internal/ZeroTierSocketRuntimeBootstrapper.cs | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index 1624075..fa3160e 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -11,7 +11,7 @@ public sealed class ZeroTierSocketRuntimeBootstrapperUdpTransportTests [Fact] public async Task CreateUdpTransport_Default_ReturnsSingleSocket() { - var transport = ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport(new ZeroTierMultipathOptions(), enableIpv6: false); + var transport = await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync(new ZeroTierMultipathOptions(), enableIpv6: false); try { Assert.Single(transport.LocalSockets); @@ -25,7 +25,7 @@ public async Task CreateUdpTransport_Default_ReturnsSingleSocket() [Fact] public async Task CreateUdpTransport_MultipathEnabled_UsesMultipleSockets() { - var transport = ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport( + var transport = await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 3 }, enableIpv6: false); @@ -41,11 +41,12 @@ public async Task CreateUdpTransport_MultipathEnabled_UsesMultipleSockets() } [Fact] - public void CreateUdpTransport_MultipathEnabled_RequiresAtLeastOneSocket() + public async Task CreateUdpTransport_MultipathEnabled_RequiresAtLeastOneSocket() { - Assert.Throws(() => ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport( - new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 0 }, - enableIpv6: false)); + await Assert.ThrowsAsync(async () => + await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 0 }, + enableIpv6: false)); } [Fact] @@ -58,7 +59,7 @@ public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() var transport = default(IZeroTierUdpTransport); try { - transport = ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport( + transport = await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 1, LocalUdpPorts = new[] { port } }, enableIpv6: false); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 7a188f1..050412c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -9,7 +9,7 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierSocketRuntimeBootstrapper { - private static async ValueTask CreateUdpTransportAsync(ZeroTierMultipathOptions multipath, bool enableIpv6) + internal static async ValueTask CreateUdpTransportAsync(ZeroTierMultipathOptions multipath, bool enableIpv6) { ArgumentNullException.ThrowIfNull(multipath); From 5938b99e8fe437c4911739d1f2007d80d229d5d8 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:19:23 +0100 Subject: [PATCH 091/296] fix(io): treat non-seekable reads as Try* failure --- ZTSharp/Internal/BoundedFileIO.cs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ZTSharp/Internal/BoundedFileIO.cs b/ZTSharp/Internal/BoundedFileIO.cs index 67a0d47..47a8b0a 100644 --- a/ZTSharp/Internal/BoundedFileIO.cs +++ b/ZTSharp/Internal/BoundedFileIO.cs @@ -24,13 +24,22 @@ public static bool TryReadAllBytes(string path, int maxBytes, out byte[] bytes) bufferSize: 16 * 1024, options: FileOptions.SequentialScan); - if (stream.Length <= 0 || stream.Length > maxBytes || stream.Length > int.MaxValue) + long length; + try + { + length = stream.Length; + } + catch (NotSupportedException) { return false; } - var length = (int)stream.Length; - var buffer = new byte[length]; + if (length <= 0 || length > maxBytes || length > int.MaxValue) + { + return false; + } + + var buffer = new byte[(int)length]; var totalRead = 0; while (totalRead < buffer.Length) @@ -55,6 +64,10 @@ public static bool TryReadAllBytes(string path, int maxBytes, out byte[] bytes) { return false; } + catch (NotSupportedException) + { + return false; + } } public static bool TryReadAllText(string path, int maxBytes, Encoding encoding, out string text) From 3d68414ccfdd9acc3a6287bdbf99a434e9591f9a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:19:29 +0100 Subject: [PATCH 092/296] fix(store): bounded read for managed IPs file --- .../Internal/ZeroTierSocketStatePersistence.cs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs index 8929851..078bc78 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs @@ -23,22 +23,14 @@ public static IPAddress[] LoadManagedIps(string statePath, ulong networkId) try { - using var stream = new FileStream( - path, - FileMode.Open, - FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete, - bufferSize: 16 * 1024, - options: FileOptions.SequentialScan); - - if (stream.Length <= 0 || stream.Length > MaxManagedIpsFileBytes) + if (!BoundedFileIO.TryReadAllText(path, maxBytes: MaxManagedIpsFileBytes, Encoding.UTF8, out var text)) { return Array.Empty(); } - using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 8 * 1024, leaveOpen: true); var ips = new List(); + using var reader = new StringReader(text); while (true) { var line = reader.ReadLine(); From 499ac060f249ffaed26f8baccd7bf048b52c9690 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:19:38 +0100 Subject: [PATCH 093/296] fix(store): reject Windows reparse-point ancestors --- ZTSharp/FileStateStore.cs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index 873ceb8..a642295 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -372,6 +372,28 @@ private void ThrowIfRootPathIsReparsePoint() { throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); } + + if (!OperatingSystem.IsWindows()) + { + return; + } + + var current = Path.GetDirectoryName(_rootPathTrimmed); + while (!string.IsNullOrWhiteSpace(current)) + { + if (IsReparsePoint(current)) + { + throw new InvalidOperationException("State root path must not be under a symlink/junction/reparse point."); + } + + var parent = Path.GetDirectoryName(current); + if (parent is null || string.Equals(parent, current, _pathComparison)) + { + break; + } + + current = parent; + } } private void EnsureParentDirectoryExistsNoReparse(string fullPath) From 4da105ed3dde90383e8e1437de13951a208b1c1c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:39:46 +0100 Subject: [PATCH 094/296] fix(socket): avoid NRE racing runtime/join tasks --- ZTSharp/ZeroTier/ZeroTierSocket.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ZTSharp/ZeroTier/ZeroTierSocket.cs b/ZTSharp/ZeroTier/ZeroTierSocket.cs index 74f72cb..4286b81 100644 --- a/ZTSharp/ZeroTier/ZeroTierSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierSocket.cs @@ -57,6 +57,7 @@ public async Task JoinAsync(CancellationToken cancellationToken = default) using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdown.Token); var token = linkedCts.Token; + Task joinTask; try { await _joinLock.WaitAsync(token).ConfigureAwait(false); @@ -79,6 +80,7 @@ public async Task JoinAsync(CancellationToken cancellationToken = default) } _joinTask ??= JoinCoreAsync(_shutdown.Token); + joinTask = _joinTask; } finally { @@ -87,7 +89,7 @@ public async Task JoinAsync(CancellationToken cancellationToken = default) try { - await _joinTask.WaitAsync(token).ConfigureAwait(false); + await joinTask.WaitAsync(token).ConfigureAwait(false); } catch (OperationCanceledException) when (_shutdown.IsCancellationRequested) { @@ -355,6 +357,7 @@ private async Task GetOrCreateRuntimeAsync( using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdown.Token); var token = linkedCts.Token; + Task runtimeTask; try { await _runtimeLock.WaitAsync(token).ConfigureAwait(false); @@ -377,6 +380,7 @@ private async Task GetOrCreateRuntimeAsync( } _runtimeTask ??= CreateRuntimeAsync(inlineCom, _shutdown.Token); + runtimeTask = _runtimeTask; } finally { @@ -385,7 +389,7 @@ private async Task GetOrCreateRuntimeAsync( try { - return await _runtimeTask.WaitAsync(token).ConfigureAwait(false); + return await runtimeTask.WaitAsync(token).ConfigureAwait(false); } catch (OperationCanceledException) when (_shutdown.IsCancellationRequested) { From 155b40c666d7cea95a6a105265753857ec710b47 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:40:17 +0100 Subject: [PATCH 095/296] fix(events): bound NodeEventStream dispatch backlog --- ZTSharp/Internal/NodeEventStream.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ZTSharp/Internal/NodeEventStream.cs b/ZTSharp/Internal/NodeEventStream.cs index 3d14262..63326ca 100644 --- a/ZTSharp/Internal/NodeEventStream.cs +++ b/ZTSharp/Internal/NodeEventStream.cs @@ -5,6 +5,8 @@ namespace ZTSharp.Internal; internal sealed class NodeEventStream { + private const int DispatchQueueCapacity = 1024; + private readonly Action _onEventRaised; private readonly ILogger _logger; private readonly Channel _dispatchQueue; @@ -15,8 +17,9 @@ public NodeEventStream(Action onEventRaised, ILogger logger) { _onEventRaised = onEventRaised ?? throw new ArgumentNullException(nameof(onEventRaised)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _dispatchQueue = Channel.CreateUnbounded(new UnboundedChannelOptions + _dispatchQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: DispatchQueueCapacity) { + FullMode = BoundedChannelFullMode.DropOldest, SingleReader = true, SingleWriter = false }); From f3771649dfaa90a52048804162bef38a81a7b235 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:40:24 +0100 Subject: [PATCH 096/296] fix(multipath): always dispose udp sockets even on faults --- .../Transport/ZeroTierUdpMultiTransport.cs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs index 0c7dc4c..f5448b3 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs @@ -71,23 +71,30 @@ public async ValueTask DisposeAsync() _disposed = true; _incoming.Writer.TryComplete(); - await _cts.CancelAsync().ConfigureAwait(false); - try { + await _cts.CancelAsync().ConfigureAwait(false); await Task.WhenAll(_forwarders).ConfigureAwait(false); } - catch (Exception ex) when (ex is OperationCanceledException or ChannelClosedException) + catch (OperationCanceledException) + { + } + catch (ChannelClosedException) + { + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch +#pragma warning restore CA1031 { } finally { _cts.Dispose(); - } - foreach (var socket in _sockets) - { - await socket.DisposeAsync().ConfigureAwait(false); + foreach (var socket in _sockets) + { + await socket.DisposeAsync().ConfigureAwait(false); + } } } From 3770bbc5706ea7cbde67152236e5297df04cc447 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:20:31 +0100 Subject: [PATCH 097/296] fix: harden ZeroTierUdpMultiTransport disposal --- .../ZeroTierUdpMultiTransportTests.cs | 82 ++++++++++++++++++ .../Transport/ZeroTierUdpMultiTransport.cs | 83 ++++++++++++++----- 2 files changed, 143 insertions(+), 22 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs b/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs index 8a71692..756c7f9 100644 --- a/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Reflection; using ZTSharp.ZeroTier.Transport; namespace ZTSharp.Tests; @@ -50,5 +51,86 @@ public async Task SendAsync_UsesSelectedSocket_AsUdpSourceEndpoint() var datagram2 = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); Assert.Equal(socket2.LocalEndpoint.Port, datagram2.RemoteEndPoint.Port); } + + [Fact] + public async Task DisposeAsync_DisposesUnderlyingSockets_AndMarksTransportDisposed() + { + var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + try + { + await multi.DisposeAsync(); + + Assert.Throws(() => _ = multi.LocalSockets); + await Assert.ThrowsAsync(() => + multi.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x00 })); + + await Assert.ThrowsAsync(() => + socket1.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x01 })); + await Assert.ThrowsAsync(() => + socket2.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x02 })); + } + finally + { + await multi.DisposeAsync(); + } + } + + [Fact] + public async Task DisposeAsync_RemainsBestEffort_WhenForwarderFaults() + { + var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + try + { + var forwardersField = typeof(ZeroTierUdpMultiTransport).GetField("_forwarders", BindingFlags.NonPublic | BindingFlags.Instance); + Assert.NotNull(forwardersField); + + var forwarders = (Task[]?)forwardersField!.GetValue(multi); + Assert.NotNull(forwarders); + Assert.NotEmpty(forwarders!); + + forwarders![0] = Task.WhenAll(forwarders[0], Task.FromException(new InvalidOperationException("boom"))); + + await multi.DisposeAsync(); + + await Assert.ThrowsAsync(() => + socket1.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x01 })); + await Assert.ThrowsAsync(() => + socket2.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x02 })); + } + finally + { + await multi.DisposeAsync(); + } + } + + [Fact] + public async Task DisposeAsync_CanBeCalledConcurrently_WithoutThrowing() + { + var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + try + { + var tasks = Enumerable.Range(0, 20) + .Select(_ => multi.DisposeAsync().AsTask()) + .ToArray(); + + await Task.WhenAll(tasks); + + await Assert.ThrowsAsync(() => + socket1.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x01 })); + } + finally + { + await multi.DisposeAsync(); + } + } } diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs index f5448b3..0cd10e2 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Diagnostics; using System.Threading.Channels; using ZTSharp.ZeroTier.Internal; @@ -10,7 +11,7 @@ internal sealed class ZeroTierUdpMultiTransport : IZeroTierUdpTransport private readonly Channel _incoming; private readonly CancellationTokenSource _cts = new(); private readonly Task[] _forwarders; - private bool _disposed; + private int _disposed; public ZeroTierUdpMultiTransport(IReadOnlyList sockets) { @@ -37,64 +38,98 @@ public ZeroTierUdpMultiTransport(IReadOnlyList sockets) } public IReadOnlyList LocalSockets - => _sockets.Select(socket => new ZeroTierUdpLocalSocket(socket.LocalSocketId, socket.LocalEndpoint)).ToArray(); + { + get + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + return _sockets.Select(socket => new ZeroTierUdpLocalSocket(socket.LocalSocketId, socket.LocalEndpoint)).ToArray(); + } + } public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return _incoming.Reader.ReadAsync(cancellationToken); } public async ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return await ZeroTierTimeouts .RunWithTimeoutAsync(timeout, operation: "UDP receive", _incoming.Reader.ReadAsync, cancellationToken) .ConfigureAwait(false); } public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) - => _sockets[0].SendAsync(remoteEndpoint, payload, cancellationToken); + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + return _sockets[0].SendAsync(remoteEndpoint, payload, cancellationToken); + } public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); var socket = GetSocket(localSocketId); return socket.SendAsync(remoteEndpoint, payload, cancellationToken); } public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } - _disposed = true; _incoming.Writer.TryComplete(); + + var forwarderCompletion = Task.WhenAll(_forwarders); try { - await _cts.CancelAsync().ConfigureAwait(false); - await Task.WhenAll(_forwarders).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - } - catch (ChannelClosedException) - { - } + try + { + await _cts.CancelAsync().ConfigureAwait(false); + } #pragma warning disable CA1031 // Dispose must be best-effort. - catch + catch (Exception ex) #pragma warning restore CA1031 - { + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] CancelAsync failed: {ex}"); +#endif + } + + foreach (var socket in _sockets) + { + try + { + await socket.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] Socket dispose failed: {ex}"); +#endif + } + } } finally { - _cts.Dispose(); - - foreach (var socket in _sockets) + try { - await socket.DisposeAsync().ConfigureAwait(false); + await forwarderCompletion.ConfigureAwait(false); } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] Forwarder completion failed: {ex}"); +#endif + } + + _cts.Dispose(); } } @@ -146,6 +181,10 @@ private async Task ForwardLoopAsync(ZeroTierUdpTransport socket, CancellationTok { return; } + catch (ObjectDisposedException) + { + return; + } } } } From 44a72951db7516d4b36488e3598f77ea1001ac4e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:21:22 +0100 Subject: [PATCH 098/296] fix: harden ZeroTierUdpTransport disposal --- ZTSharp.Tests/ZeroTierUdpTransportTests.cs | 15 +++++ .../Transport/ZeroTierUdpTransport.cs | 55 ++++++++++++++++--- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierUdpTransportTests.cs index 3795f18..8d637b9 100644 --- a/ZTSharp.Tests/ZeroTierUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierUdpTransportTests.cs @@ -28,4 +28,19 @@ public async Task CanSendAndReceiveLoopbackDatagrams() var receivedPong = await a.ReceiveAsync(cts.Token); Assert.True(receivedPong.Payload.AsSpan().SequenceEqual(pong)); } + + [Fact] + public async Task DisposeAsync_CanBeCalledConcurrently_WithoutThrowing() + { + var transport = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var tasks = Enumerable.Range(0, 20) + .Select(_ => transport.DisposeAsync().AsTask()) + .ToArray(); + + await Task.WhenAll(tasks); + + await Assert.ThrowsAsync(() => + transport.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x00 })); + } } diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index c72efb3..6ef0007 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -1,5 +1,6 @@ using System.Net; using System.Net.Sockets; +using System.Diagnostics; using System.Threading.Channels; using ZTSharp.Transport.Internal; using ZTSharp.ZeroTier.Internal; @@ -15,7 +16,7 @@ internal sealed class ZeroTierUdpTransport : IZeroTierUdpTransport private readonly Task _receiverLoop; private readonly int _localSocketId; private long _incomingBackpressureCount; - private bool _disposed; + private int _disposed; public ZeroTierUdpTransport(int localPort = 0, bool enableIpv6 = true, Action? log = null, int localSocketId = 0) { @@ -42,7 +43,7 @@ public IReadOnlyList LocalSockets public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return _incoming.Reader.ReadAsync(cancellationToken); } @@ -50,7 +51,7 @@ public async ValueTask ReceiveAsync( TimeSpan timeout, CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return await ZeroTierTimeouts .RunWithTimeoutAsync(timeout, operation: "UDP receive", _incoming.Reader.ReadAsync, cancellationToken) .ConfigureAwait(false); @@ -59,7 +60,7 @@ public async ValueTask ReceiveAsync( public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(remoteEndpoint); - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); UdpEndpointNormalization.ValidateRemoteEndpoint(remoteEndpoint, nameof(remoteEndpoint)); return _udp.SendAsync(payload, remoteEndpoint, cancellationToken).AsTask(); } @@ -76,16 +77,44 @@ public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemo public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } - _disposed = true; - await _cts.CancelAsync().ConfigureAwait(false); - _udp.Dispose(); _incoming.Writer.TryComplete(); - _cts.Dispose(); + try + { + try + { + await _cts.CancelAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] CancelAsync failed: {ex}"); +#endif + } + + try + { + _udp.Dispose(); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] UdpClient dispose failed: {ex}"); +#endif + } + } + finally + { + _cts.Dispose(); + } try { @@ -94,6 +123,14 @@ public async ValueTask DisposeAsync() catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException) { } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] Receiver loop completion failed: {ex}"); +#endif + } } private async Task ProcessReceiveLoopAsync() From 5b1c9a016c33c8b4dd2019cb552e9baf656509d7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:22:35 +0100 Subject: [PATCH 099/296] fix: reject duplicate multipath UDP ports --- ...erSocketFactoryMultipathValidationTests.cs | 27 +++++++++++++++++++ ...ketRuntimeBootstrapperUdpTransportTests.cs | 9 +++++++ .../Internal/ZeroTierSocketFactory.cs | 7 +++++ .../ZeroTierSocketRuntimeBootstrapper.cs | 20 ++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs diff --git a/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs b/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs new file mode 100644 index 0000000..c0343c1 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs @@ -0,0 +1,27 @@ +using ZTSharp.ZeroTier; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierSocketFactoryMultipathValidationTests +{ + [Fact] + public async Task CreateAsync_RejectsDuplicateNonZeroLocalUdpPorts() + { + var options = new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-state-root-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = true, + UdpSocketCount = 2, + LocalUdpPorts = new[] { 12345, 12345 } + } + }; + + await Assert.ThrowsAsync(() => + ZeroTierSocketFactory.CreateAsync(options, CancellationToken.None)); + } +} + diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index fa3160e..a0a7e2b 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -82,6 +82,15 @@ public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() Assert.Fail($"Failed to bind a UDP socket to an available port after {attempts} attempts."); } + [Fact] + public async Task CreateUdpTransport_MultipathEnabled_RejectsDuplicateNonZeroLocalUdpPorts() + { + await Assert.ThrowsAsync(async () => + await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 2, LocalUdpPorts = new[] { 12345, 12345 } }, + enableIpv6: false)); + } + private static int GetAvailableUdpPort() { using var udp = new UdpClient(new IPEndPoint(IPAddress.Loopback, 0)); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs index 8c20a45..7c66e39 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs @@ -44,12 +44,19 @@ public static Task CreateAsync(ZeroTierSocketOptions options, Ca throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts length must match UdpSocketCount."); } + var seenNonZeroPorts = new HashSet(); for (var i = 0; i < ports.Count; i++) { if (ports[i] < 0 || ports[i] > 65535) { throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts entries must be in the range [0, 65535]."); } + + var port = ports[i]; + if (port != 0 && !seenNonZeroPorts.Add(port)) + { + throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts must not contain duplicate non-zero ports."); + } } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 050412c..e94ee87 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -37,6 +37,11 @@ internal static async ValueTask CreateUdpTransportAsync(Z port = localPorts[0]; } + if (port < 0 || port > 65535) + { + throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts entries must be in the range [0, 65535]."); + } + return new ZeroTierUdpTransport(localPort: port, enableIpv6: enableIpv6, localSocketId: 0); } @@ -51,6 +56,21 @@ internal static async ValueTask CreateUdpTransportAsync(Z throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts length must match UdpSocketCount."); } + var seenNonZeroPorts = new HashSet(); + for (var i = 0; i < ports.Count; i++) + { + var port = ports[i]; + if (port < 0 || port > 65535) + { + throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts entries must be in the range [0, 65535]."); + } + + if (port != 0 && !seenNonZeroPorts.Add(port)) + { + throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts must not contain duplicate non-zero ports."); + } + } + var sockets = new List(multipath.UdpSocketCount); var success = false; try From 1953045ad52f9cefc219f8cc1f66e4c629be2e05 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:23:09 +0100 Subject: [PATCH 100/296] fix: ensure IPv6-only UDP socket is v6-only --- ZTSharp.Tests/OsUdpSocketFactoryTests.cs | 25 +++++++++++++++++++ .../Transport/Internal/OsUdpSocketFactory.cs | 1 + 2 files changed, 26 insertions(+) diff --git a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs index 6082603..709d083 100644 --- a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs +++ b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs @@ -1,4 +1,5 @@ using System.Net.Sockets; +using System.Reflection; using ZTSharp.Transport.Internal; namespace ZTSharp.Tests; @@ -44,5 +45,29 @@ public void CreateSocketCore_WhenDualModeFails_TriesIpv4BeforeIpv6Only() socket.Dispose(); } } + + [Fact] + public void CreateUdp6OnlyBound_SetsDualModeFalse() + { + if (!Socket.OSSupportsIPv6) + { + throw Xunit.Sdk.SkipException.ForSkip("IPv6 not supported on this platform."); + } + + var method = typeof(OsUdpSocketFactory).GetMethod("CreateUdp6OnlyBound", BindingFlags.NonPublic | BindingFlags.Static); + Assert.NotNull(method); + + var udp = (UdpClient?)method!.Invoke(null, new object[] { 0 }); + Assert.NotNull(udp); + + try + { + Assert.False(udp!.Client.DualMode); + } + finally + { + udp!.Dispose(); + } + } } diff --git a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs index 332b435..b5de5a3 100644 --- a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs +++ b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs @@ -108,6 +108,7 @@ private static UdpClient CreateUdp6OnlyBound(int localPort) var udp6 = new UdpClient(AddressFamily.InterNetworkV6); try { + udp6.Client.DualMode = false; udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); return udp6; } From ad0c60f0bc1c49a3f0770d8b9b107ea257bf0994 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:25:15 +0100 Subject: [PATCH 101/296] fix: avoid unused dispose exceptions in Release --- ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs | 6 ++++++ ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs index 0cd10e2..3b94ed1 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs @@ -95,6 +95,8 @@ public async ValueTask DisposeAsync() { #if DEBUG Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] CancelAsync failed: {ex}"); +#else + _ = ex; #endif } @@ -110,6 +112,8 @@ public async ValueTask DisposeAsync() { #if DEBUG Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] Socket dispose failed: {ex}"); +#else + _ = ex; #endif } } @@ -126,6 +130,8 @@ public async ValueTask DisposeAsync() { #if DEBUG Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] Forwarder completion failed: {ex}"); +#else + _ = ex; #endif } diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index 6ef0007..fdf9c1b 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -95,6 +95,8 @@ public async ValueTask DisposeAsync() { #if DEBUG Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] CancelAsync failed: {ex}"); +#else + _ = ex; #endif } @@ -108,6 +110,8 @@ public async ValueTask DisposeAsync() { #if DEBUG Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] UdpClient dispose failed: {ex}"); +#else + _ = ex; #endif } } @@ -129,6 +133,8 @@ public async ValueTask DisposeAsync() { #if DEBUG Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] Receiver loop completion failed: {ex}"); +#else + _ = ex; #endif } } From e5a695915054891ccc8a30b1923ef35f3be3be4b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:43:47 +0100 Subject: [PATCH 102/296] Fix LocalUdpPorts duplicate validation when multipath off --- ...erSocketFactoryMultipathValidationTests.cs | 20 ++++++++++++++++++- .../Internal/ZeroTierSocketFactory.cs | 9 +++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs b/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs index c0343c1..c35f8be 100644 --- a/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs @@ -23,5 +23,23 @@ public async Task CreateAsync_RejectsDuplicateNonZeroLocalUdpPorts() await Assert.ThrowsAsync(() => ZeroTierSocketFactory.CreateAsync(options, CancellationToken.None)); } -} + [Fact] + public async Task CreateAsync_MultipathDisabled_AllowsDuplicateNonZeroLocalUdpPorts() + { + var options = new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-state-root-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = false, + UdpSocketCount = 2, + LocalUdpPorts = new[] { 12345, 12345 } + } + }; + + await using var socket = await ZeroTierSocketFactory.CreateAsync(options, CancellationToken.None); + Assert.NotNull(socket); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs index 7c66e39..3b0770e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs @@ -44,7 +44,12 @@ public static Task CreateAsync(ZeroTierSocketOptions options, Ca throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts length must match UdpSocketCount."); } - var seenNonZeroPorts = new HashSet(); + HashSet? seenNonZeroPorts = null; + if (options.Multipath.Enabled) + { + seenNonZeroPorts = new HashSet(); + } + for (var i = 0; i < ports.Count; i++) { if (ports[i] < 0 || ports[i] > 65535) @@ -53,7 +58,7 @@ public static Task CreateAsync(ZeroTierSocketOptions options, Ca } var port = ports[i]; - if (port != 0 && !seenNonZeroPorts.Add(port)) + if (seenNonZeroPorts is not null && port != 0 && !seenNonZeroPorts.Add(port)) { throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts must not contain duplicate non-zero ports."); } From fda07de7cc56449fec1d67fe43cbd659e694fb9c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:44:06 +0100 Subject: [PATCH 103/296] Skip IPv6 dual-mode test when bind fails --- ZTSharp.Tests/OsUdpSocketFactoryTests.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs index 709d083..84a973d 100644 --- a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs +++ b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs @@ -57,7 +57,15 @@ public void CreateUdp6OnlyBound_SetsDualModeFalse() var method = typeof(OsUdpSocketFactory).GetMethod("CreateUdp6OnlyBound", BindingFlags.NonPublic | BindingFlags.Static); Assert.NotNull(method); - var udp = (UdpClient?)method!.Invoke(null, new object[] { 0 }); + UdpClient? udp; + try + { + udp = (UdpClient?)method!.Invoke(null, new object[] { 0 }); + } + catch (TargetInvocationException ex) when (ex.InnerException is SocketException or PlatformNotSupportedException or NotSupportedException) + { + throw Xunit.Sdk.SkipException.ForSkip($"IPv6 appears supported, but binding an IPv6 UDP socket failed: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"); + } Assert.NotNull(udp); try From 088a320b23d7c5ca15b931456b40884cd932d7bb Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:44:30 +0100 Subject: [PATCH 104/296] Throw ObjectDisposedException from ZeroTierUdpTransport.LocalSockets --- ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index fdf9c1b..28ca8d4 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -37,7 +37,13 @@ public ZeroTierUdpTransport(int localPort = 0, bool enableIpv6 = true, Action UdpEndpointNormalization.Normalize((IPEndPoint)_udp.Client.LocalEndPoint!); public IReadOnlyList LocalSockets - => new[] { new ZeroTierUdpLocalSocket(_localSocketId, LocalEndpoint) }; + { + get + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + return new[] { new ZeroTierUdpLocalSocket(_localSocketId, LocalEndpoint) }; + } + } public long IncomingBackpressureCount => Interlocked.Read(ref _incomingBackpressureCount); From 02cf1866ebf52140f2ff2bdd22fda58c0354b807 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:44:54 +0100 Subject: [PATCH 105/296] Avoid disposing SemaphoreSlim locks in DisposeAsync --- ZTSharp/Sockets/OverlayTcpClient.cs | 2 -- ZTSharp/Sockets/OverlayTcpListener.cs | 1 - ZTSharp/Sockets/OverlayTcpPortForwarder.cs | 1 - ZTSharp/Sockets/ZtUdpClient.cs | 1 - ZTSharp/VirtualNetworkInterface.cs | 2 -- 5 files changed, 7 deletions(-) diff --git a/ZTSharp/Sockets/OverlayTcpClient.cs b/ZTSharp/Sockets/OverlayTcpClient.cs index 46add9a..1b0954b 100644 --- a/ZTSharp/Sockets/OverlayTcpClient.cs +++ b/ZTSharp/Sockets/OverlayTcpClient.cs @@ -179,8 +179,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); - _sendLock.Dispose(); } } diff --git a/ZTSharp/Sockets/OverlayTcpListener.cs b/ZTSharp/Sockets/OverlayTcpListener.cs index 2a58bf1..dfc8683 100644 --- a/ZTSharp/Sockets/OverlayTcpListener.cs +++ b/ZTSharp/Sockets/OverlayTcpListener.cs @@ -60,7 +60,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); } } diff --git a/ZTSharp/Sockets/OverlayTcpPortForwarder.cs b/ZTSharp/Sockets/OverlayTcpPortForwarder.cs index 3f0fb29..96039bd 100644 --- a/ZTSharp/Sockets/OverlayTcpPortForwarder.cs +++ b/ZTSharp/Sockets/OverlayTcpPortForwarder.cs @@ -111,7 +111,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); _shutdown.Dispose(); } } diff --git a/ZTSharp/Sockets/ZtUdpClient.cs b/ZTSharp/Sockets/ZtUdpClient.cs index bd16561..292e411 100644 --- a/ZTSharp/Sockets/ZtUdpClient.cs +++ b/ZTSharp/Sockets/ZtUdpClient.cs @@ -134,7 +134,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); } } diff --git a/ZTSharp/VirtualNetworkInterface.cs b/ZTSharp/VirtualNetworkInterface.cs index 8004f4e..f8d9fa5 100644 --- a/ZTSharp/VirtualNetworkInterface.cs +++ b/ZTSharp/VirtualNetworkInterface.cs @@ -57,7 +57,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); } } @@ -114,4 +113,3 @@ private void OnFrameReceived(in RawFrame frame) DateTimeOffset.UtcNow)); } } - From 24117f51896b3b17a9add87180b211de2d3165e3 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:45:25 +0100 Subject: [PATCH 106/296] Dispose queued OverlayTcpClient instances on listener shutdown --- ZTSharp/Sockets/OverlayTcpListener.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ZTSharp/Sockets/OverlayTcpListener.cs b/ZTSharp/Sockets/OverlayTcpListener.cs index dfc8683..d78c1bb 100644 --- a/ZTSharp/Sockets/OverlayTcpListener.cs +++ b/ZTSharp/Sockets/OverlayTcpListener.cs @@ -56,6 +56,10 @@ public async ValueTask DisposeAsync() _disposed = true; _node.RawFrameReceived -= OnFrameReceived; _acceptQueue.Writer.TryComplete(); + while (_acceptQueue.Reader.TryRead(out var queued)) + { + ObserveBestEffortAsync(queued.DisposeAsync().AsTask()); + } } finally { From 95e49a7c924b94704d2462009da49399db8edd8d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:46:26 +0100 Subject: [PATCH 107/296] Suppress CA2213 for idempotent DisposeAsync locks --- ZTSharp/Sockets/OverlayTcpClient.cs | 10 ++++++++++ ZTSharp/Sockets/OverlayTcpListener.cs | 5 +++++ ZTSharp/Sockets/OverlayTcpPortForwarder.cs | 5 +++++ ZTSharp/Sockets/ZtUdpClient.cs | 5 +++++ ZTSharp/VirtualNetworkInterface.cs | 5 +++++ 5 files changed, 30 insertions(+) diff --git a/ZTSharp/Sockets/OverlayTcpClient.cs b/ZTSharp/Sockets/OverlayTcpClient.cs index 1b0954b..b7c7542 100644 --- a/ZTSharp/Sockets/OverlayTcpClient.cs +++ b/ZTSharp/Sockets/OverlayTcpClient.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Sockets; @@ -13,7 +14,16 @@ public sealed class OverlayTcpClient : IAsyncDisposable private const int MaxDataPerFrame = 1024; private readonly OverlayTcpIncomingBuffer _incoming; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); + + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _sendLock = new(1, 1); private readonly Node _node; private readonly ulong _networkId; diff --git a/ZTSharp/Sockets/OverlayTcpListener.cs b/ZTSharp/Sockets/OverlayTcpListener.cs index d78c1bb..1bb91d0 100644 --- a/ZTSharp/Sockets/OverlayTcpListener.cs +++ b/ZTSharp/Sockets/OverlayTcpListener.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Diagnostics.CodeAnalysis; using System.Net.Sockets; using System.Threading.Channels; @@ -12,6 +13,10 @@ public sealed class OverlayTcpListener : IAsyncDisposable private const int HeaderLength = OverlayTcpFrameCodec.HeaderLength; private readonly Channel _acceptQueue; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly Node _node; private readonly ulong _networkId; diff --git a/ZTSharp/Sockets/OverlayTcpPortForwarder.cs b/ZTSharp/Sockets/OverlayTcpPortForwarder.cs index 96039bd..25d0813 100644 --- a/ZTSharp/Sockets/OverlayTcpPortForwarder.cs +++ b/ZTSharp/Sockets/OverlayTcpPortForwarder.cs @@ -1,4 +1,5 @@ using System.Net.Sockets; +using System.Diagnostics.CodeAnalysis; using System.Threading.Channels; using ZTSharp.Internal; using SystemTcpClient = System.Net.Sockets.TcpClient; @@ -10,6 +11,10 @@ namespace ZTSharp.Sockets; /// public sealed class OverlayTcpPortForwarder : IAsyncDisposable { + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly CancellationTokenSource _shutdown = new(); private readonly ActiveTaskSet _connectionTasks = new(); diff --git a/ZTSharp/Sockets/ZtUdpClient.cs b/ZTSharp/Sockets/ZtUdpClient.cs index 292e411..7108fbf 100644 --- a/ZTSharp/Sockets/ZtUdpClient.cs +++ b/ZTSharp/Sockets/ZtUdpClient.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Buffers; +using System.Diagnostics.CodeAnalysis; using System.Threading.Channels; namespace ZTSharp.Sockets; @@ -16,6 +17,10 @@ public sealed class ZtUdpClient : IAsyncDisposable private const int UdpFrameHeaderV2Length = 14; private readonly Channel _incoming; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly ulong _networkId; private readonly int _localPort; diff --git a/ZTSharp/VirtualNetworkInterface.cs b/ZTSharp/VirtualNetworkInterface.cs index f8d9fa5..02a3c86 100644 --- a/ZTSharp/VirtualNetworkInterface.cs +++ b/ZTSharp/VirtualNetworkInterface.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Buffers.Binary; +using System.Diagnostics.CodeAnalysis; using System.Threading.Channels; namespace ZTSharp; @@ -14,6 +15,10 @@ public sealed class VirtualNetworkInterface : IAsyncDisposable private const int HeaderLength = 1 + 1 + sizeof(ulong); private readonly Channel _incoming; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly Node _node; private readonly ulong _networkId; From be1131098a2e0364e10b95ece0dd75ff8853adcc Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:57:23 +0100 Subject: [PATCH 108/296] fix(overlay): avoid multi-reader accept queue and writes-after-dispose --- ZTSharp/Sockets/OverlayTcpClient.cs | 7 +++++++ ZTSharp/Sockets/OverlayTcpListener.cs | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ZTSharp/Sockets/OverlayTcpClient.cs b/ZTSharp/Sockets/OverlayTcpClient.cs index b7c7542..cc564ca 100644 --- a/ZTSharp/Sockets/OverlayTcpClient.cs +++ b/ZTSharp/Sockets/OverlayTcpClient.cs @@ -143,10 +143,17 @@ internal async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationTok await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_incoming.RemoteClosed || _incoming.RemoteFinReceived) + { + throw new IOException("Remote has closed the connection."); + } + var remaining = buffer; while (!remaining.IsEmpty) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); var chunk = remaining.Slice(0, Math.Min(remaining.Length, MaxDataPerFrame)); remaining = remaining.Slice(chunk.Length); diff --git a/ZTSharp/Sockets/OverlayTcpListener.cs b/ZTSharp/Sockets/OverlayTcpListener.cs index 1bb91d0..eef582d 100644 --- a/ZTSharp/Sockets/OverlayTcpListener.cs +++ b/ZTSharp/Sockets/OverlayTcpListener.cs @@ -39,7 +39,8 @@ public OverlayTcpListener(Node node, ulong networkId, int localPort) { FullMode = BoundedChannelFullMode.Wait, SingleWriter = false, - SingleReader = true + // DisposeAsync drains queued connections for best-effort cleanup, so we must allow an additional reader. + SingleReader = false }); _node.RawFrameReceived += OnFrameReceived; From 13fff12484922b8056ad6d83cc651e8b1e41975b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:10:37 +0100 Subject: [PATCH 109/296] fix(transport): make OsUdpNodeTransport DisposeAsync idempotent --- ZTSharp/Transport/OsUdpNodeTransport.cs | 54 +++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index e006276..4865048 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -2,6 +2,7 @@ using System.Buffers; using System.Net; using System.Net.Sockets; +using System.Diagnostics.CodeAnalysis; using ZTSharp.Transport.Internal; namespace ZTSharp.Transport; @@ -18,6 +19,10 @@ private sealed record Subscriber( Func, CancellationToken, Task> OnFrameReceived); private readonly UdpClient _udp; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _gate = new(1, 1); private readonly ConcurrentDictionary> _networkSubscribers = new(); private readonly ConcurrentDictionary _advertisedEndpoints = new(); @@ -29,6 +34,9 @@ private sealed record Subscriber( private readonly CancellationTokenSource _peerRefreshCts = new(); private readonly Task? _peerRefreshLoop; + private int _disposeState; + private bool _disposed; + public OsUdpNodeTransport(int localPort = 0, bool enableIpv6 = true, bool enablePeerDiscovery = true) { _enablePeerDiscovery = enablePeerDiscovery; @@ -54,6 +62,7 @@ public IPEndPoint LocalEndpoint { get { + ObjectDisposedException.ThrowIf(_disposed, this); return UdpEndpointNormalization.Normalize((IPEndPoint)_udp.Client.LocalEndPoint!); } } @@ -68,6 +77,7 @@ public async Task JoinNetworkAsync( ArgumentOutOfRangeException.ThrowIfZero(nodeId); ArgumentNullException.ThrowIfNull(onFrameReceived); cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); var registrationId = Guid.NewGuid(); var advertisedEndpoint = localEndpoint is null @@ -95,6 +105,7 @@ await SendDiscoveryFrameAsync( public async Task LeaveNetworkAsync(ulong networkId, Guid registrationId, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); if (_networkSubscribers.TryGetValue(networkId, out var subscribers) && subscribers.TryGetValue(registrationId, out var localSubscriber)) { @@ -130,6 +141,7 @@ public async Task SendFrameAsync( CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); if (!_peers.TryGetPeers(networkId, out var peers)) { return; @@ -182,6 +194,7 @@ await _udp public Task FlushAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); return Task.CompletedTask; } @@ -189,6 +202,7 @@ public async ValueTask AddPeerAsync(ulong networkId, ulong nodeId, IPEndPoint en { ArgumentOutOfRangeException.ThrowIfZero(nodeId); ArgumentNullException.ThrowIfNull(endpoint); + ObjectDisposedException.ThrowIf(_disposed, this); UdpEndpointNormalization.ValidateRemoteEndpoint(endpoint, nameof(endpoint)); var remoteEndpoint = UdpEndpointNormalization.Normalize(endpoint); @@ -221,8 +235,28 @@ await SendDiscoveryFrameAsync( public async ValueTask DisposeAsync() { - await _receiverCts.CancelAsync().ConfigureAwait(false); - await _peerRefreshCts.CancelAsync().ConfigureAwait(false); + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + { + return; + } + + _disposed = true; + try + { + await _receiverCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + + try + { + await _peerRefreshCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + try { await _receiverLoop.ConfigureAwait(false); @@ -230,6 +264,9 @@ public async ValueTask DisposeAsync() catch (OperationCanceledException) when (_receiverCts.IsCancellationRequested) { } + catch (ObjectDisposedException) + { + } if (_peerRefreshLoop is not null) { @@ -240,13 +277,22 @@ public async ValueTask DisposeAsync() catch (OperationCanceledException) when (_peerRefreshCts.IsCancellationRequested) { } + catch (ObjectDisposedException) + { + } + } + + try + { + _udp.Dispose(); + } + catch (ObjectDisposedException) + { } - _udp.Dispose(); _peers.Cleanup(); _receiverCts.Dispose(); _peerRefreshCts.Dispose(); - _gate.Dispose(); } private async Task RefreshLocalDiscoveryEntriesAsync(CancellationToken cancellationToken) From 9a998a61eeb86ebedf483212071415f4c109162e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:11:43 +0100 Subject: [PATCH 110/296] fix(tcp): make UserSpaceTcpSender disposal concurrency-safe --- ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs index 0f9bbb1..26e6d5f 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; @@ -12,6 +13,10 @@ internal sealed class UserSpaceTcpSender : IAsyncDisposable private readonly UserSpaceTcpRemoteSendWindow _remoteSendWindow = new(); private readonly UserSpaceTcpRtoEstimator _rtoEstimator = new(); + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw in concurrent send paths.")] private readonly SemaphoreSlim _sendLock = new(1, 1); private int _effectiveMss; @@ -26,6 +31,7 @@ internal sealed class UserSpaceTcpSender : IAsyncDisposable private ushort _lastAdvertisedWindow = ushort.MaxValue; private readonly UserSpaceTcpWindowUpdateTrigger _windowUpdateTrigger; + private int _disposeState; private bool _disposed; public UserSpaceTcpSender( @@ -116,6 +122,7 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, Func getAck await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { + ObjectDisposedException.ThrowIf(_disposed, this); if (_finSeq is not null) { throw new IOException("Local has closed the connection."); @@ -125,6 +132,7 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, Func getAck while (!remaining.IsEmpty) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); await WaitForRemoteSendWindowAsync(cancellationToken).ConfigureAwait(false); @@ -190,6 +198,7 @@ public async Task SendFinWithRetriesAsync(uint ack, CancellationToken cancellati await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { + ObjectDisposedException.ThrowIf(_disposed, this); if (_finSeq is null) { _finSeq = AllocateNextSequence(bytes: 1); @@ -318,8 +327,12 @@ private async Task WaitForRemoteSendWindowAsync(CancellationToken cancellationTo public ValueTask DisposeAsync() { + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + { + return ValueTask.CompletedTask; + } + _disposed = true; - _sendLock.Dispose(); return ValueTask.CompletedTask; } } From 006ac9fda0ccd8aa4a0a7736547d17352383576b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:12:43 +0100 Subject: [PATCH 111/296] fix(tcp): make UserSpaceTcp client/server DisposeAsync idempotent --- ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs | 3 ++- ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs index 84b99cf..c645d7c 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs @@ -26,6 +26,7 @@ internal sealed class UserSpaceTcpClient : IAsyncDisposable private Task? _receiveLoopTask; private bool _disposed; + private int _disposeState; public UserSpaceTcpClient( IUserSpaceIpLink link, @@ -181,7 +182,7 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposeState, 1) != 0) { return; } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs index d048d51..7c0a410 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs @@ -20,6 +20,7 @@ internal sealed class UserSpaceTcpServerConnection : IAsyncDisposable private Task? _receiveLoopTask; private bool _disposed; + private int _disposeState; public UserSpaceTcpServerConnection( IUserSpaceIpLink link, @@ -130,7 +131,7 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposeState, 1) != 0) { return; } From 832b22c087d4aadc7df86f6b2a5b7783e3cfe490 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:30:06 +0100 Subject: [PATCH 112/296] fix(node): avoid disposing lifecycle state lock --- ZTSharp/Internal/NodeLifecycleService.cs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ZTSharp/Internal/NodeLifecycleService.cs b/ZTSharp/Internal/NodeLifecycleService.cs index 10dbe27..9ea7b8b 100644 --- a/ZTSharp/Internal/NodeLifecycleService.cs +++ b/ZTSharp/Internal/NodeLifecycleService.cs @@ -1,11 +1,16 @@ using Microsoft.Extensions.Logging; using ZTSharp.Transport; +using System.Diagnostics.CodeAnalysis; namespace ZTSharp.Internal; internal sealed class NodeLifecycleService : IAsyncDisposable { private readonly NodeRuntimeState _runtime; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be safe during concurrent lifecycle operations; disposing this lock can throw on in-flight Release calls.")] private readonly SemaphoreSlim _stateLock; private readonly CancellationTokenSource _nodeCts; private readonly INodeTransport _transport; @@ -17,6 +22,8 @@ internal sealed class NodeLifecycleService : IAsyncDisposable private readonly NodeTransportService _transportService; private readonly bool _ownsTransport; + private int _disposeState; + public NodeLifecycleService( NodeRuntimeState runtime, SemaphoreSlim stateLock, @@ -197,12 +204,23 @@ public async Task ExecuteWhileRunningAsync(Func public async ValueTask DisposeAsync() { + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + { + return; + } + if (_runtime.Disposed) { return; } - await _nodeCts.CancelAsync().ConfigureAwait(false); + try + { + await _nodeCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } using var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); try @@ -228,7 +246,6 @@ public async ValueTask DisposeAsync() await asyncTransport.DisposeAsync().ConfigureAwait(false); } - _stateLock.Dispose(); _events.Complete(); _nodeCts.Dispose(); } From 67d5ae2fd20c37181decf307ae73f279c897aaa3 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:30:14 +0100 Subject: [PATCH 113/296] fix(node): prevent leave-gate map growth --- ZTSharp/Internal/NodeNetworkService.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ZTSharp/Internal/NodeNetworkService.cs b/ZTSharp/Internal/NodeNetworkService.cs index 18ec675..7a40f2b 100644 --- a/ZTSharp/Internal/NodeNetworkService.cs +++ b/ZTSharp/Internal/NodeNetworkService.cs @@ -16,7 +16,7 @@ internal sealed class NodeNetworkService private readonly ConcurrentDictionary _joinedNetworks = new(); private readonly ConcurrentDictionary _networkRegistrations = new(); - private readonly ConcurrentDictionary _leaveGates = new(); + private readonly SemaphoreSlim _leaveGate = new(1, 1); private readonly NetworkIdReadOnlyCollection _joinedNetworkIds; public NodeNetworkService(IStateStore store, INodeTransport transport, NodeEventStream events, NodePeerService peerService) @@ -86,8 +86,7 @@ public async Task JoinNetworkAsync( public async Task LeaveNetworkAsync(ulong networkId, CancellationToken cancellationToken) { var key = BuildNetworkFileKey(networkId); - var gate = _leaveGates.GetOrAdd(networkId, static _ => new SemaphoreSlim(1, 1)); - await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + await _leaveGate.WaitAsync(cancellationToken).ConfigureAwait(false); try { if (_networkRegistrations.TryGetValue(networkId, out var registration)) @@ -105,7 +104,7 @@ public async Task LeaveNetworkAsync(ulong networkId, CancellationToken cancellat } finally { - gate.Release(); + _leaveGate.Release(); } } @@ -223,8 +222,7 @@ public async Task UnregisterAllNetworksAsync(CancellationToken cancellationToken { foreach (var kv in _networkRegistrations) { - var gate = _leaveGates.GetOrAdd(kv.Key, static _ => new SemaphoreSlim(1, 1)); - await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + await _leaveGate.WaitAsync(cancellationToken).ConfigureAwait(false); try { if (_networkRegistrations.TryGetValue(kv.Key, out var registration)) @@ -235,7 +233,7 @@ public async Task UnregisterAllNetworksAsync(CancellationToken cancellationToken } finally { - gate.Release(); + _leaveGate.Release(); } } From a9a317ada8baa1bd0dd7c4cf177e5cdfd943ef6f Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:30:21 +0100 Subject: [PATCH 114/296] fix(tcp): unblock pending sender waits on dispose --- ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs index 26e6d5f..7a99d1c 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs @@ -32,7 +32,7 @@ internal sealed class UserSpaceTcpSender : IAsyncDisposable private readonly UserSpaceTcpWindowUpdateTrigger _windowUpdateTrigger; private int _disposeState; - private bool _disposed; + private volatile bool _disposed; public UserSpaceTcpSender( IUserSpaceIpLink link, @@ -333,6 +333,7 @@ public ValueTask DisposeAsync() } _disposed = true; + FailPendingOperations(new ObjectDisposedException(nameof(UserSpaceTcpSender))); return ValueTask.CompletedTask; } } From ee7912a7db73fa6aa371d36adb3f3425c3a97e04 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:30:25 +0100 Subject: [PATCH 115/296] fix(transport): make disposed flag volatile --- ZTSharp/Transport/OsUdpNodeTransport.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index 4865048..4e48375 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -35,7 +35,7 @@ private sealed record Subscriber( private readonly Task? _peerRefreshLoop; private int _disposeState; - private bool _disposed; + private volatile bool _disposed; public OsUdpNodeTransport(int localPort = 0, bool enableIpv6 = true, bool enablePeerDiscovery = true) { From 39663e321aaa79115e652c58b53c7626dd7cafe5 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:31:51 +0100 Subject: [PATCH 116/296] chore(analyzers): suppress CA1001 for NodeNetworkService gate --- ZTSharp/Internal/NodeNetworkService.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ZTSharp/Internal/NodeNetworkService.cs b/ZTSharp/Internal/NodeNetworkService.cs index 7a40f2b..ba65d97 100644 --- a/ZTSharp/Internal/NodeNetworkService.cs +++ b/ZTSharp/Internal/NodeNetworkService.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Net; using System.Text.Json; @@ -7,6 +8,10 @@ namespace ZTSharp.Internal; +[SuppressMessage( + "Design", + "CA1001:Types that own disposable fields should be disposable", + Justification = "This service is owned by NodeCore/NodeLifecycleService; the async gate is intentionally not disposed to keep shutdown idempotent and avoid races with in-flight leave operations.")] internal sealed class NodeNetworkService { private readonly IStateStore _store; From aa678c8f0aa6fb39a8fc21d3f26f009576c3c9e3 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:46:24 +0100 Subject: [PATCH 117/296] fix(node): serialize disposal with lifecycle operations --- ZTSharp/Internal/NodeLifecycleService.cs | 65 +++++++++++++++++++++--- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/ZTSharp/Internal/NodeLifecycleService.cs b/ZTSharp/Internal/NodeLifecycleService.cs index 9ea7b8b..74bb9bc 100644 --- a/ZTSharp/Internal/NodeLifecycleService.cs +++ b/ZTSharp/Internal/NodeLifecycleService.cs @@ -209,9 +209,21 @@ public async ValueTask DisposeAsync() return; } - if (_runtime.Disposed) + // Serialize disposal with other lifecycle operations (Start/Stop/ExecuteWhileRunning), + // so we don't dispose the transport while another operation is mid-flight. + await _stateLock.WaitAsync().ConfigureAwait(false); + try { - return; + if (_runtime.Disposed) + { + return; + } + + _runtime.Disposed = true; + } + finally + { + _stateLock.Release(); } try @@ -222,14 +234,46 @@ public async ValueTask DisposeAsync() { } - using var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + using var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + try { - await StopAsync(shutdownCts.Token).ConfigureAwait(false); + await _stateLock.WaitAsync(shutdownCts.Token).ConfigureAwait(false); + try + { + if (_runtime.State is not NodeState.Stopped and not NodeState.Faulted) + { + _runtime.State = NodeState.Stopping; + _events.Publish(EventCode.NodeStopping, DateTimeOffset.UtcNow); + + await _networkService.UnregisterAllNetworksAsync(shutdownCts.Token).ConfigureAwait(false); + await _transport.FlushAsync(shutdownCts.Token).ConfigureAwait(false); + await _store.FlushAsync(shutdownCts.Token).ConfigureAwait(false); + _runtime.State = NodeState.Stopped; + _events.Publish(EventCode.NodeStopped, DateTimeOffset.UtcNow); + } + } + finally + { + _stateLock.Release(); + } } catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) { } + catch (ObjectDisposedException) + { + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { + _runtime.State = NodeState.Faulted; +#pragma warning disable CA1848 + _logger.LogError(ex, "Failed to stop node during disposal"); +#pragma warning restore CA1848 + _events.Publish(EventCode.NodeFaulted, DateTimeOffset.UtcNow, message: ex.Message, error: ex); + } try { @@ -238,12 +282,19 @@ public async ValueTask DisposeAsync() catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) { } - - _runtime.Disposed = true; + catch (ObjectDisposedException) + { + } if (_ownsTransport && _transport is IAsyncDisposable asyncTransport) { - await asyncTransport.DisposeAsync().ConfigureAwait(false); + try + { + await asyncTransport.DisposeAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } } _events.Complete(); From 9392a40f9ff4b58eb80195c3ca49a4983f458673 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:48:22 +0100 Subject: [PATCH 118/296] fix(node): make runtime Disposed flag thread-safe --- ZTSharp/Internal/NodeRuntimeState.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ZTSharp/Internal/NodeRuntimeState.cs b/ZTSharp/Internal/NodeRuntimeState.cs index e2c3e42..5814d78 100644 --- a/ZTSharp/Internal/NodeRuntimeState.cs +++ b/ZTSharp/Internal/NodeRuntimeState.cs @@ -6,6 +6,12 @@ internal sealed class NodeRuntimeState public NodeId NodeId { get; set; } - public bool Disposed { get; set; } + private int _disposed; + + public bool Disposed + { + get => Volatile.Read(ref _disposed) != 0; + set => Volatile.Write(ref _disposed, value ? 1 : 0); + } } From 0160eb34c30367a94c8c9824af0d7b2d7f7d6fd2 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:48:27 +0100 Subject: [PATCH 119/296] fix(node): avoid wedging dispose and block ops early --- ZTSharp/Internal/NodeLifecycleService.cs | 32 +++++++++++++++--------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/ZTSharp/Internal/NodeLifecycleService.cs b/ZTSharp/Internal/NodeLifecycleService.cs index 74bb9bc..09fd152 100644 --- a/ZTSharp/Internal/NodeLifecycleService.cs +++ b/ZTSharp/Internal/NodeLifecycleService.cs @@ -209,21 +209,31 @@ public async ValueTask DisposeAsync() return; } - // Serialize disposal with other lifecycle operations (Start/Stop/ExecuteWhileRunning), - // so we don't dispose the transport while another operation is mid-flight. - await _stateLock.WaitAsync().ConfigureAwait(false); + using var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + try { - if (_runtime.Disposed) + // Best-effort: block new operations promptly, but don't wedge forever if some operation is stuck. + await _stateLock.WaitAsync(shutdownCts.Token).ConfigureAwait(false); + try { - return; - } + if (_runtime.Disposed) + { + return; + } - _runtime.Disposed = true; + _runtime.Disposed = true; + } + finally + { + _stateLock.Release(); + } } - finally + catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) { - _stateLock.Release(); + // Worst case: a concurrent lifecycle operation is wedged under the state lock. Mark the node as disposed + // anyway so future operations fail fast. + _runtime.Disposed = true; } try @@ -234,8 +244,6 @@ public async ValueTask DisposeAsync() { } - using var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - try { await _stateLock.WaitAsync(shutdownCts.Token).ConfigureAwait(false); @@ -303,6 +311,6 @@ public async ValueTask DisposeAsync() private void EnsureNotDisposed() { - ObjectDisposedException.ThrowIf(_runtime.Disposed, nameof(Node)); + ObjectDisposedException.ThrowIf(_runtime.Disposed || Volatile.Read(ref _disposeState) != 0, nameof(Node)); } } From 246b477f89a7d15b1b40fb1699a6ed59079c0bdd Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:02:34 +0100 Subject: [PATCH 120/296] fix(node): make JoinNetworkAsync idempotent --- .../NodeNetworkLeaveOrderingTests.cs | 55 +++++++++++++++++++ ZTSharp/Internal/NodeNetworkService.cs | 11 ++++ 2 files changed, 66 insertions(+) diff --git a/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs b/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs index 0bb0bbd..46d57f7 100644 --- a/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs +++ b/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs @@ -6,6 +6,34 @@ namespace ZTSharp.Tests; public sealed class NodeNetworkLeaveOrderingTests { + [Fact] + public async Task JoinNetworkAsync_DuplicateJoin_IsIdempotentAndDoesNotLeakRegistrations() + { + var store = new MemoryStateStore(); + var transport = new RecordingJoinTransport(); + var events = new NodeEventStream(_ => { }, NullLogger.Instance); + var peers = new NodePeerService(store); + var service = new NodeNetworkService(store, transport, events, peers); + + var networkId = 0xCAFE5003UL; + + await service.JoinNetworkAsync( + networkId, + localNodeId: 1, + localEndpoint: null, + onFrameReceived: (_, _, _, _) => Task.CompletedTask, + CancellationToken.None); + + await service.JoinNetworkAsync( + networkId, + localNodeId: 1, + localEndpoint: null, + onFrameReceived: (_, _, _, _) => Task.CompletedTask, + CancellationToken.None); + + Assert.Equal(1, transport.JoinCallCount); + } + [Fact] public async Task LeaveNetworkAsync_FailedLeave_DoesNotLoseRegistration_AndRetryUnsubscribes() { @@ -155,4 +183,31 @@ public Task FlushAsync(CancellationToken cancellationToken = default) public void Dispose() => Inner.Dispose(); } + + private sealed class RecordingJoinTransport : INodeTransport + { + private int _joinCallCount; + + public int JoinCallCount => Volatile.Read(ref _joinCallCount); + + public Task JoinNetworkAsync( + ulong networkId, + ulong nodeId, + Func, CancellationToken, Task> onFrameReceived, + System.Net.IPEndPoint? localEndpoint = null, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _joinCallCount); + return Task.FromResult(Guid.NewGuid()); + } + + public Task LeaveNetworkAsync(ulong networkId, Guid registrationId, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendFrameAsync(ulong networkId, ulong sourceNodeId, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task FlushAsync(CancellationToken cancellationToken = default) + => Task.CompletedTask; + } } diff --git a/ZTSharp/Internal/NodeNetworkService.cs b/ZTSharp/Internal/NodeNetworkService.cs index ba65d97..bd29235 100644 --- a/ZTSharp/Internal/NodeNetworkService.cs +++ b/ZTSharp/Internal/NodeNetworkService.cs @@ -48,6 +48,12 @@ public async Task JoinNetworkAsync( { _events.Publish(EventCode.NetworkJoinRequested, DateTimeOffset.UtcNow, networkId); + // Joining a network is idempotent: avoid leaking registrations/subscribers and keep the transport state consistent. + if (_networkRegistrations.ContainsKey(networkId)) + { + return; + } + Guid registration = default; var now = DateTimeOffset.UtcNow; var key = BuildNetworkFileKey(networkId); @@ -206,6 +212,11 @@ public async Task RecoverNetworksAsync( foreach (var network in _joinedNetworks.Keys) { + if (_networkRegistrations.ContainsKey(network)) + { + continue; + } + var registration = await _transport.JoinNetworkAsync( network, localNodeId, From 69af99126fb48c41377fb0740a8bd3556e5f7700 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:02:43 +0100 Subject: [PATCH 121/296] fix(transport): only teardown OS-UDP network state when last subscriber leaves --- ZTSharp/Transport/OsUdpNodeTransport.cs | 32 +++++++++++++------------ 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index 4e48375..81478d6 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -106,27 +106,29 @@ public async Task LeaveNetworkAsync(ulong networkId, Guid registrationId, Cancel { cancellationToken.ThrowIfCancellationRequested(); ObjectDisposedException.ThrowIf(_disposed, this); - if (_networkSubscribers.TryGetValue(networkId, out var subscribers) && - subscribers.TryGetValue(registrationId, out var localSubscriber)) - { - _ = _peers.TryRemoveLocalNodeIdIfMatch(networkId, localSubscriber.NodeId); - } - - _peers.RemoveNetworkPeers(networkId); - _advertisedEndpoints.TryRemove(networkId, out _); - if (!_networkSubscribers.TryGetValue(networkId, out var networkSubscribers)) - { - return; - } await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { - networkSubscribers.TryRemove(registrationId, out _); - if (networkSubscribers.IsEmpty) + if (!_networkSubscribers.TryGetValue(networkId, out var networkSubscribers)) { - _networkSubscribers.TryRemove(networkId, out _); + return; } + + if (!networkSubscribers.TryRemove(registrationId, out var localSubscriber)) + { + return; + } + + if (!networkSubscribers.IsEmpty) + { + return; + } + + _networkSubscribers.TryRemove(networkId, out _); + _advertisedEndpoints.TryRemove(networkId, out _); + _ = _peers.TryRemoveLocalNodeIdIfMatch(networkId, localSubscriber.NodeId); + _peers.RemoveNetworkPeers(networkId); } finally { From 967cead8980fe3f14a42c49c381be677be15b3ed Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:06:35 +0100 Subject: [PATCH 122/296] fix(node): time-bound transport disposal --- ZTSharp/Internal/NodeLifecycleService.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ZTSharp/Internal/NodeLifecycleService.cs b/ZTSharp/Internal/NodeLifecycleService.cs index 09fd152..f832e57 100644 --- a/ZTSharp/Internal/NodeLifecycleService.cs +++ b/ZTSharp/Internal/NodeLifecycleService.cs @@ -296,13 +296,24 @@ public async ValueTask DisposeAsync() if (_ownsTransport && _transport is IAsyncDisposable asyncTransport) { + using var transportDisposeCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); try { - await asyncTransport.DisposeAsync().ConfigureAwait(false); + await asyncTransport + .DisposeAsync() + .AsTask() + .WaitAsync(transportDisposeCts.Token) + .ConfigureAwait(false); } catch (ObjectDisposedException) { } + catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) + { + } + catch (OperationCanceledException) when (transportDisposeCts.IsCancellationRequested) + { + } } _events.Complete(); From aacf56c1ab7a1cf9494ad97a5b04c1c7023c6258 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:06:41 +0100 Subject: [PATCH 123/296] fix(transport): time-bound OS-UDP receiver shutdown --- ZTSharp/Transport/OsUdpNodeTransport.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index 81478d6..c149c86 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -13,6 +13,7 @@ namespace ZTSharp.Transport; internal sealed class OsUdpNodeTransport : INodeTransport, IAsyncDisposable { private static readonly TimeSpan PeerDiscoveryRefreshInterval = TimeSpan.FromSeconds(90); + private static readonly TimeSpan ShutdownTimeout = TimeSpan.FromSeconds(1); private sealed record Subscriber( ulong NodeId, @@ -261,7 +262,10 @@ public async ValueTask DisposeAsync() try { - await _receiverLoop.ConfigureAwait(false); + await _receiverLoop.WaitAsync(ShutdownTimeout).ConfigureAwait(false); + } + catch (TimeoutException) + { } catch (OperationCanceledException) when (_receiverCts.IsCancellationRequested) { @@ -274,7 +278,10 @@ public async ValueTask DisposeAsync() { try { - await _peerRefreshLoop.ConfigureAwait(false); + await _peerRefreshLoop.WaitAsync(ShutdownTimeout).ConfigureAwait(false); + } + catch (TimeoutException) + { } catch (OperationCanceledException) when (_peerRefreshCts.IsCancellationRequested) { From 2a1b31a1cc3f99306f2d6ab79f3f88c8c3b4f1d1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:20:33 +0100 Subject: [PATCH 124/296] fix(node): make peer recovery best-effort and avoid join inconsistent state --- ZTSharp/Internal/NodeNetworkService.cs | 34 +++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/ZTSharp/Internal/NodeNetworkService.cs b/ZTSharp/Internal/NodeNetworkService.cs index bd29235..bda7754 100644 --- a/ZTSharp/Internal/NodeNetworkService.cs +++ b/ZTSharp/Internal/NodeNetworkService.cs @@ -46,14 +46,14 @@ public async Task JoinNetworkAsync( Func, CancellationToken, Task> onFrameReceived, CancellationToken cancellationToken) { - _events.Publish(EventCode.NetworkJoinRequested, DateTimeOffset.UtcNow, networkId); - // Joining a network is idempotent: avoid leaking registrations/subscribers and keep the transport state consistent. if (_networkRegistrations.ContainsKey(networkId)) { return; } + _events.Publish(EventCode.NetworkJoinRequested, DateTimeOffset.UtcNow, networkId); + Guid registration = default; var now = DateTimeOffset.UtcNow; var key = BuildNetworkFileKey(networkId); @@ -71,9 +71,22 @@ public async Task JoinNetworkAsync( JsonContext.Default.NetworkState); await _store.WriteAsync(key, payload, cancellationToken).ConfigureAwait(false); + try + { + await _peerService.RecoverPeersAsync(networkId, _transport, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } +#pragma warning disable CA1031 // Peer recovery is best-effort during join. + catch +#pragma warning restore CA1031 + { + } + _joinedNetworks[networkId] = new NetworkInfo(networkId, now); _networkRegistrations[networkId] = registration; - await _peerService.RecoverPeersAsync(networkId, _transport, cancellationToken).ConfigureAwait(false); _events.Publish(EventCode.NetworkJoined, DateTimeOffset.UtcNow, networkId); } @@ -223,8 +236,21 @@ public async Task RecoverNetworksAsync( onFrameReceived, localEndpoint, cancellationToken).ConfigureAwait(false); + try + { + await _peerService.RecoverPeersAsync(network, _transport, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } +#pragma warning disable CA1031 // Peer recovery is best-effort during recovery. + catch +#pragma warning restore CA1031 + { + } + _networkRegistrations[network] = registration; - await _peerService.RecoverPeersAsync(network, _transport, cancellationToken).ConfigureAwait(false); } } From 5b4c3cf8daeae20b852308cbd5a6d4057e45cc7b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:20:39 +0100 Subject: [PATCH 125/296] fix(sockets): make ZtTcpListener DisposeAsync idempotent --- ZTSharp.Tests/ZtTcpListenerTests.cs | 8 ++++++++ ZTSharp/Sockets/ZtTcpListener.cs | 29 +++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/ZTSharp.Tests/ZtTcpListenerTests.cs b/ZTSharp.Tests/ZtTcpListenerTests.cs index 223d9d6..f7bfba1 100644 --- a/ZTSharp.Tests/ZtTcpListenerTests.cs +++ b/ZTSharp.Tests/ZtTcpListenerTests.cs @@ -6,6 +6,14 @@ namespace ZTSharp.Tests; public sealed class ZtTcpListenerTests { + [Fact] + public async Task DisposeAsync_IsIdempotent() + { + var listener = new ZtTcpListener(IPAddress.Loopback, 0); + await listener.DisposeAsync(); + await listener.DisposeAsync(); + } + [Fact] public async Task TcpListener_EchoesPayloadOffline() { diff --git a/ZTSharp/Sockets/ZtTcpListener.cs b/ZTSharp/Sockets/ZtTcpListener.cs index 8e41e54..118844c 100644 --- a/ZTSharp/Sockets/ZtTcpListener.cs +++ b/ZTSharp/Sockets/ZtTcpListener.cs @@ -12,6 +12,7 @@ public sealed class ZtTcpListener : IAsyncDisposable { private readonly SystemTcpListener _listener; private bool _started; + private int _disposeState; public ZtTcpListener(IPAddress address, int port) { @@ -30,6 +31,7 @@ public void Start(int backlog = 100) public async Task AcceptTcpClientAsync(CancellationToken cancellationToken = default) { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this); if (!_started) { _listener.Start(); @@ -42,10 +44,29 @@ public async Task AcceptTcpClientAsync(CancellationToken cancellati return new ZtTcpClient(client); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - await Task.Yield(); - _listener.Stop(); - _listener.Dispose(); + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + { + return ValueTask.CompletedTask; + } + + try + { + _listener.Stop(); + } + catch (ObjectDisposedException) + { + } + + try + { + _listener.Dispose(); + } + catch (ObjectDisposedException) + { + } + + return ValueTask.CompletedTask; } } From f541654e9d3870879140dccb564397fb17a80f88 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:21:55 +0100 Subject: [PATCH 126/296] test: avoid networkId collision across parallel in-memory tests --- ZTSharp.Tests/VirtualNetworkInterfaceTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs b/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs index 9cded1c..e3b9823 100644 --- a/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs +++ b/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs @@ -5,7 +5,7 @@ public sealed class VirtualNetworkInterfaceTests [Fact] public async Task InMemoryVirtualInterface_DeliversPacket() { - var networkId = 0xCAFE0002UL; + var networkId = 0xCAFE0004UL; await using var node1 = new Node(new NodeOptions { From 855d926d88edb3cdf7a70910b90a06c274759f82 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:36:35 +0100 Subject: [PATCH 127/296] test: use xUnit v2 skip exception --- ZTSharp.Tests/FileStateStoreSecurityTests.cs | 4 ++-- ZTSharp.Tests/OsUdpSocketFactoryTests.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/FileStateStoreSecurityTests.cs b/ZTSharp.Tests/FileStateStoreSecurityTests.cs index e0842c4..d3092e9 100644 --- a/ZTSharp.Tests/FileStateStoreSecurityTests.cs +++ b/ZTSharp.Tests/FileStateStoreSecurityTests.cs @@ -9,7 +9,7 @@ public async Task ReadAsync_Throws_WhenPathTraversesJunction() { if (!OperatingSystem.IsWindows()) { - throw Xunit.Sdk.SkipException.ForSkip("Junction traversal tests require Windows."); + throw new Xunit.Sdk.SkipException("Junction traversal tests require Windows."); } var root = TestTempPaths.CreateGuidSuffixed("zt-state-root-"); @@ -24,7 +24,7 @@ public async Task ReadAsync_Throws_WhenPathTraversesJunction() var junction = Path.Combine(root, "escape"); if (!TryCreateJunction(junction, outside)) { - throw Xunit.Sdk.SkipException.ForSkip("Failed to create junction (insufficient privileges or mklink unavailable)."); + throw new Xunit.Sdk.SkipException("Failed to create junction (insufficient privileges or mklink unavailable)."); } var store = new FileStateStore(root); diff --git a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs index 84a973d..ef729a9 100644 --- a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs +++ b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs @@ -11,7 +11,7 @@ public void WindowsSioUdpConnResetInput_IsDword() { if (!OperatingSystem.IsWindows()) { - throw Xunit.Sdk.SkipException.ForSkip("Windows-only IOControl buffer test."); + throw new Xunit.Sdk.SkipException("Windows-only IOControl buffer test."); } var buffer = OsUdpSocketFactory.CreateWindowsSioUdpConnResetInputBuffer(disableConnReset: true); @@ -51,7 +51,7 @@ public void CreateUdp6OnlyBound_SetsDualModeFalse() { if (!Socket.OSSupportsIPv6) { - throw Xunit.Sdk.SkipException.ForSkip("IPv6 not supported on this platform."); + throw new Xunit.Sdk.SkipException("IPv6 not supported on this platform."); } var method = typeof(OsUdpSocketFactory).GetMethod("CreateUdp6OnlyBound", BindingFlags.NonPublic | BindingFlags.Static); @@ -64,7 +64,7 @@ public void CreateUdp6OnlyBound_SetsDualModeFalse() } catch (TargetInvocationException ex) when (ex.InnerException is SocketException or PlatformNotSupportedException or NotSupportedException) { - throw Xunit.Sdk.SkipException.ForSkip($"IPv6 appears supported, but binding an IPv6 UDP socket failed: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"); + throw new Xunit.Sdk.SkipException($"IPv6 appears supported, but binding an IPv6 UDP socket failed: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"); } Assert.NotNull(udp); From 10ca4e2335852e28c3a8153b2f12e51a78f6c0dd Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:37:21 +0100 Subject: [PATCH 128/296] fix: run root reparse checks cross-platform --- ZTSharp/FileStateStore.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index a642295..1a30938 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -15,7 +15,11 @@ public sealed class FileStateStore : IStateStore public FileStateStore(string rootPath) { - ArgumentException.ThrowIfNullOrWhiteSpace(rootPath); + if (string.IsNullOrEmpty(rootPath)) + { + throw new ArgumentException("Root path must not be null or empty.", nameof(rootPath)); + } + _rootPath = Path.GetFullPath(rootPath); _rootPathTrimmed = Path.TrimEndingDirectorySeparator(_rootPath); _rootPathPrefix = _rootPathTrimmed + Path.DirectorySeparatorChar; @@ -373,13 +377,8 @@ private void ThrowIfRootPathIsReparsePoint() throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); } - if (!OperatingSystem.IsWindows()) - { - return; - } - var current = Path.GetDirectoryName(_rootPathTrimmed); - while (!string.IsNullOrWhiteSpace(current)) + while (!string.IsNullOrEmpty(current)) { if (IsReparsePoint(current)) { From 4af2da0a8cb2d94cba7d8a502d87f3e666174569 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:37:37 +0100 Subject: [PATCH 129/296] fix: make node event dispatch queue lossless --- ZTSharp/Internal/NodeEventStream.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ZTSharp/Internal/NodeEventStream.cs b/ZTSharp/Internal/NodeEventStream.cs index 63326ca..3d14262 100644 --- a/ZTSharp/Internal/NodeEventStream.cs +++ b/ZTSharp/Internal/NodeEventStream.cs @@ -5,8 +5,6 @@ namespace ZTSharp.Internal; internal sealed class NodeEventStream { - private const int DispatchQueueCapacity = 1024; - private readonly Action _onEventRaised; private readonly ILogger _logger; private readonly Channel _dispatchQueue; @@ -17,9 +15,8 @@ public NodeEventStream(Action onEventRaised, ILogger logger) { _onEventRaised = onEventRaised ?? throw new ArgumentNullException(nameof(onEventRaised)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _dispatchQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: DispatchQueueCapacity) + _dispatchQueue = Channel.CreateUnbounded(new UnboundedChannelOptions { - FullMode = BoundedChannelFullMode.DropOldest, SingleReader = true, SingleWriter = false }); From 22b89567d1a7fa3a47e8251aebe61d0dab6cba5b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:37:56 +0100 Subject: [PATCH 130/296] fix: always complete events on dispose --- ZTSharp/Internal/NodeLifecycleService.cs | 45 +++++++++++++----------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/ZTSharp/Internal/NodeLifecycleService.cs b/ZTSharp/Internal/NodeLifecycleService.cs index f832e57..8c0de5c 100644 --- a/ZTSharp/Internal/NodeLifecycleService.cs +++ b/ZTSharp/Internal/NodeLifecycleService.cs @@ -294,30 +294,35 @@ public async ValueTask DisposeAsync() { } - if (_ownsTransport && _transport is IAsyncDisposable asyncTransport) + try { - using var transportDisposeCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); - try - { - await asyncTransport - .DisposeAsync() - .AsTask() - .WaitAsync(transportDisposeCts.Token) - .ConfigureAwait(false); - } - catch (ObjectDisposedException) - { - } - catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) - { - } - catch (OperationCanceledException) when (transportDisposeCts.IsCancellationRequested) + if (_ownsTransport && _transport is IAsyncDisposable asyncTransport) { + using var transportDisposeCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + try + { + await asyncTransport + .DisposeAsync() + .AsTask() + .WaitAsync(transportDisposeCts.Token) + .ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) + { + } + catch (OperationCanceledException) when (transportDisposeCts.IsCancellationRequested) + { + } } } - - _events.Complete(); - _nodeCts.Dispose(); + finally + { + _events.Complete(); + _nodeCts.Dispose(); + } } private void EnsureNotDisposed() From 9a06ef277ad78f71651622e6244252b87198c8a2 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:38:21 +0100 Subject: [PATCH 131/296] fix: gate join rollback leave operations --- ZTSharp/Internal/NodeNetworkService.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ZTSharp/Internal/NodeNetworkService.cs b/ZTSharp/Internal/NodeNetworkService.cs index bda7754..7e8b870 100644 --- a/ZTSharp/Internal/NodeNetworkService.cs +++ b/ZTSharp/Internal/NodeNetworkService.cs @@ -96,7 +96,16 @@ public async Task JoinNetworkAsync( { try { - await _transport.LeaveNetworkAsync(networkId, registration, CancellationToken.None).ConfigureAwait(false); + await _leaveGate.WaitAsync(CancellationToken.None).ConfigureAwait(false); + try + { + await _transport.LeaveNetworkAsync(networkId, registration, CancellationToken.None).ConfigureAwait(false); + _networkRegistrations.TryRemove(new KeyValuePair(networkId, registration)); + } + finally + { + _leaveGate.Release(); + } } catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException) { From d242cba239e5c8cb37385e6d10a5447078cdbc7b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:38:32 +0100 Subject: [PATCH 132/296] fix: keep RX loop running after queue drop --- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 832ab8e..9183cce 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -180,7 +180,7 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca } } - return; + continue; } } } From bb12cfa031d77c24f6ff0f4090b28c277c4ac5a1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:38:46 +0100 Subject: [PATCH 133/296] fix: bound hole-punch cache size --- .../Internal/ZeroTierDirectEndpointManager.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 4aabd40..eb7758c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -281,6 +281,19 @@ private void CleanupHolePunchCacheIfNeeded(long nowMs) _holePunchLastSentMs.TryRemove(entry.Key, out _); } } + + if (_holePunchLastSentMs.Count <= HolePunchCacheMaxEntries) + { + return; + } + + var snapshot = _holePunchLastSentMs.ToArray(); + Array.Sort(snapshot, static (left, right) => left.Value.CompareTo(right.Value)); + + for (var i = 0; i < snapshot.Length && _holePunchLastSentMs.Count > HolePunchCacheMaxEntries; i++) + { + _holePunchLastSentMs.TryRemove(snapshot[i].Key, out _); + } } private static string FormatEndpointKey(IPEndPoint endpoint) From 9c90c9dc7ad22d74a125ea6e220ee0b7008b885e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:39:00 +0100 Subject: [PATCH 134/296] fix: compare-and-remove stale negotiation state --- .../ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs index 5165348..ecac0dc 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs @@ -43,7 +43,7 @@ public bool TryGetRemoteUtility(NodeId peerNodeId, int localSocketId, IPEndPoint { if (state.LastReceivedMs != 0 && unchecked(now - state.LastReceivedMs) > NegotiationStateTtlMs) { - _state.TryRemove(key, out _); + _state.TryRemove(new KeyValuePair(key, state)); remoteUtility = 0; return false; } @@ -107,7 +107,7 @@ private void CleanupIfNeeded(long now) var touched = Math.Max(state.LastReceivedMs, state.LastSentMs); if (touched != 0 && unchecked(now - touched) > NegotiationStateTtlMs) { - _state.TryRemove(pair.Key, out _); + _state.TryRemove(pair); } } } From 7265c82ef54f5c344c01a8d83240570806976051 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:39:10 +0100 Subject: [PATCH 135/296] fix: avoid orphaned ACK waiter on dispose race --- ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs index 7a99d1c..6dc9e76 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs @@ -246,6 +246,12 @@ private async Task SendTcpWithRetriesAsync( try { + if (_disposed) + { + ackTcs.TrySetException(new ObjectDisposedException(nameof(UserSpaceTcpSender))); + return; + } + for (var attempt = 0; attempt < retries; attempt++) { if (UserSpaceTcpSequenceNumbers.GreaterThanOrEqual(_sendUna, expectedAck)) From 3e1526b6030b2af4ad5a444797e62f129374afd1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:39:23 +0100 Subject: [PATCH 136/296] fix: synchronize runtime dispose with creation --- ZTSharp/ZeroTier/ZeroTierSocket.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ZTSharp/ZeroTier/ZeroTierSocket.cs b/ZTSharp/ZeroTier/ZeroTierSocket.cs index 4286b81..32d4f77 100644 --- a/ZTSharp/ZeroTier/ZeroTierSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierSocket.cs @@ -277,8 +277,17 @@ public async ValueTask DisposeAsync() await _shutdown.CancelAsync().ConfigureAwait(false); - _runtimeTask = null; - var runtime = Interlocked.Exchange(ref _runtime, null); + ZeroTierDataplaneRuntime? runtime; + await _runtimeLock.WaitAsync().ConfigureAwait(false); + try + { + _runtimeTask = null; + runtime = Interlocked.Exchange(ref _runtime, null); + } + finally + { + _runtimeLock.Release(); + } if (runtime is not null) { From 889a0cee7a3d245d5be2b480a84bb0e7ea149021 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:39:36 +0100 Subject: [PATCH 137/296] fix: filter UDP receives by bound local address --- ZTSharp/ZeroTier/ZeroTierUdpSocket.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs index 77ab20d..a07df54 100644 --- a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs @@ -158,6 +158,11 @@ public async ValueTask ReceiveFromAsync( } } + if (!dst.Equals(_localAddress)) + { + continue; + } + if (!UdpCodec.TryParse(ipPayload, out srcPort, out dstPort, out udpPayload)) { continue; From 4e4c44f4891e3ee982f3402fe7da5f7b8e9fdb58 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:43:34 +0100 Subject: [PATCH 138/296] test: use SkippableFact for runtime skips --- ZTSharp.Tests/FileStateStoreSecurityTests.cs | 12 +++--------- ZTSharp.Tests/OsUdpSocketFactoryTests.cs | 17 ++++++----------- ZTSharp.Tests/ZTSharp.Tests.csproj | 1 + 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/ZTSharp.Tests/FileStateStoreSecurityTests.cs b/ZTSharp.Tests/FileStateStoreSecurityTests.cs index d3092e9..58ba976 100644 --- a/ZTSharp.Tests/FileStateStoreSecurityTests.cs +++ b/ZTSharp.Tests/FileStateStoreSecurityTests.cs @@ -4,13 +4,10 @@ namespace ZTSharp.Tests; public sealed class FileStateStoreSecurityTests { - [Fact] + [SkippableFact] public async Task ReadAsync_Throws_WhenPathTraversesJunction() { - if (!OperatingSystem.IsWindows()) - { - throw new Xunit.Sdk.SkipException("Junction traversal tests require Windows."); - } + Skip.IfNot(OperatingSystem.IsWindows(), "Junction traversal tests require Windows."); var root = TestTempPaths.CreateGuidSuffixed("zt-state-root-"); Directory.CreateDirectory(root); @@ -22,10 +19,7 @@ public async Task ReadAsync_Throws_WhenPathTraversesJunction() await File.WriteAllBytesAsync(secretPath, new byte[] { 1, 2, 3 }); var junction = Path.Combine(root, "escape"); - if (!TryCreateJunction(junction, outside)) - { - throw new Xunit.Sdk.SkipException("Failed to create junction (insufficient privileges or mklink unavailable)."); - } + Skip.IfNot(TryCreateJunction(junction, outside), "Failed to create junction (insufficient privileges or mklink unavailable)."); var store = new FileStateStore(root); _ = await Assert.ThrowsAsync(async () => await store.ReadAsync("escape/secret.bin")); diff --git a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs index ef729a9..c5c564d 100644 --- a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs +++ b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs @@ -6,13 +6,10 @@ namespace ZTSharp.Tests; public sealed class OsUdpSocketFactoryTests { - [Fact] + [SkippableFact] public void WindowsSioUdpConnResetInput_IsDword() { - if (!OperatingSystem.IsWindows()) - { - throw new Xunit.Sdk.SkipException("Windows-only IOControl buffer test."); - } + Skip.IfNot(OperatingSystem.IsWindows(), "Windows-only IOControl buffer test."); var buffer = OsUdpSocketFactory.CreateWindowsSioUdpConnResetInputBuffer(disableConnReset: true); Assert.Equal(4, buffer.Length); @@ -46,13 +43,10 @@ public void CreateSocketCore_WhenDualModeFails_TriesIpv4BeforeIpv6Only() } } - [Fact] + [SkippableFact] public void CreateUdp6OnlyBound_SetsDualModeFalse() { - if (!Socket.OSSupportsIPv6) - { - throw new Xunit.Sdk.SkipException("IPv6 not supported on this platform."); - } + Skip.IfNot(Socket.OSSupportsIPv6, "IPv6 not supported on this platform."); var method = typeof(OsUdpSocketFactory).GetMethod("CreateUdp6OnlyBound", BindingFlags.NonPublic | BindingFlags.Static); Assert.NotNull(method); @@ -64,7 +58,8 @@ public void CreateUdp6OnlyBound_SetsDualModeFalse() } catch (TargetInvocationException ex) when (ex.InnerException is SocketException or PlatformNotSupportedException or NotSupportedException) { - throw new Xunit.Sdk.SkipException($"IPv6 appears supported, but binding an IPv6 UDP socket failed: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"); + Skip.If(true, $"IPv6 appears supported, but binding an IPv6 UDP socket failed: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"); + return; } Assert.NotNull(udp); diff --git a/ZTSharp.Tests/ZTSharp.Tests.csproj b/ZTSharp.Tests/ZTSharp.Tests.csproj index 5d69b5b..54a7ebb 100644 --- a/ZTSharp.Tests/ZTSharp.Tests.csproj +++ b/ZTSharp.Tests/ZTSharp.Tests.csproj @@ -11,6 +11,7 @@ + From 0261fac502a5ecc0d819423c7154ca6cf92b1d0c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:44:05 +0100 Subject: [PATCH 139/296] fix: allow wildcard UDP sockets to receive any dst --- ZTSharp/ZeroTier/ZeroTierUdpSocket.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs index a07df54..9d5bc7f 100644 --- a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs @@ -158,7 +158,9 @@ public async ValueTask ReceiveFromAsync( } } - if (!dst.Equals(_localAddress)) + if (!_localAddress.Equals(IPAddress.Any) && + !_localAddress.Equals(IPAddress.IPv6Any) && + !dst.Equals(_localAddress)) { continue; } From 95a73cbb89c14fe5d18f4197f50ef7001b10243b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:45:54 +0100 Subject: [PATCH 140/296] fix: prevent runtime resurrection without deadlocking dispose --- ZTSharp/ZeroTier/ZeroTierSocket.cs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/ZTSharp/ZeroTier/ZeroTierSocket.cs b/ZTSharp/ZeroTier/ZeroTierSocket.cs index 32d4f77..b67c374 100644 --- a/ZTSharp/ZeroTier/ZeroTierSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierSocket.cs @@ -277,17 +277,8 @@ public async ValueTask DisposeAsync() await _shutdown.CancelAsync().ConfigureAwait(false); - ZeroTierDataplaneRuntime? runtime; - await _runtimeLock.WaitAsync().ConfigureAwait(false); - try - { - _runtimeTask = null; - runtime = Interlocked.Exchange(ref _runtime, null); - } - finally - { - _runtimeLock.Release(); - } + _runtimeTask = null; + var runtime = Interlocked.Exchange(ref _runtime, null); if (runtime is not null) { @@ -429,6 +420,7 @@ private async Task CreateRuntimeAsync(byte[] inlineCom ZeroTierDataplaneRuntime? toDispose = null; ZeroTierDataplaneRuntime runtime; + var published = false; await _runtimeLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { @@ -445,6 +437,7 @@ private async Task CreateRuntimeAsync(byte[] inlineCom _runtime = created; runtime = created; createdNeedsDispose = false; + published = true; } } finally @@ -458,6 +451,16 @@ private async Task CreateRuntimeAsync(byte[] inlineCom createdNeedsDispose = false; } + if (published && Volatile.Read(ref _disposeState) != 0) + { + if (ReferenceEquals(Interlocked.CompareExchange(ref _runtime, null, created), created)) + { + await created.DisposeAsync().ConfigureAwait(false); + } + + throw new ObjectDisposedException(nameof(ZeroTierSocket)); + } + return runtime; } catch From 98e6c63727cb520c2a1969fc6c39285c5815824a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:47:07 +0100 Subject: [PATCH 141/296] test: treat UDP wildcard bind as IPAddress.Any --- ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs b/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs index 8575cef..866ccc0 100644 --- a/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs +++ b/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs @@ -17,7 +17,7 @@ public async Task ReceiveFromAsync_WildcardSemantics_AllowAnyLocalManagedIp() var ip = GetIpHandler(runtime); const ushort localPort = 12010; - await using var socket = new ZeroTierUdpSocket(runtime, localIpA, localPort); + await using var socket = new ZeroTierUdpSocket(runtime, IPAddress.Any, localPort); var buffer = new byte[32]; var receiveTask = socket.ReceiveFromAsync(buffer, TimeSpan.FromSeconds(1)).AsTask(); @@ -67,4 +67,3 @@ private static ZeroTierDataplaneRuntime CreateRuntime(IPAddress localManagedIpV4 localManagedIpsV6: Array.Empty(), inlineCom: new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); } - From ec7dea03654e17257ec98fade9921376d488585f Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 1 Mar 2026 12:33:10 +0100 Subject: [PATCH 142/296] fix: enhance multipath configuration validation in CLI commands --- ZTSharp.Tests/AtomicFileTests.cs | 23 +++++--- ZTSharp.Tests/TunnelAndHttpTests.cs | 27 ++++++++- .../ZeroTierSizeCapHardeningTests.cs | 2 +- ZTSharp/EventLoop.cs | 5 +- .../Transport/Internal/OsUdpPeerRegistry.cs | 58 +++++++++++++++++-- .../Internal/ManagedIpToNodeIdCache.cs | 2 - .../ZeroTierDataplaneRouteRegistry.cs | 16 +++++ .../ZeroTierExternalSurfaceAddressTracker.cs | 5 +- .../Internal/ZeroTierPeerEchoManager.cs | 7 ++- docs/ZEROTIER_STACK.md | 1 + samples/ZTSharp.Cli/Commands/CallCommand.cs | 8 ++- samples/ZTSharp.Cli/Commands/ExposeCommand.cs | 8 ++- samples/ZTSharp.Cli/Commands/JoinCommand.cs | 8 ++- samples/ZTSharp.Cli/Commands/ListenCommand.cs | 8 ++- .../ZTSharp.Cli/Commands/UdpListenCommand.cs | 8 ++- .../ZTSharp.Cli/Commands/UdpSendCommand.cs | 8 ++- 16 files changed, 163 insertions(+), 31 deletions(-) diff --git a/ZTSharp.Tests/AtomicFileTests.cs b/ZTSharp.Tests/AtomicFileTests.cs index 02421aa..8f1b179 100644 --- a/ZTSharp.Tests/AtomicFileTests.cs +++ b/ZTSharp.Tests/AtomicFileTests.cs @@ -10,16 +10,23 @@ public async Task WriteAllBytesAsync_Throws_WhenAtomicMoveNeverSucceeds() var root = TestTempPaths.CreateGuidSuffixed("zt-atomic-file-"); Directory.CreateDirectory(root); - var destination = Path.Combine(root, "dest"); - Directory.CreateDirectory(destination); - - var ex = await Assert.ThrowsAsync(async () => + try { - await AtomicFile.WriteAllBytesAsync(destination, new byte[] { 1, 2, 3 }, CancellationToken.None); - }); + var destination = Path.Combine(root, "dest"); + Directory.CreateDirectory(destination); + + var ex = await Assert.ThrowsAsync(async () => + { + await AtomicFile.WriteAllBytesAsync(destination, new byte[] { 1, 2, 3 }, CancellationToken.None); + }); - Assert.Contains("Atomic replace failed", ex.Message, StringComparison.OrdinalIgnoreCase); - Assert.True(Directory.Exists(destination)); + Assert.Contains("Atomic replace failed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.True(Directory.Exists(destination)); + } + finally + { + Directory.Delete(root, recursive: true); + } } } diff --git a/ZTSharp.Tests/TunnelAndHttpTests.cs b/ZTSharp.Tests/TunnelAndHttpTests.cs index f251c12..5a88ced 100644 --- a/ZTSharp.Tests/TunnelAndHttpTests.cs +++ b/ZTSharp.Tests/TunnelAndHttpTests.cs @@ -306,7 +306,7 @@ public async Task InMemoryOverlayHttpHandler_LocalPortAllocator_RetriesUnderConc using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); var releaseTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var connectionTasks = new List(capacity: 3); + var connectionTasks = new Task?[3]; static async Task HandleConnectionAsync( TcpClient tcp, @@ -349,7 +349,7 @@ static async Task HandleConnectionAsync( for (var i = 0; i < 3; i++) { var tcp = await httpListener.AcceptTcpClientAsync(cts.Token).ConfigureAwait(false); - connectionTasks.Add(HandleConnectionAsync(tcp, releaseTcs.Task, cts.Token)); + connectionTasks[i] = HandleConnectionAsync(tcp, releaseTcs.Task, cts.Token); } }, cts.Token); @@ -402,7 +402,28 @@ static async Task HandleConnectionAsync( try { - await Task.WhenAll(connectionTasks.Concat(new[] { acceptTask, forwarderTask })).WaitAsync(TimeSpan.FromSeconds(2)); + var deadlineMs = Environment.TickCount64 + 2000; + static TimeSpan Remaining(long deadlineMs) + { + var remainingMs = deadlineMs - Environment.TickCount64; + return remainingMs <= 0 ? TimeSpan.Zero : TimeSpan.FromMilliseconds(remainingMs); + } + + try + { + await acceptTask.WaitAsync(Remaining(deadlineMs)); + } + catch (OperationCanceledException) when (cts.IsCancellationRequested) + { + } + + var snapshot = connectionTasks + .Where(t => t is not null) + .Select(t => t!) + .ToArray(); + await Task + .WhenAll(snapshot.Concat(new[] { forwarderTask })) + .WaitAsync(Remaining(deadlineMs)); } catch (OperationCanceledException) when (cts.IsCancellationRequested) { diff --git a/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs b/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs index bd86d1f..3068708 100644 --- a/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs +++ b/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs @@ -52,7 +52,7 @@ public async Task PeerSecurity_DropsOversizedHello_WithoutCachingKeys() networkId: 1, inlineCom: Array.Empty()); - var peerSecurity = new ZeroTierDataplanePeerSecurity(udp, rootClient, localIdentity); + using var peerSecurity = new ZeroTierDataplanePeerSecurity(udp, rootClient, localIdentity); var packetBytes = new byte[ZeroTierProtocolLimits.MaxPacketBytes + 1]; await peerSecurity.HandleHelloAsync( diff --git a/ZTSharp/EventLoop.cs b/ZTSharp/EventLoop.cs index c33abfa..6a50c13 100644 --- a/ZTSharp/EventLoop.cs +++ b/ZTSharp/EventLoop.cs @@ -287,9 +287,10 @@ public void Dispose() private void ThrowIfDisposed() { - if (_fault is not null) + var fault = Volatile.Read(ref _fault); + if (fault is not null) { - throw new InvalidOperationException("Event loop is faulted.", _fault); + throw new InvalidOperationException("Event loop is faulted.", fault); } ObjectDisposedException.ThrowIf(_disposed, this); diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 4b00c7d..f874a02 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -7,7 +7,13 @@ namespace ZTSharp.Transport.Internal; internal sealed class OsUdpPeerRegistry { - internal readonly record struct PeerEntry(IPEndPoint Endpoint, long LastSeenTicks); + internal enum PeerEntrySource + { + Manual = 0, + Directory = 1 + } + + internal readonly record struct PeerEntry(IPEndPoint Endpoint, long LastSeenTicks, PeerEntrySource Source); private const int DirectoryMaxNetworks = 256; internal const int DirectoryMaxPeersPerNetwork = 1024; @@ -94,7 +100,7 @@ public void AddOrUpdatePeer(ulong networkId, ulong nodeId, IPEndPoint endpoint) var nowTicks = GetNowTicks(); var normalized = _normalizeEndpoint(endpoint); var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - peers[nodeId] = new PeerEntry(normalized, nowTicks); + peers[nodeId] = new PeerEntry(normalized, nowTicks, PeerEntrySource.Manual); SweepNetworkPeers(nowTicks); } @@ -113,7 +119,7 @@ public IEnumerable> RegisterLocalAndGetKnownPeer var nowTicks = GetNowTicks(); var normalizedAdvertisedEndpoint = _normalizeEndpoint(advertisedEndpoint); var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks); + discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks, PeerEntrySource.Directory); SweepDirectory(nowTicks); var localPeers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); @@ -124,9 +130,11 @@ public IEnumerable> RegisterLocalAndGetKnownPeer continue; } - localPeers[peer.Key] = new PeerEntry(peer.Value.Endpoint, nowTicks); + localPeers[peer.Key] = new PeerEntry(peer.Value.Endpoint, nowTicks, PeerEntrySource.Directory); } + PruneDirectoryPeers(localPeers, discoveredPeers, localNodeId); + return discoveredPeers .Where(p => p.Key != localNodeId) .Select(p => new KeyValuePair(p.Key, p.Value.Endpoint)) @@ -148,8 +156,13 @@ public void RefreshLocalRegistration(ulong networkId, ulong localNodeId, IPEndPo var nowTicks = GetNowTicks(); var normalizedAdvertisedEndpoint = _normalizeEndpoint(advertisedEndpoint); var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks); + discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks, PeerEntrySource.Directory); SweepDirectory(nowTicks); + + if (_networkPeers.TryGetValue(networkId, out var localPeers)) + { + PruneDirectoryPeers(localPeers, discoveredPeers, localNodeId); + } } public void Cleanup() @@ -298,6 +311,35 @@ private static void SweepDirectory(long nowTicks) } } + private static void PruneDirectoryPeers( + ConcurrentDictionary localPeers, + ConcurrentDictionary discoveredPeers, + ulong localNodeId) + { + foreach (var peer in localPeers) + { + if (peer.Value.Source != PeerEntrySource.Directory || + peer.Key == localNodeId || + discoveredPeers.ContainsKey(peer.Key)) + { + continue; + } + + localPeers.TryRemove(peer.Key, out _); + } + } + + private static void RemoveAllDirectoryPeers(ConcurrentDictionary peers) + { + foreach (var peer in peers) + { + if (peer.Value.Source == PeerEntrySource.Directory) + { + peers.TryRemove(peer.Key, out _); + } + } + } + private void ImportDirectoryPeers(ulong networkId, ConcurrentDictionary peers, long nowTicks) { if (!_enablePeerDiscovery) @@ -307,6 +349,7 @@ private void ImportDirectoryPeers(ulong networkId, ConcurrentDictionary, CancellationToken, Task> onSyn) { ArgumentNullException.ThrowIfNull(localAddress); + ArgumentNullException.ThrowIfNull(onSyn); return localAddress.AddressFamily switch { @@ -330,6 +331,21 @@ public bool TryGetTcpSynHandler( ushort localPort, out Func, CancellationToken, Task> handler) { + ArgumentNullException.ThrowIfNull(destinationIp); + if (addressFamily == AddressFamily.InterNetwork && + destinationIp.AddressFamily == AddressFamily.InterNetworkV6 && + destinationIp.IsIPv4MappedToIPv6) + { + destinationIp = destinationIp.MapToIPv4(); + } + + if (destinationIp.AddressFamily != addressFamily) + { + throw new ArgumentException( + "Destination IP address family must match the provided address family.", + nameof(destinationIp)); + } + if (addressFamily == AddressFamily.InterNetwork) { if (_tcpListenersV4.TryGetValue(localPort, out var registrations) && registrations.TryGet(destinationIp, out var existing)) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs b/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs index 4803846..36e0f13 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs @@ -27,7 +27,8 @@ public void Observe(NodeId reportingPeerNodeId, int localSocketId, IPEndPoint su var now = _nowUnixMs(); var key = new ZeroTierExternalSurfaceKey(localSocketId, reportingPeerNodeId); - _entries[key] = new Entry(surfaceAddress, now); + var stored = new IPEndPoint(surfaceAddress.Address, surfaceAddress.Port); + _entries[key] = new Entry(stored, now); CleanupIfNeeded(now); } @@ -41,7 +42,7 @@ public IPEndPoint[] GetSnapshot(int localSocketId) { if (key.LocalSocketId == localSocketId) { - list.Add(entry.SurfaceAddress); + list.Add(new IPEndPoint(entry.SurfaceAddress.Address, entry.SurfaceAddress.Port)); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs index 7c1e618..c04d7f4 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs @@ -170,7 +170,7 @@ public void HandleEchoOk( { ArgumentNullException.ThrowIfNull(remoteEndPoint); - if (!_pendingByPacketId.TryRemove(inRePacketId, out var pending)) + if (!_pendingByPacketId.TryGetValue(inRePacketId, out var pending)) { return; } @@ -195,6 +195,11 @@ public void HandleEchoOk( return; } + if (!_pendingByPacketId.TryRemove(inRePacketId, out _)) + { + return; + } + _lastRttMsByPath[pending.PathKey] = new RttEntry((int)rtt, LastUpdatedMs: now); } diff --git a/docs/ZEROTIER_STACK.md b/docs/ZEROTIER_STACK.md index 14da5aa..2891e3d 100644 --- a/docs/ZEROTIER_STACK.md +++ b/docs/ZEROTIER_STACK.md @@ -64,6 +64,7 @@ Managed stack CLI commands: `call`, `listen`, `udp-listen`, `udp-send`. To enable experimental multipath/bonding for the managed stack, use the CLI flags: `--multipath`, `--mp-bond`, `--mp-udp-sockets`, `--mp-udp-ports`, `--mp-warmup-root`. +Constraints: `--mp-udp-sockets` must be in `1..8`. If `--mp-udp-ports` is set, it must include exactly that many entries, each in `0..65535` (`0` = ephemeral). --- diff --git a/samples/ZTSharp.Cli/Commands/CallCommand.cs b/samples/ZTSharp.Cli/Commands/CallCommand.cs index cd85653..273ad25 100644 --- a/samples/ZTSharp.Cli/Commands/CallCommand.cs +++ b/samples/ZTSharp.Cli/Commands/CallCommand.cs @@ -147,11 +147,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/ExposeCommand.cs b/samples/ZTSharp.Cli/Commands/ExposeCommand.cs index 0cbc41e..9cc53bb 100644 --- a/samples/ZTSharp.Cli/Commands/ExposeCommand.cs +++ b/samples/ZTSharp.Cli/Commands/ExposeCommand.cs @@ -146,11 +146,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/JoinCommand.cs b/samples/ZTSharp.Cli/Commands/JoinCommand.cs index 385cd40..a3e4e55 100644 --- a/samples/ZTSharp.Cli/Commands/JoinCommand.cs +++ b/samples/ZTSharp.Cli/Commands/JoinCommand.cs @@ -123,11 +123,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/ListenCommand.cs b/samples/ZTSharp.Cli/Commands/ListenCommand.cs index 891ae58..9a0e052 100644 --- a/samples/ZTSharp.Cli/Commands/ListenCommand.cs +++ b/samples/ZTSharp.Cli/Commands/ListenCommand.cs @@ -99,11 +99,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs b/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs index 51c04ec..840d219 100644 --- a/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs +++ b/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs @@ -93,11 +93,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs b/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs index e3d159e..7127991 100644 --- a/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs +++ b/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs @@ -107,11 +107,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; From c11476a3f788820ca5a3d36a12b0aa47e3b11556 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 21:52:10 +0100 Subject: [PATCH 143/296] feat: add initial documentation for ZTSharp library and usage rules --- AGENTS.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..82984b8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,8 @@ + +# ZTSharp +ZTSharp is a fully managed c# library that implements the Zerotier protocol without relying on unmanaged code or external dependencies. + +# Rules +- Do not use or reuse official Zerotier services, connections or identities from the current host. The library **must** be self-contained and not rely on any existing Zerotier configuration or services on the host machine. +- The source code must be written in clean, modular, easy to understand and maintainable style. It should follow best practices in c#. +- The source code must be high-performant and efficient, minimizing resource usage and maximizing speed (Eg use ArrayPool, Span, etc... instead of creating new arrays or lists all of the time (especially hot paths)). \ No newline at end of file From 45161d64ebc157d52e9998590c555f8b287639da Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 21:52:17 +0100 Subject: [PATCH 144/296] feat: implement multipath options and direct path selection enhancements --- .../Http/ZeroTierHttpMessageHandler.cs | 5 +- .../Internal/ZeroTierDataplaneRuntime.cs | 430 +++++++++++++++++- .../ZeroTierDataplaneRuntimeFactory.cs | 5 +- .../Internal/ZeroTierDirectEndpointManager.cs | 25 + .../Internal/ZeroTierHelloPacketBuilder.cs | 31 +- .../Internal/ZeroTierSocketFactory.cs | 3 +- .../Protocol/ZeroTierPushDirectPathsCodec.cs | 69 +++ ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs | 6 + ZTSharp/ZeroTier/ZeroTierSocket.cs | 5 + samples/ZTSharp.Cli/CliHelp.cs | 1 + samples/ZTSharp.Cli/Commands/CallCommand.cs | 9 +- scripts/zt_join_accept_fetch.ps1 | 75 +-- 12 files changed, 618 insertions(+), 46 deletions(-) diff --git a/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs b/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs index 36ecbcc..4ad3a6d 100644 --- a/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs +++ b/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs @@ -8,7 +8,6 @@ namespace ZTSharp.ZeroTier.Http; public sealed class ZeroTierHttpMessageHandler : DelegatingHandler { private readonly ZeroTier.ZeroTierSocket _socket; - private static readonly TimeSpan DefaultPerAddressConnectTimeout = TimeSpan.FromSeconds(2); public ZeroTierHttpMessageHandler(ZeroTier.ZeroTierSocket socket) { @@ -35,7 +34,7 @@ private async ValueTask ConnectAsync(SocketsHttpConnectionContext contex endpoint, new[] { ip }, connectAsync: (ep, ct) => _socket.ConnectTcpAsync(ep, ct), - perAddressConnectTimeout: DefaultPerAddressConnectTimeout, + perAddressConnectTimeout: _socket.SuggestedHttpConnectTimeout, cancellationToken) .ConfigureAwait(false); } @@ -68,7 +67,7 @@ private async ValueTask ConnectAsync(SocketsHttpConnectionContext contex endpoint, addresses, connectAsync: (ep, ct) => _socket.ConnectTcpAsync(ep, ct), - perAddressConnectTimeout: DefaultPerAddressConnectTimeout, + perAddressConnectTimeout: _socket.SuggestedHttpConnectTimeout, cancellationToken) .ConfigureAwait(false); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c3f7a95..3c086d9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Sockets; @@ -23,9 +24,12 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierIdentity _localIdentity; private readonly ulong _networkId; private readonly byte[] _inlineCom; + private readonly ulong _planetId; + private readonly ulong _planetTimestamp; private readonly IPAddress[] _localManagedIpsV4; private readonly byte[][] _localManagedIpsV4Bytes; private readonly IPAddress[] _localManagedIpsV6; + private readonly IPEndPoint[] _localDirectPathAdvertisements; private readonly ZeroTierMac _localMac; private readonly ConcurrentDictionary _directEndpoints = new(); private readonly ConcurrentDictionary _directEndpointLastUsedMs = new(); @@ -37,6 +41,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierMultipathOptions _multipath; + private readonly ConcurrentDictionary _directBootstrapTasks = new(); private readonly Channel _peerQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) { @@ -82,7 +87,10 @@ public ZeroTierDataplaneRuntime( localManagedIpsV4, localManagedIpsV6, inlineCom, - multipath: new ZeroTierMultipathOptions()) + multipath: new ZeroTierMultipathOptions(), + planetId: 0, + planetTimestamp: 0, + localExternalSurfaceAddress: null) { } @@ -97,7 +105,10 @@ public ZeroTierDataplaneRuntime( IReadOnlyList localManagedIpsV4, IReadOnlyList localManagedIpsV6, byte[] inlineCom, - ZeroTierMultipathOptions multipath) + ZeroTierMultipathOptions multipath, + ulong planetId = 0, + ulong planetTimestamp = 0, + IPEndPoint? localExternalSurfaceAddress = null) { ArgumentNullException.ThrowIfNull(udp); ArgumentNullException.ThrowIfNull(rootEndpoint); @@ -140,12 +151,15 @@ public ZeroTierDataplaneRuntime( _localIdentity = localIdentity; _networkId = networkId; _inlineCom = inlineCom; + _planetId = planetId; + _planetTimestamp = planetTimestamp; _multipath = multipath; _localManagedIpsV4 = localManagedIpsV4.Count == 0 ? Array.Empty() : localManagedIpsV4.ToArray(); _localManagedIpsV4Bytes = _localManagedIpsV4.Length == 0 ? Array.Empty() : _localManagedIpsV4.Select(ip => ip.GetAddressBytes()).ToArray(); _localManagedIpsV6 = localManagedIpsV6.Count == 0 ? Array.Empty() : localManagedIpsV6.ToArray(); + _localDirectPathAdvertisements = BuildLocalDirectPathAdvertisements(udp, localExternalSurfaceAddress); _localMac = ZeroTierMac.FromAddress(localIdentity.NodeId, networkId); _routes = new ZeroTierDataplaneRouteRegistry(this); @@ -214,6 +228,37 @@ public ZeroTierDataplaneRuntime( : null; } + private static IPEndPoint[] BuildLocalDirectPathAdvertisements( + IZeroTierUdpTransport udp, + IPEndPoint? localExternalSurfaceAddress) + { + var endpoints = new List(); + if (localExternalSurfaceAddress is not null && localExternalSurfaceAddress.Port != 0) + { + endpoints.Add(UdpEndpointNormalization.Normalize(localExternalSurfaceAddress)); + } + + var localSockets = udp.LocalSockets; + for (var i = 0; i < localSockets.Count; i++) + { + var endpoint = UdpEndpointNormalization.Normalize(localSockets[i].LocalEndpoint); + if (endpoint.Port == 0 || + endpoint.Address.Equals(IPAddress.Any) || + endpoint.Address.Equals(IPAddress.IPv6Any) || + IPAddress.IsLoopback(endpoint.Address)) + { + continue; + } + + endpoints.Add(endpoint); + } + + return endpoints + .Distinct() + .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) + .ToArray(); + } + public NodeId NodeId => _localIdentity.NodeId; public IPEndPoint LocalUdp => _udp.LocalSockets[0].LocalEndpoint; @@ -324,10 +369,13 @@ private async ValueTask SendToPeerAsync( if (!_multipath.Enabled) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "multipath direct paths are disabled"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; } + await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + if (_multipath.BondPolicy == ZeroTierBondPolicy.Broadcast) { await SendToPeerBroadcastAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); @@ -336,6 +384,7 @@ private async ValueTask SendToPeerAsync( if (!TrySelectDirectPath(peerNodeId, flowId, out var direct)) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "no direct peer path is available"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; } @@ -344,7 +393,7 @@ private async ValueTask SendToPeerAsync( var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; var confirmed = _peerEcho.TryGetLastRttMs(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, out _); - if (_multipath.WarmupDuplicateToRoot && !confirmed) + if (_multipath.WarmupDuplicateToRoot && _multipath.AllowRootRelayFallback && !confirmed) { if (shouldRecord) { @@ -397,6 +446,7 @@ private async ValueTask SendToPeerAsync( _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); } + EnsureRelayAllowedForPayload(peerNodeId, reason: $"direct send to {direct.RemoteEndPoint} failed"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); } } @@ -405,11 +455,14 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo { cancellationToken.ThrowIfCancellationRequested(); + await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + var observed = _peerPaths.GetSnapshot(peerNodeId); var hinted = observed.Length == 0 ? GetOrCreateDirectEndpointManager(peerNodeId).Endpoints : Array.Empty(); if (observed.Length == 0 && hinted.Length == 0) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "no direct peer paths are available for broadcast"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; } @@ -476,7 +529,7 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo } } - if (_multipath.WarmupDuplicateToRoot && !anyConfirmed) + if (_multipath.WarmupDuplicateToRoot && _multipath.AllowRootRelayFallback && !anyConfirmed) { await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; @@ -484,10 +537,360 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo if (directSuccess == 0) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "all direct broadcast sends failed"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); } } + private void EnsureRelayAllowedForPayload(NodeId peerNodeId, string reason) + { + if (_multipath.AllowRootRelayFallback) + { + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Drop payload relay fallback: peer={peerNodeId} reason={reason}."); + } + + throw new InvalidOperationException( + $"No direct ZeroTier payload path is available for peer {peerNodeId}; relay fallback is disabled ({reason})."); + } + + private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_multipath.Enabled || HasSufficientDirectPath(peerNodeId)) + { + return; + } + + var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + if (HasSufficientDirectPath(peerNodeId)) + { + return; + } + + var bootstrapTask = _directBootstrapTasks.GetOrAdd( + peerNodeId, + id => BootstrapDirectPathCoreAsync(id, (byte[])sharedKey.Clone(), _cts.Token)); + + try + { + await bootstrapTask.WaitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + if (bootstrapTask.IsCompleted) + { + _directBootstrapTasks.TryRemove(new KeyValuePair(peerNodeId, bootstrapTask)); + } + } + + if (!_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId)) + { + EnsureRelayAllowedForPayload(peerNodeId, reason: "direct bootstrap did not confirm a peer path"); + } + + if (ZeroTierTrace.Enabled) + { + var observed = _peerPaths.GetSnapshot(peerNodeId); + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + ZeroTierTrace.WriteLine( + $"[zerotier] Direct bootstrap result: peer={peerNodeId} confirmed={HasConfirmedDirectPath(peerNodeId)} observed={observed.Length} hinted={ZeroTierDirectEndpointSelection.Format(hinted)} relayAllowed={_multipath.AllowRootRelayFallback}."); + } + } + + private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var bootstrapTimeout = _multipath.AllowRootRelayFallback + ? TimeSpan.FromSeconds(5) + : TimeSpan.FromSeconds(35); + var deadline = Environment.TickCount64 + (long)bootstrapTimeout.TotalMilliseconds; + var nextRootHelloAt = Environment.TickCount64; + + while (!HasSufficientDirectPath(peerNodeId)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var now = Environment.TickCount64; + if (unchecked(now - nextRootHelloAt) >= 0) + { + await SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + nextRootHelloAt = now + (long)TimeSpan.FromSeconds(1).TotalMilliseconds; + } + + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length > 0) + { + await SendEchoDirectAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); + await SendHelloDirectAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); + } + + if (unchecked(Environment.TickCount64 - deadline) >= 0) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken).ConfigureAwait(false); + } + } + + private bool HasDirectPathCandidate(NodeId peerNodeId) + => _peerPaths.GetSnapshot(peerNodeId).Length != 0 || + GetOrCreateDirectEndpointManager(peerNodeId).Endpoints.Length != 0; + + private bool HasSufficientDirectPath(NodeId peerNodeId) + => _multipath.AllowRootRelayFallback + ? HasDirectPathCandidate(peerNodeId) + : HasConfirmedDirectPath(peerNodeId); + + private bool HasConfirmedDirectPath(NodeId peerNodeId) + { + if (_peerPaths.GetSnapshot(peerNodeId).Length != 0) + { + return true; + } + + var endpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (endpoints.Length == 0) + { + return false; + } + + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + for (var i = 0; i < endpoints.Length; i++) + { + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId: 0, endpoints[i], out _)) + { + return true; + } + } + + return false; + } + + for (var i = 0; i < endpoints.Length; i++) + { + for (var s = 0; s < localSockets.Count; s++) + { + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSockets[s].Id, endpoints[i], out _)) + { + return true; + } + } + } + + return false; + } + + private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + await SendHelloPacketAsync(localSocketId: 0, peerNodeId, _rootEndpoint, _rootEndpoint, sharedKey, cancellationToken) + .ConfigureAwait(false); + return; + } + + for (var i = 0; i < localSockets.Count; i++) + { + await SendHelloPacketAsync( + localSockets[i].Id, + peerNodeId, + _rootEndpoint, + _rootEndpoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } + } + + private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + if (_localDirectPathAdvertisements.Length == 0) + { + return; + } + + try + { + var payload = ZeroTierPushDirectPathsCodec.BuildPayload(_localDirectPathAdvertisements); + if (payload.Length <= 2) + { + return; + } + + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var header = new ZeroTierPacketHeader( + PacketId: packetId, + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.PushDirectPaths); + + var packet = ZeroTierPacketCodec.Encode(header, payload); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] TX PUSH_DIRECT_PATHS via root for {peerNodeId}: {ZeroTierDirectEndpointSelection.Format(_localDirectPathAdvertisements)}."); + } + + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] PUSH_DIRECT_PATHS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private async Task SendHelloDirectAsync( + NodeId peerNodeId, + IPEndPoint[] endpoints, + byte[] sharedKey, + CancellationToken cancellationToken) + { + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + for (var i = 0; i < endpoints.Length; i++) + { + await SendHelloPacketAsync(localSocketId: 0, peerNodeId, endpoints[i], endpoints[i], sharedKey, cancellationToken) + .ConfigureAwait(false); + } + + return; + } + + for (var i = 0; i < endpoints.Length; i++) + { + for (var s = 0; s < localSockets.Count; s++) + { + await SendHelloPacketAsync( + localSockets[s].Id, + peerNodeId, + endpoints[i], + endpoints[i], + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } + } + } + + private async Task SendEchoDirectAsync( + NodeId peerNodeId, + IPEndPoint[] endpoints, + byte[] sharedKey, + CancellationToken cancellationToken) + { + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + for (var i = 0; i < endpoints.Length; i++) + { + await TrySendEchoDirectProbeAsync(peerNodeId, localSocketId: 0, endpoints[i], sharedKey, cancellationToken) + .ConfigureAwait(false); + } + + return; + } + + for (var i = 0; i < endpoints.Length; i++) + { + for (var s = 0; s < localSockets.Count; s++) + { + await TrySendEchoDirectProbeAsync(peerNodeId, localSockets[s].Id, endpoints[i], sharedKey, cancellationToken) + .ConfigureAwait(false); + } + } + } + + private async Task TrySendEchoDirectProbeAsync( + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + byte[] sharedKey, + CancellationToken cancellationToken) + { + try + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX ECHO bootstrap to {remoteEndPoint} (socket={localSocketId})."); + } + + await _peerEcho + .TrySendEchoProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] ECHO bootstrap send failed to {remoteEndPoint}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private async Task SendHelloPacketAsync( + int localSocketId, + NodeId peerNodeId, + IPEndPoint physicalDestination, + IPEndPoint sendTo, + byte[] sharedKey, + CancellationToken cancellationToken) + { + try + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, advertised={physicalDestination})."); + } + + var packet = ZeroTierHelloPacketBuilder.BuildPacket( + _localIdentity, + peerNodeId, + physicalDestination, + timestamp: (ulong)Environment.TickCount64, + _planetId, + _planetTimestamp, + sharedKey, + ZeroTierHelloClient.AdvertisedProtocolVersion, + ZeroTierHelloClient.AdvertisedMajorVersion, + ZeroTierHelloClient.AdvertisedMinorVersion, + ZeroTierHelloClient.AdvertisedRevision, + out _); + + await _udp.SendAsync(localSocketId, sendTo, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] HELLO bootstrap send failed to {sendTo}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSelectedPeerPath selected) { var observed = _peerPaths.GetSnapshot(peerNodeId); @@ -496,6 +899,19 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel return _bondEngine.TrySelectSinglePath(peerNodeId, observed, flowId, _multipath.BondPolicy, out selected); } + if (!_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId)) + { + if (ZeroTierTrace.Enabled) + { + var rejectedHints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + ZeroTierTrace.WriteLine( + $"[zerotier] Reject hinted direct path selection: peer={peerNodeId} hinted={ZeroTierDirectEndpointSelection.Format(rejectedHints)}."); + } + + selected = default; + return false; + } + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; if (hinted.Length > 0) { @@ -505,6 +921,12 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel ? 0 : (int)((flowId / (uint)hinted.Length) % (uint)localSockets.Count); var localSocketId = localSockets.Count == 0 ? 0 : localSockets[socketIndex].Id; + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Select hinted direct path: peer={peerNodeId} endpoint={hinted[endpointIndex]} socket={localSocketId} relayAllowed={_multipath.AllowRootRelayFallback} confirmed={HasConfirmedDirectPath(peerNodeId)}."); + } + selected = new ZeroTierSelectedPeerPath(localSocketId, hinted[endpointIndex]); return true; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs index a521fa8..cbe7d2b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs @@ -55,7 +55,10 @@ internal static class ZeroTierDataplaneRuntimeFactory localManagedIpsV4: localManagedIpsV4, localManagedIpsV6: localManagedIpsV6, inlineCom: inlineCom, - multipath: multipath); + multipath: multipath, + planetId: planet.Id, + planetTimestamp: planet.Timestamp, + localExternalSurfaceAddress: helloOk.ExternalSurfaceAddress); await TrySubscribeForAddressResolutionAsync( udp, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index eb7758c..940a47e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -149,6 +149,31 @@ public ValueTask HandlePushDirectPathsFromRemoteAsync(ReadOnlyMemory paylo return ValueTask.CompletedTask; } + public void SeedEndpoints(IEnumerable endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + IPEndPoint[] normalized; + lock (_lock) + { + normalized = ZeroTierDirectEndpointSelection.Normalize( + _directEndpoints.Concat(endpoints), + _relayEndpoint, + maxEndpoints: MaxEndpoints); + _directEndpoints = normalized; + } + + if (ZeroTierTrace.Enabled && normalized.Length > 0) + { + ZeroTierTrace.WriteLine($"[zerotier] Seed direct endpoints for {_remoteNodeId}: {ZeroTierDirectEndpointSelection.Format(normalized)}."); + } + + foreach (var endpoint in normalized) + { + TrySendHolePunch(endpoint); + } + } + private bool RateGatePushDirectPaths(long nowMs) { lock (_lock) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs index 0cf8d74..ec7e11b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs @@ -19,6 +19,33 @@ public static byte[] BuildPacket( byte advertisedMinorVersion, ushort advertisedRevision, out ulong packetId) + => BuildPacket( + localIdentity, + destination, + physicalDestination, + timestamp, + planet.Id, + planet.Timestamp, + sharedKey, + advertisedProtocolVersion, + advertisedMajorVersion, + advertisedMinorVersion, + advertisedRevision, + out packetId); + + public static byte[] BuildPacket( + ZeroTierIdentity localIdentity, + NodeId destination, + IPEndPoint physicalDestination, + ulong timestamp, + ulong planetId, + ulong planetTimestamp, + ReadOnlySpan sharedKey, + byte advertisedProtocolVersion, + byte advertisedMajorVersion, + byte advertisedMinorVersion, + ushort advertisedRevision, + out ulong packetId) { var iv = new byte[8]; RandomNumberGenerator.Fill(iv); @@ -55,9 +82,9 @@ public static byte[] BuildPacket( p += ZeroTierIdentityCodec.Serialize(localIdentity, payload.AsSpan(p), includePrivate: false); p += ZeroTierInetAddressCodec.Serialize(physicalDestination, payload.AsSpan(p)); - BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planet.Id); + BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planetId); p += 8; - BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planet.Timestamp); + BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planetTimestamp); p += 8; // Encrypted portion: moon count (0) (no moons advertised). diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs index 3b0770e..75da8ba 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs @@ -72,7 +72,8 @@ public static Task CreateAsync(ZeroTierSocketOptions options, Ca BondPolicy = options.Multipath.BondPolicy, UdpSocketCount = options.Multipath.UdpSocketCount, LocalUdpPorts = options.Multipath.LocalUdpPorts is { } localPorts ? localPorts.ToArray() : null, - WarmupDuplicateToRoot = options.Multipath.WarmupDuplicateToRoot + WarmupDuplicateToRoot = options.Multipath.WarmupDuplicateToRoot, + AllowRootRelayFallback = options.Multipath.AllowRootRelayFallback }; var normalizedOptions = new ZeroTierSocketOptions diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs index 608f173..856f424 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs @@ -9,6 +9,75 @@ internal readonly record struct ZeroTierPushedDirectPath( internal static class ZeroTierPushDirectPathsCodec { + public static byte[] BuildPayload(IEnumerable endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + var normalized = endpoints + .Where(static ep => ep is not null && ep.Port != 0) + .Select(static ep => + { + var address = ep.Address.IsIPv4MappedToIPv6 ? ep.Address.MapToIPv4() : ep.Address; + return new IPEndPoint(address, ep.Port); + }) + .Distinct() + .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) + .ToArray(); + + var payloadLength = 2; + for (var i = 0; i < normalized.Length; i++) + { + payloadLength += normalized[i].AddressFamily switch + { + System.Net.Sockets.AddressFamily.InterNetwork => 1 + 2 + 1 + 1 + 6, + System.Net.Sockets.AddressFamily.InterNetworkV6 => 1 + 2 + 1 + 1 + 18, + _ => 0 + }; + } + + var payload = new byte[payloadLength]; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(0, 2), (ushort)normalized.Length); + + var ptr = 2; + for (var i = 0; i < normalized.Length; i++) + { + var endpoint = normalized[i]; + payload[ptr++] = 0; // flags + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); // extension length + ptr += 2; + + if (endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + payload[ptr++] = 4; + payload[ptr++] = 6; + endpoint.Address.GetAddressBytes().CopyTo(payload, ptr); + ptr += 4; + } + else if (endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + payload[ptr++] = 6; + payload[ptr++] = 18; + endpoint.Address.GetAddressBytes().CopyTo(payload, ptr); + ptr += 16; + } + else + { + continue; + } + + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), (ushort)endpoint.Port); + ptr += 2; + } + + if (ptr != payload.Length) + { + Array.Resize(ref payload, ptr); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(0, 2), (ushort)Math.Min(normalized.Length, ZeroTierProtocolLimits.MaxPushedDirectPaths)); + } + + return payload; + } + public static bool TryParse(ReadOnlySpan payload, out ZeroTierPushedDirectPath[] paths) { if (payload.Length < 2) diff --git a/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs b/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs index 89c2efb..cab71b2 100644 --- a/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs +++ b/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs @@ -28,6 +28,12 @@ public sealed class ZeroTierMultipathOptions /// When true, unconfirmed direct paths are "warmed up" by duplicating outbound packets to the root relay as a fallback. /// public bool WarmupDuplicateToRoot { get; init; } = true; + + /// + /// When false, payload traffic must use a negotiated direct peer path and will not be relayed via the root. + /// Root/controller traffic needed for join, address resolution, and path negotiation still uses the root. + /// + public bool AllowRootRelayFallback { get; init; } = true; } public enum ZeroTierBondPolicy diff --git a/ZTSharp/ZeroTier/ZeroTierSocket.cs b/ZTSharp/ZeroTier/ZeroTierSocket.cs index b67c374..49fe30a 100644 --- a/ZTSharp/ZeroTier/ZeroTierSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierSocket.cs @@ -44,6 +44,11 @@ internal ZeroTierSocket(ZeroTierSocketOptions options, string statePath, ZeroTie public IReadOnlyList ManagedIps { get; private set; } = Array.Empty(); + internal TimeSpan SuggestedHttpConnectTimeout + => _options.Multipath.Enabled && !_options.Multipath.AllowRootRelayFallback + ? TimeSpan.FromSeconds(45) + : TimeSpan.FromSeconds(10); + public static Task CreateAsync( ZeroTierSocketOptions options, CancellationToken cancellationToken = default) diff --git a/samples/ZTSharp.Cli/CliHelp.cs b/samples/ZTSharp.Cli/CliHelp.cs index 04ee08e..e8ddb52 100644 --- a/samples/ZTSharp.Cli/CliHelp.cs +++ b/samples/ZTSharp.Cli/CliHelp.cs @@ -29,6 +29,7 @@ libzt call --network --url [options] --map-ip Map IP to node id for overlay HTTP (repeatable) --multipath Enable experimental multipath direct-path selection and bonding (managed stack only) + --direct-only For 'call': require negotiated direct peer paths for payloads and disallow root relay fallback --mp-bond Bond policy: off|active-backup|broadcast|balance-rr|balance-xor|balance-aware (default: off) --mp-udp-sockets Number of UDP sockets to open when multipath is enabled (default: 1) --mp-udp-ports Explicit local UDP ports for multipath sockets (0 = ephemeral; length must match --mp-udp-sockets) diff --git a/samples/ZTSharp.Cli/Commands/CallCommand.cs b/samples/ZTSharp.Cli/Commands/CallCommand.cs index 273ad25..6fa76aa 100644 --- a/samples/ZTSharp.Cli/Commands/CallCommand.cs +++ b/samples/ZTSharp.Cli/Commands/CallCommand.cs @@ -26,6 +26,7 @@ public static async Task RunAsync(string[] commandArgs) int? mpUdpSockets = null; IReadOnlyList? mpUdpPorts = null; var mpWarmupRoot = true; + var directOnly = false; for (var i = 0; i < commandArgs.Length; i++) { @@ -87,6 +88,11 @@ public static async Task RunAsync(string[] commandArgs) case "--multipath": mpEnabled = true; break; + case "--direct-only": + directOnly = true; + mpEnabled = true; + mpWarmupRoot = false; + break; case "--mp-bond": { var value = CliParsing.ReadOptionValue(commandArgs, ref i, "--mp-bond"); @@ -159,7 +165,8 @@ public static async Task RunAsync(string[] commandArgs) BondPolicy = mpBondPolicy, UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, - WarmupDuplicateToRoot = mpWarmupRoot + WarmupDuplicateToRoot = mpWarmupRoot, + AllowRootRelayFallback = !directOnly }; await RunCallZeroTierAsync(statePath, networkId, multipath, url, cancellation.Token).ConfigureAwait(false); diff --git a/scripts/zt_join_accept_fetch.ps1 b/scripts/zt_join_accept_fetch.ps1 index fa369a6..4e2efaa 100644 --- a/scripts/zt_join_accept_fetch.ps1 +++ b/scripts/zt_join_accept_fetch.ps1 @@ -3,9 +3,7 @@ param( [string]$Configuration = "Release", [string]$Org = "cmlpb7m960006np01jgwg4dr7", [string]$Network = "9ad07d01093a69e3", - [string]$Url = "http://10.121.15.99:5380/", - [int]$UdpPort = 0, - [string]$HttpMode = "os", + [string]$Url = "https://jaeger.pdcs.kamsker.at/", [string]$StateRootPath = "", [switch]$SkipBuild ) @@ -16,26 +14,30 @@ function Write-Step([string]$Message) { Write-Host ("== " + $Message) } -function Get-ZtNodeIdFromState([string]$StateRoot) { - $secretPath = Join-Path $StateRoot "identity.secret" - $bytes = [System.IO.File]::ReadAllBytes($secretPath) - if ($bytes.Length -lt 32) { - throw "Invalid identity secret file: $secretPath" +function Resolve-ZtNetCommand() { + foreach ($candidate in @("ztnet-cli", "ztnet")) { + $command = Get-Command $candidate -ErrorAction SilentlyContinue + if ($null -ne $command) { + return $command.Source + } } - $secret = New-Object byte[] 32 - [Array]::Copy($bytes, 0, $secret, 0, 32) - $sha = [System.Security.Cryptography.SHA256]::Create() - try { - $hash = $sha.ComputeHash($secret) + throw "Neither 'ztnet-cli' nor 'ztnet' was found on PATH." +} + +function Get-ZtNodeIdFromState([string]$StateRoot) { + $identityPath = Join-Path $StateRoot "zerotier\\identity.bin" + $bytes = [System.IO.File]::ReadAllBytes($identityPath) + if ($bytes.Length -lt 13) { + throw "Invalid identity file: $identityPath" } - finally { - $sha.Dispose() + + $magic = [System.Text.Encoding]::ASCII.GetString($bytes, 0, 4) + if ($magic -ne "ZTID") { + throw "Unexpected identity file format: $identityPath" } - $value = [System.BitConverter]::ToUInt64($hash, 0) - $mask = [UInt64]0xFFFFFFFFFF - $value = $value -band $mask + $value = [System.BitConverter]::ToUInt64($bytes, 5) return ("0x{0:x10}" -f $value) } @@ -52,24 +54,28 @@ if ([string]::IsNullOrWhiteSpace($StateRootPath)) { New-Item -ItemType Directory -Path $StateRootPath -Force | Out-Null +$ztnetCommand = Resolve-ZtNetCommand + Write-Step "Checking ztnet auth..." -ztnet auth test | Out-Host +& $ztnetCommand auth test | Out-Host if (-not $SkipBuild) { Write-Step "Building ($Configuration)..." dotnet build -c $Configuration | Out-Host } -Write-Step "Joining network $Network (state: $StateRootPath)..." -& dotnet run -c $Configuration --no-build ` +Write-Step "Bootstrapping managed ZeroTier identity for $Network (state: $StateRootPath)..." +$bootstrapOutput = & dotnet run -c $Configuration --no-build ` --project samples/ZTSharp.Cli/ZTSharp.Cli.csproj -- ` join ` --once ` --network $Network ` - --stack overlay ` - --transport osudp ` - --udp-port $UdpPort ` - --state $StateRootPath | Out-Host + --stack managed ` + --state $StateRootPath 2>&1 +$bootstrapOutput | Out-Host +if ($LASTEXITCODE -ne 0) { + Write-Step "Initial join did not complete before authorization; continuing with generated node identity." +} $nodeId = Get-ZtNodeIdFromState -StateRoot $StateRootPath Write-Step "NodeId: $nodeId" @@ -79,17 +85,18 @@ if ($nodeIdForZtnet.StartsWith("0x")) { } Write-Step "Adding + authorizing member in ZTNet (org: $Org)..." -ztnet --org $Org --yes network member add $Network $nodeIdForZtnet | Out-Host -ztnet --org $Org --yes network member authorize $Network $nodeIdForZtnet | Out-Host +& $ztnetCommand --org $Org --yes network member add $Network $nodeIdForZtnet | Out-Host +& $ztnetCommand --org $Org --yes network member authorize $Network $nodeIdForZtnet | Out-Host -Write-Step "Fetching $Url (call --http $HttpMode)..." -& dotnet run -c $Configuration --no-build ` +Write-Step "Fetching $Url over managed ZeroTier HTTP..." +$callOutput = & dotnet run -c $Configuration --no-build ` --project samples/ZTSharp.Cli/ZTSharp.Cli.csproj -- ` call ` - --http $HttpMode ` --network $Network ` - --stack overlay ` - --transport osudp ` - --udp-port 0 ` + --stack managed ` --state $StateRootPath ` - --url $Url | Out-Host + --url $Url 2>&1 +$callOutput | Out-Host +if ($LASTEXITCODE -ne 0) { + throw "Managed ZeroTier HTTP call failed (exit code $LASTEXITCODE)." +} From 628ccd23adad12235fc03a2ff669753d16ccea05 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 22:49:23 +0100 Subject: [PATCH 145/296] Preserve direct endpoint socket affinity --- ...TierDirectEndpointManagerPushFlagsTests.cs | 3 +- ...irectEndpointManagerSocketAffinityTests.cs | 51 +++ .../ZeroTierDataplanePeerDatagramProcessor.cs | 67 +++- .../ZeroTierDataplanePeerPacketHandler.cs | 13 +- .../Internal/ZeroTierDataplaneRuntime.cs | 337 +++++++++++++++--- .../Internal/ZeroTierDataplaneRxLoops.cs | 6 +- .../Internal/ZeroTierDirectEndpointManager.cs | 124 ++++++- .../Internal/ZeroTierIpv4LinkReceiver.cs | 4 +- 8 files changed, 532 insertions(+), 73 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 65e6844..1d3d7f7 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -19,13 +19,14 @@ public async Task PushDirectPaths_ForgetFlag_RemovesEndpoint() var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); var endpoint = TestUdpEndpoints.ToLoopback(receiver.LocalEndpoint); - await manager.HandlePushDirectPathsFromRemoteAsync(BuildPushDirectPathsPayload(endpoint, flags: 0), CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync(BuildPushDirectPathsPayload(endpoint, flags: 0), receivedLocalSocketId: 0, CancellationToken.None); _ = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); Assert.Contains(manager.Endpoints, ep => ep.Equals(endpoint)); await manager.HandlePushDirectPathsFromRemoteAsync( BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagForgetPath), + receivedLocalSocketId: 0, CancellationToken.None); Assert.DoesNotContain(manager.Endpoints, ep => ep.Equals(endpoint)); diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs new file mode 100644 index 0000000..4c77644 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -0,0 +1,51 @@ +using System.Buffers.Binary; +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointManagerSocketAffinityTests +{ + [Fact] + public async Task Rendezvous_UsesReceivingLocalSocket_ForInitialHolePunch() + { + await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 0); + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + await using var udp = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); + await using var receiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = TestUdpEndpoints.ToLoopback(receiver.LocalEndpoint); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + + var datagram = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(4, datagram.Payload.Length); + Assert.Equal(socket1.LocalEndpoint.Port, datagram.RemoteEndPoint.Port); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + + await Assert.ThrowsAsync(async () => await receiver.ReceiveAsync(TimeSpan.FromMilliseconds(250))); + } + + private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) + { + var addressBytes = endpoint.Address.GetAddressBytes(); + var payload = new byte[1 + 5 + 2 + 1 + addressBytes.Length]; + + payload[0] = 0; + ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, 5), with.Value); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(1 + 5, 2), (ushort)endpoint.Port); + payload[1 + 5 + 2] = (byte)addressBytes.Length; + addressBytes.CopyTo(payload.AsSpan(1 + 5 + 2 + 1)); + + return payload; + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index ddd7db4..0d70784 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -1,4 +1,5 @@ using System.Buffers.Binary; +using System.Net; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -7,6 +8,7 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDataplanePeerDatagramProcessor : IZeroTierDataplanePeerDatagramProcessor { + private int _traceDecryptedRemaining = 100; private readonly NodeId _localNodeId; private readonly ZeroTierDataplanePeerSecurity _peerSecurity; private readonly ZeroTierDataplanePeerPacketHandler _peerPackets; @@ -16,6 +18,7 @@ internal sealed class ZeroTierDataplanePeerDatagramProcessor private readonly ZeroTierPeerQosManager _peerQos; private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly bool _multipathEnabled; + private readonly Action? _handleHelloOk; public ZeroTierDataplanePeerDatagramProcessor( NodeId localNodeId, @@ -26,7 +29,8 @@ public ZeroTierDataplanePeerDatagramProcessor( ZeroTierExternalSurfaceAddressTracker surfaceAddresses, ZeroTierPeerQosManager peerQos, ZeroTierPeerPathNegotiationManager peerNegotiation, - bool multipathEnabled) + bool multipathEnabled, + Action? handleHelloOk = null) { ArgumentNullException.ThrowIfNull(peerSecurity); ArgumentNullException.ThrowIfNull(peerPackets); @@ -45,6 +49,7 @@ public ZeroTierDataplanePeerDatagramProcessor( _peerQos = peerQos; _peerNegotiation = peerNegotiation; _multipathEnabled = multipathEnabled; + _handleHelloOk = handleHelloOk; } public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken cancellationToken) @@ -66,9 +71,33 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c if (decoded.Header.CipherSuite == 0 && decoded.Header.Verb == ZeroTierVerb.Hello) { - await _peerSecurity - .HandleHelloAsync(peerNodeId, decoded.Header.PacketId, packetBytes, datagram.RemoteEndPoint, cancellationToken) + var inboundHello = await _peerSecurity + .HandleHelloAsync( + peerNodeId, + datagram.LocalSocketId, + decoded.Header.PacketId, + packetBytes, + datagram.RemoteEndPoint, + cancellationToken) .ConfigureAwait(false); + + if (inboundHello is { } hello && decoded.Header.HopCount == 0) + { + _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + + if (hello.ReportedLocalSurfaceAddress is { } reportedSurface) + { + _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, reportedSurface); + } + + if (ZeroTierTrace.Enabled) + { + var surfaceText = hello.ReportedLocalSurfaceAddress?.ToString() ?? ""; + ZeroTierTrace.WriteLine( + $"[zerotier] RX HELLO direct: peer={peerNodeId} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint} reportedLocalSurface={surfaceText}."); + } + } + return; } @@ -93,6 +122,14 @@ await _peerSecurity packetBytes = uncompressed; } + if (ZeroTierTrace.Enabled && _traceDecryptedRemaining > 0) + { + _traceDecryptedRemaining--; + var decryptedVerb = (ZeroTierVerb)(packetBytes[ZeroTierPacketHeader.IndexVerb] & 0x1F); + ZeroTierTrace.WriteLine( + $"[zerotier] RX peer decrypted: src={peerNodeId} hop={decoded.Header.HopCount} verb={decryptedVerb} socket={datagram.LocalSocketId} via {datagram.RemoteEndPoint}."); + } + if (_multipathEnabled) { if (decoded.Header.HopCount == 0) @@ -163,11 +200,23 @@ await _peerEcho if (ZeroTierHelloOkParser.TryParseDecryptedOkHello(packetBytes, out var ok)) { _peerSecurity.ObservePeerProtocolVersion(peerNodeId, ok.RemoteProtocolVersion); - _peerEcho.ObserveHelloOkRtt(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, ok.TimestampEcho); - - if (ok.ExternalSurfaceAddress is { } surface) + if (_handleHelloOk is not null) + { + _handleHelloOk( + peerNodeId, + datagram.LocalSocketId, + datagram.RemoteEndPoint, + decoded.Header.HopCount, + ok); + } + else { - _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, surface); + _peerEcho.ObserveHelloOkRtt(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, ok.TimestampEcho); + + if (ok.ExternalSurfaceAddress is { } surface) + { + _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, surface); + } } } @@ -176,6 +225,8 @@ await _peerEcho } } - await _peerPackets.HandleAsync(peerNodeId, packetBytes, cancellationToken).ConfigureAwait(false); + await _peerPackets + .HandleAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, packetBytes, cancellationToken) + .ConfigureAwait(false); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs index 5ea91f2..ca814d0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs @@ -9,13 +9,13 @@ internal sealed class ZeroTierDataplanePeerPacketHandler private readonly ulong _networkId; private readonly ZeroTierMac _localMac; private readonly ZeroTierDataplaneIpHandler _ip; - private readonly Func, CancellationToken, ValueTask>? _handleControlAsync; + private readonly Func, CancellationToken, ValueTask>? _handleControlAsync; public ZeroTierDataplanePeerPacketHandler( ulong networkId, ZeroTierMac localMac, ZeroTierDataplaneIpHandler ip, - Func, CancellationToken, ValueTask>? handleControlAsync = null) + Func, CancellationToken, ValueTask>? handleControlAsync = null) { ArgumentNullException.ThrowIfNull(ip); _networkId = networkId; @@ -24,7 +24,12 @@ public ZeroTierDataplanePeerPacketHandler( _handleControlAsync = handleControlAsync; } - public async ValueTask HandleAsync(NodeId peerNodeId, byte[] packetBytes, CancellationToken cancellationToken) + public async ValueTask HandleAsync( + NodeId peerNodeId, + int receivedLocalSocketId, + IPEndPoint receivedVia, + byte[] packetBytes, + CancellationToken cancellationToken) { if (packetBytes.Length <= ZeroTierPacketHeader.IndexVerb) { @@ -40,7 +45,7 @@ public async ValueTask HandleAsync(NodeId peerNodeId, byte[] packetBytes, Cancel { if (_handleControlAsync is not null) { - await _handleControlAsync(peerNodeId, verb, payload, cancellationToken).ConfigureAwait(false); + await _handleControlAsync(peerNodeId, verb, receivedLocalSocketId, receivedVia, payload, cancellationToken).ConfigureAwait(false); } return; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 3c086d9..94a7781 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -42,6 +42,8 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _directBootstrapTasks = new(); + private readonly ConcurrentDictionary _pendingHelloProbes = new(); + private long _lastPendingHelloCleanupMs; private readonly Channel _peerQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) { @@ -108,7 +110,8 @@ public ZeroTierDataplaneRuntime( ZeroTierMultipathOptions multipath, ulong planetId = 0, ulong planetTimestamp = 0, - IPEndPoint? localExternalSurfaceAddress = null) + IPEndPoint? localExternalSurfaceAddress = null, + IReadOnlyList? initialExternalSurfaceObservations = null) { ArgumentNullException.ThrowIfNull(udp); ArgumentNullException.ThrowIfNull(rootEndpoint); @@ -176,6 +179,7 @@ public ZeroTierDataplaneRuntime( _peerPaths = new ZeroTierPeerPhysicalPathTracker(ttl: TimeSpan.FromSeconds(30)); _peerEcho = new ZeroTierPeerEchoManager(udp, localIdentity.NodeId, _peerSecurity.GetPeerProtocolVersionOrDefault); _surfaceAddresses = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromMinutes(30)); + SeedInitialExternalSurfaceObservations(initialExternalSurfaceObservations); _peerQos = new ZeroTierPeerQosManager(); _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); @@ -201,7 +205,8 @@ public ZeroTierDataplaneRuntime( _surfaceAddresses, _peerQos, _peerNegotiation, - multipath.Enabled); + multipath.Enabled, + HandlePeerHelloOk); _rxLoops = new ZeroTierDataplaneRxLoops( _udp, _rootNodeId, @@ -259,6 +264,57 @@ private static IPEndPoint[] BuildLocalDirectPathAdvertisements( .ToArray(); } + private IPEndPoint[] GetLocalDirectPathAdvertisements() + { + var endpoints = new List(_localDirectPathAdvertisements); + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + endpoints.AddRange(_surfaceAddresses.GetSnapshot(localSocketId: 0)); + } + else + { + for (var i = 0; i < localSockets.Count; i++) + { + endpoints.AddRange(_surfaceAddresses.GetSnapshot(localSockets[i].Id)); + } + } + + return endpoints + .Where(static endpoint => endpoint.Port != 0) + .Select(UdpEndpointNormalization.Normalize) + .Where(endpoint => !endpoint.Equals(_rootEndpoint)) + .Distinct() + .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) + .ToArray(); + } + + private void SeedInitialExternalSurfaceObservations(IReadOnlyList? observations) + { + if (observations is null || observations.Count == 0) + { + return; + } + + for (var i = 0; i < observations.Count; i++) + { + var observation = observations[i]; + if (observation.SurfaceAddress.Port == 0) + { + continue; + } + + _surfaceAddresses.Observe(_rootNodeId, observation.LocalSocketId, UdpEndpointNormalization.Normalize(observation.SurfaceAddress)); + } + + if (ZeroTierTrace.Enabled) + { + var advertised = GetLocalDirectPathAdvertisements(); + ZeroTierTrace.WriteLine( + $"[zerotier] Seed local direct advertisements from root: {ZeroTierDirectEndpointSelection.Format(advertised)}."); + } + } + public NodeId NodeId => _localIdentity.NodeId; public IPEndPoint LocalUdp => _udp.LocalSockets[0].LocalEndpoint; @@ -696,7 +752,7 @@ private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, Ca var localSockets = _udp.LocalSockets; if (localSockets.Count == 0) { - await SendHelloPacketAsync(localSocketId: 0, peerNodeId, _rootEndpoint, _rootEndpoint, sharedKey, cancellationToken) + await SendHelloPacketAsync(localSocketId: 0, peerNodeId, physicalDestination: null, _rootEndpoint, sharedKey, cancellationToken) .ConfigureAwait(false); return; } @@ -706,7 +762,7 @@ await SendHelloPacketAsync(localSocketId: 0, peerNodeId, _rootEndpoint, _rootEnd await SendHelloPacketAsync( localSockets[i].Id, peerNodeId, - _rootEndpoint, + physicalDestination: null, _rootEndpoint, sharedKey, cancellationToken) @@ -716,14 +772,15 @@ await SendHelloPacketAsync( private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { - if (_localDirectPathAdvertisements.Length == 0) + var advertisements = GetLocalDirectPathAdvertisements(); + if (advertisements.Length == 0) { return; } try { - var payload = ZeroTierPushDirectPathsCodec.BuildPayload(_localDirectPathAdvertisements); + var payload = ZeroTierPushDirectPathsCodec.BuildPayload(advertisements); if (payload.Length <= 2) { return; @@ -748,7 +805,7 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine( - $"[zerotier] TX PUSH_DIRECT_PATHS via root for {peerNodeId}: {ZeroTierDirectEndpointSelection.Format(_localDirectPathAdvertisements)}."); + $"[zerotier] TX PUSH_DIRECT_PATHS via root for {peerNodeId}: {ZeroTierDirectEndpointSelection.Format(advertisements)}."); } await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); @@ -768,24 +825,13 @@ private async Task SendHelloDirectAsync( byte[] sharedKey, CancellationToken cancellationToken) { - var localSockets = _udp.LocalSockets; - if (localSockets.Count == 0) - { - for (var i = 0; i < endpoints.Length; i++) - { - await SendHelloPacketAsync(localSocketId: 0, peerNodeId, endpoints[i], endpoints[i], sharedKey, cancellationToken) - .ConfigureAwait(false); - } - - return; - } - for (var i = 0; i < endpoints.Length; i++) { - for (var s = 0; s < localSockets.Count; s++) + var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoints[i]); + for (var s = 0; s < localSocketIds.Length; s++) { await SendHelloPacketAsync( - localSockets[s].Id, + localSocketIds[s], peerNodeId, endpoints[i], endpoints[i], @@ -802,26 +848,85 @@ private async Task SendEchoDirectAsync( byte[] sharedKey, CancellationToken cancellationToken) { - var localSockets = _udp.LocalSockets; - if (localSockets.Count == 0) + for (var i = 0; i < endpoints.Length; i++) { - for (var i = 0; i < endpoints.Length; i++) + var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoints[i]); + for (var s = 0; s < localSocketIds.Length; s++) { - await TrySendEchoDirectProbeAsync(peerNodeId, localSocketId: 0, endpoints[i], sharedKey, cancellationToken) + await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpoints[i], sharedKey, cancellationToken) .ConfigureAwait(false); } + } + } - return; + private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + { + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return new[] { 0 }; } - for (var i = 0; i < endpoints.Length; i++) + var preferredFromHints = GetOrCreateDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); + if (preferredFromHints.Length != 0) { - for (var s = 0; s < localSockets.Count; s++) + return preferredFromHints; + } + + var preferred = new List(localSockets.Count); + for (var i = 0; i < localSockets.Count; i++) + { + var socketId = localSockets[i].Id; + if (_surfaceAddresses.GetSnapshot(socketId).Length != 0) { - await TrySendEchoDirectProbeAsync(peerNodeId, localSockets[s].Id, endpoints[i], sharedKey, cancellationToken) - .ConfigureAwait(false); + preferred.Add(socketId); } } + + if (preferred.Count != 0) + { + return preferred.ToArray(); + } + + return localSockets.Select(static socket => socket.Id).ToArray(); + } + + private async ValueTask HandleDirectEndpointHintAsync( + NodeId peerNodeId, + int receivedLocalSocketId, + IPEndPoint endpoint, + CancellationToken cancellationToken) + { + if (!_multipath.Enabled) + { + return; + } + + byte[] sharedKey; + try + { + sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is InvalidOperationException or OperationCanceledException) + { + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId}."); + } + + await TrySendEchoDirectProbeAsync(peerNodeId, receivedLocalSocketId, endpoint, sharedKey, cancellationToken) + .ConfigureAwait(false); + await SendHelloPacketAsync( + receivedLocalSocketId, + peerNodeId, + endpoint, + endpoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); } private async Task TrySendEchoDirectProbeAsync( @@ -854,7 +959,7 @@ await _peerEcho private async Task SendHelloPacketAsync( int localSocketId, NodeId peerNodeId, - IPEndPoint physicalDestination, + IPEndPoint? physicalDestination, IPEndPoint sendTo, byte[] sharedKey, CancellationToken cancellationToken) @@ -878,7 +983,9 @@ private async Task SendHelloPacketAsync( ZeroTierHelloClient.AdvertisedMajorVersion, ZeroTierHelloClient.AdvertisedMinorVersion, ZeroTierHelloClient.AdvertisedRevision, - out _); + out var packetId); + + TrackPendingHello(peerNodeId, localSocketId, sendTo, physicalDestination, packetId); await _udp.SendAsync(localSocketId, sendTo, packet, cancellationToken).ConfigureAwait(false); } @@ -899,28 +1006,31 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel return _bondEngine.TrySelectSinglePath(peerNodeId, observed, flowId, _multipath.BondPolicy, out selected); } + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (TrySelectConfirmedHintedDirectPath(peerNodeId, hinted, out selected)) + { + return true; + } + if (!_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId)) { if (ZeroTierTrace.Enabled) { - var rejectedHints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; - ZeroTierTrace.WriteLine( - $"[zerotier] Reject hinted direct path selection: peer={peerNodeId} hinted={ZeroTierDirectEndpointSelection.Format(rejectedHints)}."); + ZeroTierTrace.WriteLine($"[zerotier] Reject hinted direct path selection: peer={peerNodeId} hinted={ZeroTierDirectEndpointSelection.Format(hinted)}."); } selected = default; return false; } - var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; if (hinted.Length > 0) { - var localSockets = _udp.LocalSockets; var endpointIndex = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); - var socketIndex = localSockets.Count <= 1 + var preferredLocalSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, hinted[endpointIndex]); + var socketIndex = preferredLocalSocketIds.Length <= 1 ? 0 - : (int)((flowId / (uint)hinted.Length) % (uint)localSockets.Count); - var localSocketId = localSockets.Count == 0 ? 0 : localSockets[socketIndex].Id; + : (int)((flowId / (uint)hinted.Length) % (uint)preferredLocalSocketIds.Length); + var localSocketId = preferredLocalSocketIds[socketIndex]; if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine( @@ -935,6 +1045,43 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel return false; } + private bool TrySelectConfirmedHintedDirectPath( + NodeId peerNodeId, + IPEndPoint[] hinted, + out ZeroTierSelectedPeerPath selected) + { + selected = default; + + var haveBest = false; + var bestRttMs = int.MaxValue; + for (var i = 0; i < hinted.Length; i++) + { + var socketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, hinted[i]); + for (var s = 0; s < socketIds.Length; s++) + { + if (!_peerEcho.TryGetLastRttMs(peerNodeId, socketIds[s], hinted[i], out var rttMs)) + { + continue; + } + + if (!haveBest || rttMs < bestRttMs) + { + bestRttMs = rttMs; + selected = new ZeroTierSelectedPeerPath(socketIds[s], hinted[i]); + haveBest = true; + } + } + } + + if (haveBest && ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Select confirmed hinted direct path: peer={peerNodeId} endpoint={selected.RemoteEndPoint} socket={selected.LocalSocketId} rttMs={bestRttMs}."); + } + + return haveBest; + } + private int? GetPathLatencyMsOrNull(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) { if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) @@ -950,6 +1097,100 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel return null; } + private void HandlePeerHelloOk( + NodeId peerNodeId, + int receivedLocalSocketId, + IPEndPoint receivedVia, + byte hopCount, + ZeroTierHelloOkPayload ok) + { + ArgumentNullException.ThrowIfNull(receivedVia); + + var matchedPending = TryTakePendingHello(peerNodeId, ok.InRePacketId, out var pending); + var observedLocalSocketId = matchedPending ? pending.LocalSocketId : receivedLocalSocketId; + var trustedSurface = matchedPending && + hopCount == 0 && + pending.PhysicalDestination is { } physicalDestination && + receivedVia.Equals(physicalDestination); + + if (ok.ExternalSurfaceAddress is { } surface && trustedSurface) + { + _surfaceAddresses.Observe(peerNodeId, observedLocalSocketId, surface); + } + + if (ZeroTierTrace.Enabled) + { + var surfaceText = ok.ExternalSurfaceAddress?.ToString() ?? ""; + var probeText = matchedPending + ? $"sendTo={pending.SendTo} physical={pending.PhysicalDestination?.ToString() ?? ""}" + : "sendTo= physical="; + ZeroTierTrace.WriteLine( + $"[zerotier] RX OK(HELLO): peer={peerNodeId} via={receivedVia} hop={hopCount} inRe=0x{ok.InRePacketId:x16} localSocket={observedLocalSocketId} surface={surfaceText} matched={matchedPending} trustedSurface={trustedSurface} {probeText}."); + } + + if (matchedPending && + hopCount == 0 && + pending.PhysicalDestination is { } confirmedPhysicalDestination && + receivedVia.Equals(confirmedPhysicalDestination)) + { + _peerEcho.ObserveHelloOkRtt(peerNodeId, pending.LocalSocketId, receivedVia, ok.TimestampEcho); + return; + } + + _peerEcho.ObserveHelloOkRtt(peerNodeId, receivedLocalSocketId, receivedVia, ok.TimestampEcho); + } + + private void TrackPendingHello( + NodeId peerNodeId, + int localSocketId, + IPEndPoint sendTo, + IPEndPoint? physicalDestination, + ulong packetId) + { + CleanupPendingHellosIfNeeded(Environment.TickCount64); + _pendingHelloProbes[packetId] = new PendingHelloProbe( + PeerNodeId: peerNodeId, + LocalSocketId: localSocketId, + SendTo: sendTo, + PhysicalDestination: physicalDestination, + SentAtMs: Environment.TickCount64); + } + + private bool TryTakePendingHello(NodeId peerNodeId, ulong packetId, out PendingHelloProbe pending) + { + CleanupPendingHellosIfNeeded(Environment.TickCount64); + if (_pendingHelloProbes.TryRemove(packetId, out pending) && pending.PeerNodeId == peerNodeId) + { + return true; + } + + pending = default; + return false; + } + + private void CleanupPendingHellosIfNeeded(long nowMs) + { + var last = Volatile.Read(ref _lastPendingHelloCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 5_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastPendingHelloCleanupMs, nowMs, last) != last) + { + return; + } + + var cutoff = nowMs - (long)TimeSpan.FromMinutes(2).TotalMilliseconds; + foreach (var pair in _pendingHelloProbes) + { + if (pair.Value.SentAtMs <= cutoff) + { + _pendingHelloProbes.TryRemove(pair.Key, out _); + } + } + } + private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) => _peerNegotiation.TryGetRemoteUtility(peerNodeId, localSocketId, remoteEndPoint, out var util) ? util : (short)0; @@ -1195,6 +1436,7 @@ private Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken cancel private ValueTask HandleRootControlPacketAsync( ZeroTierVerb verb, ReadOnlyMemory payload, + int receivedLocalSocketId, IPEndPoint receivedVia, CancellationToken cancellationToken) { @@ -1209,12 +1451,14 @@ private ValueTask HandleRootControlPacketAsync( } var directEndpoints = GetOrCreateDirectEndpointManager(rendezvous.With); - return directEndpoints.HandleRendezvousFromRootAsync(payload, receivedVia, cancellationToken); + return directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken); } private ValueTask HandlePeerControlPacketAsync( NodeId peerNodeId, ZeroTierVerb verb, + int receivedLocalSocketId, + IPEndPoint receivedVia, ReadOnlyMemory payload, CancellationToken cancellationToken) { @@ -1224,13 +1468,13 @@ private ValueTask HandlePeerControlPacketAsync( } var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - return directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, cancellationToken); + return directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId, cancellationToken); } private ZeroTierDirectEndpointManager GetOrCreateDirectEndpointManager(NodeId peerNodeId) { _directEndpointLastUsedMs[peerNodeId] = Environment.TickCount64; - return _directEndpoints.GetOrAdd(peerNodeId, id => new ZeroTierDirectEndpointManager(_udp, _rootEndpoint, id)); + return _directEndpoints.GetOrAdd(peerNodeId, id => new ZeroTierDirectEndpointManager(_udp, _rootEndpoint, id, HandleDirectEndpointHintAsync)); } private void CleanupDirectEndpointManagers(long nowMs) @@ -1257,3 +1501,10 @@ private void CleanupDirectEndpointManagers(long nowMs) } } + +internal readonly record struct PendingHelloProbe( + NodeId PeerNodeId, + int LocalSocketId, + IPEndPoint SendTo, + IPEndPoint? PhysicalDestination, + long SentAtMs); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 9183cce..66d8539 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -14,7 +14,7 @@ internal sealed class ZeroTierDataplaneRxLoops private readonly NodeId _localNodeId; private readonly ZeroTierDataplaneRootClient _rootClient; private readonly IZeroTierDataplanePeerDatagramProcessor _peerDatagrams; - private readonly Func, IPEndPoint, CancellationToken, ValueTask>? _handleRootControlAsync; + private readonly Func, int, IPEndPoint, CancellationToken, ValueTask>? _handleRootControlAsync; private readonly Action? _onPeerQueueDrop; private readonly bool _acceptDirectPeerDatagrams; @@ -29,7 +29,7 @@ public ZeroTierDataplaneRxLoops( ZeroTierDataplaneRootClient rootClient, IZeroTierDataplanePeerDatagramProcessor peerDatagrams, bool acceptDirectPeerDatagrams = false, - Func, IPEndPoint, CancellationToken, ValueTask>? handleRootControlAsync = null, + Func, int, IPEndPoint, CancellationToken, ValueTask>? handleRootControlAsync = null, Action? onPeerQueueDrop = null) { ArgumentNullException.ThrowIfNull(udp); @@ -138,7 +138,7 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca { try { - await _handleRootControlAsync(verb, payload, datagram.RemoteEndPoint, cancellationToken).ConfigureAwait(false); + await _handleRootControlAsync(verb, payload, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 940a47e..4e52ad0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -23,15 +23,21 @@ internal sealed class ZeroTierDirectEndpointManager private readonly IZeroTierUdpTransport _udp; private readonly IPEndPoint _relayEndpoint; private readonly NodeId _remoteNodeId; + private readonly Func? _handleDirectEndpointHintAsync; private readonly object _lock = new(); private IPEndPoint[] _directEndpoints = Array.Empty(); + private readonly Dictionary> _preferredLocalSocketsByEndpoint = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _holePunchLastSentMs = new(StringComparer.Ordinal); private long _lastHolePunchCleanupMs; private long _lastDirectPathPushReceiveMs; private int _directPathPushCutoffCount; - public ZeroTierDirectEndpointManager(IZeroTierUdpTransport udp, IPEndPoint relayEndpoint, NodeId remoteNodeId) + public ZeroTierDirectEndpointManager( + IZeroTierUdpTransport udp, + IPEndPoint relayEndpoint, + NodeId remoteNodeId, + Func? handleDirectEndpointHintAsync = null) { ArgumentNullException.ThrowIfNull(udp); ArgumentNullException.ThrowIfNull(relayEndpoint); @@ -39,11 +45,44 @@ public ZeroTierDirectEndpointManager(IZeroTierUdpTransport udp, IPEndPoint relay _udp = udp; _relayEndpoint = relayEndpoint; _remoteNodeId = remoteNodeId; + _handleDirectEndpointHintAsync = handleDirectEndpointHintAsync; } public IPEndPoint[] Endpoints => _directEndpoints; - public ValueTask HandleRendezvousFromRootAsync(ReadOnlyMemory payload, IPEndPoint receivedVia, CancellationToken cancellationToken) + public int[] GetPreferredLocalSocketIds(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + int[] preferredSocketIds; + lock (_lock) + { + if (!_preferredLocalSocketsByEndpoint.TryGetValue(FormatEndpointKey(endpoint), out var socketIds) || socketIds.Count == 0) + { + return Array.Empty(); + } + + preferredSocketIds = socketIds.ToArray(); + } + + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return preferredSocketIds.Contains(0) ? new[] { 0 } : Array.Empty(); + } + + var available = localSockets.Select(static socket => socket.Id).ToHashSet(); + return preferredSocketIds + .Where(available.Contains) + .Distinct() + .ToArray(); + } + + public async ValueTask HandleRendezvousFromRootAsync( + ReadOnlyMemory payload, + int receivedLocalSocketId, + IPEndPoint receivedVia, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -58,28 +97,36 @@ public ValueTask HandleRendezvousFromRootAsync(ReadOnlyMemory payload, IPE lock (_lock) { _directEndpoints = endpoints; + RememberPreferredLocalSockets_NoLock(endpoints, receivedLocalSocketId); } foreach (var endpoint in endpoints) { - TrySendHolePunch(endpoint); + TrySendHolePunch(endpoint, preferredLocalSocketIds: new[] { receivedLocalSocketId }); + if (_handleDirectEndpointHintAsync is not null) + { + await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); + } } - return ValueTask.CompletedTask; + return; } ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS (ignored) via {receivedVia}."); - return ValueTask.CompletedTask; + return; } - public ValueTask HandlePushDirectPathsFromRemoteAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + public async ValueTask HandlePushDirectPathsFromRemoteAsync( + ReadOnlyMemory payload, + int receivedLocalSocketId, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); if (!ZeroTierPushDirectPathsCodec.TryParse(payload.Span, out var paths) || paths.Length == 0) { ZeroTierTrace.WriteLine("[zerotier] Drop: failed to parse PUSH_DIRECT_PATHS payload."); - return ValueTask.CompletedTask; + return; } var now = Environment.TickCount64; @@ -90,7 +137,7 @@ public ValueTask HandlePushDirectPathsFromRemoteAsync(ReadOnlyMemory paylo ZeroTierTrace.WriteLine($"[zerotier] Drop: PUSH_DIRECT_PATHS rate-gated (peer={_remoteNodeId})."); } - return ValueTask.CompletedTask; + return; } var forget = new HashSet(StringComparer.Ordinal); @@ -129,11 +176,13 @@ public ValueTask HandlePushDirectPathsFromRemoteAsync(ReadOnlyMemory paylo endpoints = ZeroTierDirectEndpointSelection.Normalize(merged, _relayEndpoint, maxEndpoints: MaxEndpoints); _directEndpoints = endpoints; + PrunePreferredLocalSockets_NoLock(endpoints); + RememberPreferredLocalSockets_NoLock(redirect.Concat(add), receivedLocalSocketId); } if (endpoints.Length == 0) { - return ValueTask.CompletedTask; + return; } if (ZeroTierTrace.Enabled) @@ -143,10 +192,14 @@ public ValueTask HandlePushDirectPathsFromRemoteAsync(ReadOnlyMemory paylo foreach (var endpoint in endpoints) { - TrySendHolePunch(endpoint); + TrySendHolePunch(endpoint, preferredLocalSocketIds: new[] { receivedLocalSocketId }); + if (_handleDirectEndpointHintAsync is not null) + { + await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); + } } - return ValueTask.CompletedTask; + return; } public void SeedEndpoints(IEnumerable endpoints) @@ -161,6 +214,7 @@ public void SeedEndpoints(IEnumerable endpoints) _relayEndpoint, maxEndpoints: MaxEndpoints); _directEndpoints = normalized; + PrunePreferredLocalSockets_NoLock(normalized); } if (ZeroTierTrace.Enabled && normalized.Length > 0) @@ -192,7 +246,7 @@ private bool RateGatePushDirectPaths(long nowMs) } } - private void TrySendHolePunch(IPEndPoint endpoint) + private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketIds = null) { var localSockets = _udp.LocalSockets; var now = Environment.TickCount64; @@ -201,6 +255,22 @@ private void TrySendHolePunch(IPEndPoint endpoint) var junk = new byte[4]; RandomNumberGenerator.Fill(junk); + if (preferredLocalSocketIds is { Length: > 0 }) + { + for (var i = 0; i < preferredLocalSocketIds.Length; i++) + { + var socketId = preferredLocalSocketIds[i]; + if (!ShouldSendHolePunch(socketId, endpoint, now)) + { + continue; + } + + TrySendHolePunchCore(socketId, endpoint, junk); + } + + return; + } + if (localSockets.Count == 0) { if (!ShouldSendHolePunch(localSocketId: 0, endpoint, now)) @@ -321,6 +391,36 @@ private void CleanupHolePunchCacheIfNeeded(long nowMs) } } + private void RememberPreferredLocalSockets_NoLock(IEnumerable endpoints, int localSocketId) + { + foreach (var endpoint in endpoints) + { + var key = FormatEndpointKey(endpoint); + if (!_preferredLocalSocketsByEndpoint.TryGetValue(key, out var socketIds)) + { + socketIds = new HashSet(); + _preferredLocalSocketsByEndpoint[key] = socketIds; + } + + socketIds.Add(localSocketId); + } + } + + private void PrunePreferredLocalSockets_NoLock(IEnumerable endpoints) + { + var keep = endpoints + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); + + foreach (var key in _preferredLocalSocketsByEndpoint.Keys.ToArray()) + { + if (!keep.Contains(key)) + { + _preferredLocalSocketsByEndpoint.Remove(key); + } + } + } + private static string FormatEndpointKey(IPEndPoint endpoint) { var address = endpoint.Address; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs b/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs index da680ee..c617d1c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs @@ -157,13 +157,13 @@ public async ValueTask> ReceiveAsync(CancellationToken canc case ZeroTierVerb.Rendezvous when isFromRoot: { await _directEndpoints - .HandleRendezvousFromRootAsync(payload, datagram.RemoteEndPoint, cancellationToken) + .HandleRendezvousFromRootAsync(payload, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) .ConfigureAwait(false); continue; } case ZeroTierVerb.PushDirectPaths when isFromRemote: { - await _directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, cancellationToken).ConfigureAwait(false); + await _directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, datagram.LocalSocketId, cancellationToken).ConfigureAwait(false); continue; } case ZeroTierVerb.MulticastFrame: From 3ce1facf113eca8c405dbffa9fb33cc0913e28ac Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 22:55:01 +0100 Subject: [PATCH 146/296] Try hinted direct probes on mapped sockets --- .../Internal/ZeroTierDataplaneRuntime.cs | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 94a7781..220ebdc 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -867,17 +867,20 @@ private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint return new[] { 0 }; } + var preferred = new List(localSockets.Count); var preferredFromHints = GetOrCreateDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); - if (preferredFromHints.Length != 0) + for (var i = 0; i < preferredFromHints.Length; i++) { - return preferredFromHints; + if (!preferred.Contains(preferredFromHints[i])) + { + preferred.Add(preferredFromHints[i]); + } } - var preferred = new List(localSockets.Count); for (var i = 0; i < localSockets.Count; i++) { var socketId = localSockets[i].Id; - if (_surfaceAddresses.GetSnapshot(socketId).Length != 0) + if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && !preferred.Contains(socketId)) { preferred.Add(socketId); } @@ -917,16 +920,20 @@ private async ValueTask HandleDirectEndpointHintAsync( ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId}."); } - await TrySendEchoDirectProbeAsync(peerNodeId, receivedLocalSocketId, endpoint, sharedKey, cancellationToken) - .ConfigureAwait(false); - await SendHelloPacketAsync( - receivedLocalSocketId, - peerNodeId, - endpoint, - endpoint, - sharedKey, - cancellationToken) - .ConfigureAwait(false); + var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoint); + for (var i = 0; i < localSocketIds.Length; i++) + { + await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) + .ConfigureAwait(false); + await SendHelloPacketAsync( + localSocketIds[i], + peerNodeId, + endpoint, + endpoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } } private async Task TrySendEchoDirectProbeAsync( From 0496e5b83e9c9f44bf666c7432728c4a90517f52 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:02:31 +0100 Subject: [PATCH 147/296] Retry root surface probing across sockets --- ZTSharp.Tests/ZeroTierHelloClientTests.cs | 212 +++++++++++- .../ZeroTier/Internal/ZeroTierHelloClient.cs | 317 ++++++++++++++++-- 2 files changed, 499 insertions(+), 30 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierHelloClientTests.cs b/ZTSharp.Tests/ZeroTierHelloClientTests.cs index f392bb0..8f323b4 100644 --- a/ZTSharp.Tests/ZeroTierHelloClientTests.cs +++ b/ZTSharp.Tests/ZeroTierHelloClientTests.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Net; +using System.Linq; using ZTSharp.ZeroTier.Internal; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -52,6 +53,9 @@ public async Task HelloRootsAsync_SendsHello_And_ParsesOk() Assert.Equal(12, ok.RemoteMinorVersion); Assert.Equal(0, ok.RemoteRevision); Assert.Equal(new IPEndPoint(IPAddress.Loopback, 9993), ok.ExternalSurfaceAddress); + var observation = Assert.Single(ok.ExternalSurfaceObservations); + Assert.Equal(0, observation.LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 9993), observation.SurfaceAddress); await serverTask; } @@ -144,6 +148,112 @@ public async Task HelloAsync_SendsHello_AndCompletesOnOk() await serverTask; } + [Fact] + public async Task HelloRootsAsync_WithMultiTransport_CollectsSurfaceObservationsPerSocket() + { + Assert.True(ZeroTierIdentity.TryParse(KnownGoodIdentity, out var localIdentity)); + Assert.NotNull(localIdentity.PrivateKey); + + var rootIdentity = new ZeroTierIdentity( + new NodeId(0x1122334455), + (byte[])localIdentity.PublicKey.Clone(), + (byte[])localIdentity.PrivateKey.Clone()); + + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true); + await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 0); + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 1); + await using var multi = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); + + var rootEndpoint = TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint); + var planet = new ZeroTierWorld( + ZeroTierWorldType.Planet, + id: 1, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: new[] + { + new ZeroTierWorldRoot(rootIdentity, new[] { rootEndpoint }) + }); + + var serverTask = RunHelloServerTwiceAsync(rootUdp, rootIdentity, localIdentity); + + var ok = await ZeroTierHelloClient.HelloRootsAsync( + multi, + localIdentity, + planet, + timeout: TimeSpan.FromSeconds(2), + cancellationToken: CancellationToken.None); + + Assert.Equal(2, ok.ExternalSurfaceObservations.Count); + Assert.Equal(0, ok.ExternalSurfaceObservations[0].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 11000), ok.ExternalSurfaceObservations[0].SurfaceAddress); + Assert.Equal(1, ok.ExternalSurfaceObservations[1].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 11001), ok.ExternalSurfaceObservations[1].SurfaceAddress); + + await serverTask; + } + + [Fact] + public async Task ProbeRootExternalSurfacesAsync_RetriesSocketsUntilAllMappingsAreObserved() + { + Assert.True(ZeroTierIdentity.TryParse(KnownGoodIdentity, out var localIdentity)); + Assert.NotNull(localIdentity.PrivateKey); + + var rootIdentity = new ZeroTierIdentity( + new NodeId(0x1122334455), + (byte[])localIdentity.PublicKey.Clone(), + (byte[])localIdentity.PrivateKey.Clone()); + + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true); + await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 0); + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 1); + await using var multi = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); + + var rootEndpoint = TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint); + var planet = new ZeroTierWorld( + ZeroTierWorldType.Planet, + id: 1, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: new[] + { + new ZeroTierWorldRoot(rootIdentity, new[] { rootEndpoint }) + }); + + var rootKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, rootIdentity.PublicKey, rootKey); + + var firstSocketPort = socket0.LocalEndpoint.Port; + var delayedSocketPort = socket1.LocalEndpoint.Port; + var serverTask = RunHelloSurfaceProbeRetryServerAsync( + rootUdp, + rootIdentity, + localIdentity, + firstSocketPort, + delayedSocketPort); + + var observations = await ZeroTierHelloClient.ProbeRootExternalSurfacesAsync( + multi, + localIdentity, + planet, + rootIdentity.NodeId, + rootEndpoint, + rootKey, + knownObservations: Array.Empty(), + timeout: TimeSpan.FromSeconds(3), + cancellationToken: CancellationToken.None); + + Assert.Equal(2, observations.Count); + Assert.Equal(0, observations[0].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 12000), observations[0].SurfaceAddress); + Assert.Equal(1, observations[1].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 12001), observations[1].SurfaceAddress); + + await serverTask; + } + private static async Task RunHelloServerOnceAsync( ZeroTierUdpTransport transport, ZeroTierIdentity rootIdentity, @@ -244,14 +354,112 @@ private static async Task RunHelloRootsMismatchServerOnceAsync( await rootBUdp.SendAsync(helloBDatagram.RemoteEndPoint, okFromB).ConfigureAwait(false); } + private static async Task RunHelloServerTwiceAsync( + ZeroTierUdpTransport transport, + ZeroTierIdentity rootIdentity, + ZeroTierIdentity localIdentity) + { + var firstHelloTask = transport.ReceiveAsync(TimeSpan.FromSeconds(2)).AsTask(); + var secondHelloTask = transport.ReceiveAsync(TimeSpan.FromSeconds(2)).AsTask(); + + await Task.WhenAll(firstHelloTask, secondHelloTask).ConfigureAwait(false); + + var hellos = new[] { await firstHelloTask.ConfigureAwait(false), await secondHelloTask.ConfigureAwait(false) } + .OrderBy(static datagram => datagram.LocalSocketId) + .ThenBy(static datagram => datagram.RemoteEndPoint.Port) + .ToArray(); + + for (var i = 0; i < hellos.Length; i++) + { + var helloDatagram = hellos[i]; + var helloPacketBytes = helloDatagram.Payload; + Assert.True(ZeroTierPacketCodec.TryDecode(helloPacketBytes, out var hello)); + Assert.Equal(ZeroTierVerb.Hello, hello.Header.Verb); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(rootIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); + + Assert.True(ZeroTierPacketCrypto.Dearmor(helloPacketBytes, sharedKey)); + + var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian( + helloPacketBytes.AsSpan(ZeroTierPacketHeader.Length + 5, 8)); + + var okPacket = BuildHelloOkPacket( + rootNodeId: rootIdentity.NodeId, + localNodeId: localIdentity.NodeId, + sharedKey: sharedKey, + inRePacketId: hello.Header.PacketId, + helloTimestampEcho: helloTimestamp, + surface: new IPEndPoint(IPAddress.Loopback, 11000 + i)); + + await transport.SendAsync(helloDatagram.RemoteEndPoint, okPacket).ConfigureAwait(false); + } + } + + private static async Task RunHelloSurfaceProbeRetryServerAsync( + ZeroTierUdpTransport transport, + ZeroTierIdentity rootIdentity, + ZeroTierIdentity localIdentity, + int immediateSocketPort, + int delayedSocketPort) + { + var helloCountsByRemotePort = new Dictionary(); + var responsesSent = 0; + + while (responsesSent < 2) + { + var helloDatagram = await transport.ReceiveAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); + var helloPacketBytes = helloDatagram.Payload; + Assert.True(ZeroTierPacketCodec.TryDecode(helloPacketBytes, out var hello)); + Assert.Equal(ZeroTierVerb.Hello, hello.Header.Verb); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(rootIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); + + Assert.True(ZeroTierPacketCrypto.Dearmor(helloPacketBytes, sharedKey)); + + var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian( + helloPacketBytes.AsSpan(ZeroTierPacketHeader.Length + 5, 8)); + + var remotePort = helloDatagram.RemoteEndPoint.Port; + helloCountsByRemotePort.TryGetValue(remotePort, out var helloCount); + helloCount++; + helloCountsByRemotePort[remotePort] = helloCount; + + IPEndPoint? surface = remotePort switch + { + _ when remotePort == immediateSocketPort && helloCount == 1 => new IPEndPoint(IPAddress.Loopback, 12000), + _ when remotePort == delayedSocketPort && helloCount == 2 => new IPEndPoint(IPAddress.Loopback, 12001), + _ => null + }; + + if (surface is null) + { + continue; + } + + var okPacket = BuildHelloOkPacket( + rootNodeId: rootIdentity.NodeId, + localNodeId: localIdentity.NodeId, + sharedKey: sharedKey, + inRePacketId: hello.Header.PacketId, + helloTimestampEcho: helloTimestamp, + surface: surface); + + await transport.SendAsync(helloDatagram.RemoteEndPoint, okPacket).ConfigureAwait(false); + responsesSent++; + } + } + private static byte[] BuildHelloOkPacket( NodeId rootNodeId, NodeId localNodeId, byte[] sharedKey, ulong inRePacketId, - ulong helloTimestampEcho) + ulong helloTimestampEcho, + IPEndPoint? surface = null) { - var surface = new IPEndPoint(IPAddress.Loopback, 9993); + surface ??= new IPEndPoint(IPAddress.Loopback, 9993); var surfaceLength = ZeroTierInetAddressCodec.GetSerializedLength(surface); var okPayloadLength = 1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs index f8700d9..84926da 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Linq; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -13,7 +14,12 @@ internal readonly record struct ZeroTierHelloOk( byte RemoteMajorVersion, byte RemoteMinorVersion, ushort RemoteRevision, - IPEndPoint? ExternalSurfaceAddress); + IPEndPoint? ExternalSurfaceAddress, + IReadOnlyList ExternalSurfaceObservations); + +internal readonly record struct ZeroTierExternalSurfaceObservation( + int LocalSocketId, + IPEndPoint SurfaceAddress); internal static class ZeroTierHelloClient { @@ -21,6 +27,7 @@ internal static class ZeroTierHelloClient internal const byte AdvertisedMajorVersion = 1; internal const byte AdvertisedMinorVersion = 12; internal const ushort AdvertisedRevision = 0; + private const int RootSurfaceProbeRoundReceiveWindowMs = 1_000; public static async Task HelloRootsAsync( IZeroTierUdpTransport udp, @@ -45,7 +52,10 @@ public static async Task HelloRootsAsync( var rootKeys = ZeroTierRootKeyDerivation.BuildRootKeys(localIdentity, planet); var helloTimestamp = (ulong)Environment.TickCount64; - var pending = new Dictionary(capacity: planet.Roots.Count); + var pending = new Dictionary(capacity: planet.Roots.Count); + var localSocketIds = udp.LocalSockets.Count == 0 + ? new[] { 0 } + : udp.LocalSockets.Select(static socket => socket.Id).ToArray(); foreach (var root in planet.Roots) { @@ -56,28 +66,40 @@ public static async Task HelloRootsAsync( foreach (var endpoint in root.StableEndpoints) { - var packet = ZeroTierHelloPacketBuilder.BuildPacket( - localIdentity, - destination: root.Identity.NodeId, - physicalDestination: endpoint, - planet, - helloTimestamp, - key, - advertisedProtocolVersion: AdvertisedProtocolVersion, - advertisedMajorVersion: AdvertisedMajorVersion, - advertisedMinorVersion: AdvertisedMinorVersion, - advertisedRevision: AdvertisedRevision, - out var packetId); - - try + for (var i = 0; i < localSocketIds.Length; i++) { - await udp.SendAsync(endpoint, packet, cancellationToken).ConfigureAwait(false); - pending[packetId] = root.Identity.NodeId; - } - catch (System.Net.Sockets.SocketException) - { - // Some environments don't have IPv6 connectivity. Ignore send failures and wait for any - // reachable root to respond. + var localSocketId = localSocketIds[i]; + var packet = ZeroTierHelloPacketBuilder.BuildPacket( + localIdentity, + destination: root.Identity.NodeId, + physicalDestination: endpoint, + planet, + helloTimestamp, + key, + advertisedProtocolVersion: AdvertisedProtocolVersion, + advertisedMajorVersion: AdvertisedMajorVersion, + advertisedMinorVersion: AdvertisedMinorVersion, + advertisedRevision: AdvertisedRevision, + out var packetId); + + try + { + if (udp.LocalSockets.Count == 0) + { + await udp.SendAsync(endpoint, packet, cancellationToken).ConfigureAwait(false); + } + else + { + await udp.SendAsync(localSocketId, endpoint, packet, cancellationToken).ConfigureAwait(false); + } + + pending[packetId] = new PendingRootHello(root.Identity.NodeId, localSocketId); + } + catch (System.Net.Sockets.SocketException) + { + // Some environments don't have IPv6 connectivity. Ignore send failures and wait for any + // reachable root to respond. + } } } } @@ -93,6 +115,7 @@ public static async Task HelloRootsAsync( async ValueTask WaitForHelloOkAsync(CancellationToken token) { + var surfacesBySocket = new Dictionary(); while (true) { var datagram = await udp.ReceiveAsync(token).ConfigureAwait(false); @@ -113,18 +136,23 @@ async ValueTask WaitForHelloOkAsync(CancellationToken token) continue; } - if (!pending.TryGetValue(ok.InRePacketId, out var rootNodeId)) + if (!pending.TryGetValue(ok.InRePacketId, out var pendingHello)) { continue; } - if (rootNodeId != packet.Header.Source) + if (pendingHello.RootNodeId != packet.Header.Source) { continue; } - return new ZeroTierHelloOk( - RootNodeId: rootNodeId, + if (ok.ExternalSurfaceAddress is { } surface) + { + surfacesBySocket[pendingHello.LocalSocketId] = surface; + } + + var result = new ZeroTierHelloOk( + RootNodeId: pendingHello.RootNodeId, RootEndpoint: datagram.RemoteEndPoint, HelloPacketId: ok.InRePacketId, HelloTimestampEcho: ok.TimestampEcho, @@ -132,9 +160,238 @@ async ValueTask WaitForHelloOkAsync(CancellationToken token) RemoteMajorVersion: ok.RemoteMajorVersion, RemoteMinorVersion: ok.RemoteMinorVersion, RemoteRevision: ok.RemoteRevision, - ExternalSurfaceAddress: ok.ExternalSurfaceAddress); + ExternalSurfaceAddress: ok.ExternalSurfaceAddress, + ExternalSurfaceObservations: Array.Empty()); + + return await CollectAdditionalSurfaceReportsAsync(result, surfacesBySocket, token).ConfigureAwait(false); + } + } + + async ValueTask CollectAdditionalSurfaceReportsAsync( + ZeroTierHelloOk initial, + Dictionary surfacesBySocket, + CancellationToken token) + { + var outstandingSockets = localSocketIds.Length - surfacesBySocket.Count; + if (outstandingSockets <= 0) + { + return initial with + { + ExternalSurfaceObservations = surfacesBySocket + .Select(static pair => new ZeroTierExternalSurfaceObservation(pair.Key, pair.Value)) + .OrderBy(static pair => pair.LocalSocketId) + .ToArray() + }; + } + + var graceDeadline = Environment.TickCount64 + 400; + while (outstandingSockets > 0) + { + var remainingMs = unchecked(graceDeadline - Environment.TickCount64); + if (remainingMs <= 0) + { + break; + } + + ZeroTierUdpDatagram datagram; + try + { + datagram = await udp.ReceiveAsync(TimeSpan.FromMilliseconds(remainingMs), token).ConfigureAwait(false); + } + catch (TimeoutException) + { + break; + } + + var packetBytes = datagram.Payload; + if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var packet)) + { + continue; + } + + if (!rootKeys.TryGetValue(packet.Header.Source, out var key)) + { + continue; + } + + if (!ZeroTierHelloOkParser.TryParse(packetBytes, key, out var ok)) + { + continue; + } + + if (!pending.TryGetValue(ok.InRePacketId, out var pendingHello)) + { + continue; + } + + if (pendingHello.RootNodeId != packet.Header.Source || ok.ExternalSurfaceAddress is not { } surface) + { + continue; + } + + if (surfacesBySocket.TryAdd(pendingHello.LocalSocketId, surface)) + { + outstandingSockets--; + } + } + + return initial with + { + ExternalSurfaceObservations = surfacesBySocket + .Select(static pair => new ZeroTierExternalSurfaceObservation(pair.Key, pair.Value)) + .OrderBy(static pair => pair.LocalSocketId) + .ToArray() + }; + } + } + + public static async Task> ProbeRootExternalSurfacesAsync( + IZeroTierUdpTransport udp, + ZeroTierIdentity localIdentity, + ZeroTierWorld planet, + NodeId rootNodeId, + IPEndPoint rootEndpoint, + byte[] rootKey, + IReadOnlyList knownObservations, + TimeSpan timeout, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(localIdentity); + ArgumentNullException.ThrowIfNull(planet); + ArgumentNullException.ThrowIfNull(rootEndpoint); + ArgumentNullException.ThrowIfNull(rootKey); + ArgumentNullException.ThrowIfNull(knownObservations); + if (timeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(timeout), timeout, "Timeout must be positive."); + } + + var observations = knownObservations + .Where(static observation => observation.SurfaceAddress.Port != 0) + .GroupBy(static observation => observation.LocalSocketId) + .ToDictionary(static group => group.Key, static group => group.Last().SurfaceAddress); + + var localSocketIds = udp.LocalSockets.Count == 0 + ? new[] { 0 } + : udp.LocalSockets.Select(static socket => socket.Id).ToArray(); + var deadline = Environment.TickCount64 + (long)timeout.TotalMilliseconds; + var pending = new Dictionary(); + + while (observations.Count < localSocketIds.Length) + { + var remainingMs = unchecked(deadline - Environment.TickCount64); + if (remainingMs <= 0) + { + break; + } + + var sentAny = false; + for (var i = 0; i < localSocketIds.Length; i++) + { + var localSocketId = localSocketIds[i]; + if (observations.ContainsKey(localSocketId)) + { + continue; + } + + var packet = ZeroTierHelloPacketBuilder.BuildPacket( + localIdentity, + destination: rootNodeId, + physicalDestination: rootEndpoint, + planet, + timestamp: (ulong)Environment.TickCount64, + rootKey, + advertisedProtocolVersion: AdvertisedProtocolVersion, + advertisedMajorVersion: AdvertisedMajorVersion, + advertisedMinorVersion: AdvertisedMinorVersion, + advertisedRevision: AdvertisedRevision, + out var packetId); + + try + { + if (udp.LocalSockets.Count == 0) + { + await udp.SendAsync(rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + else + { + await udp.SendAsync(localSocketId, rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + } + catch (System.Net.Sockets.SocketException) + { + continue; + } + + pending[packetId] = localSocketId; + sentAny = true; + } + + if (!sentAny) + { + break; + } + + var roundDeadline = Math.Min( + deadline, + Environment.TickCount64 + Math.Min((int)remainingMs, RootSurfaceProbeRoundReceiveWindowMs)); + + while (pending.Count > 0) + { + remainingMs = Math.Min( + unchecked(deadline - Environment.TickCount64), + unchecked(roundDeadline - Environment.TickCount64)); + if (remainingMs <= 0) + { + break; + } + + ZeroTierUdpDatagram datagram; + try + { + datagram = await udp.ReceiveAsync(TimeSpan.FromMilliseconds(remainingMs), cancellationToken).ConfigureAwait(false); + } + catch (TimeoutException) + { + break; + } + + var packetBytes = datagram.Payload; + if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var decoded)) + { + continue; + } + + if (decoded.Header.Source != rootNodeId) + { + continue; + } + + if (!ZeroTierHelloOkParser.TryParse(packetBytes, rootKey, out var ok)) + { + continue; + } + + if (!pending.Remove(ok.InRePacketId, out var localSocketId) || ok.ExternalSurfaceAddress is not { } surface) + { + continue; + } + + observations[localSocketId] = surface; + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Root surface probe: socket={localSocketId} surface={surface} via {datagram.RemoteEndPoint}."); + } + + break; } } + + return observations + .Select(static pair => new ZeroTierExternalSurfaceObservation(pair.Key, pair.Value)) + .OrderBy(static observation => observation.LocalSocketId) + .ToArray(); } public static async Task HelloAsync( @@ -233,3 +490,7 @@ public static async Task HelloRootsAsync( } } + +internal readonly record struct PendingRootHello( + NodeId RootNodeId, + int LocalSocketId); From a069804d8b2e442da6abab81ff7699ebe997f342 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:14:39 +0100 Subject: [PATCH 148/296] Bootstrap peer trust before direct path probing --- ...ierNetworkCredentialsPacketBuilderTests.cs | 33 ++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 83 +++++++++++++++++++ ...ZeroTierNetworkCredentialsPacketBuilder.cs | 45 ++++++++++ 3 files changed, 161 insertions(+) create mode 100644 ZTSharp.Tests/ZeroTierNetworkCredentialsPacketBuilderTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierNetworkCredentialsPacketBuilder.cs diff --git a/ZTSharp.Tests/ZeroTierNetworkCredentialsPacketBuilderTests.cs b/ZTSharp.Tests/ZeroTierNetworkCredentialsPacketBuilderTests.cs new file mode 100644 index 0000000..d49dbae --- /dev/null +++ b/ZTSharp.Tests/ZeroTierNetworkCredentialsPacketBuilderTests.cs @@ -0,0 +1,33 @@ +using System.Buffers.Binary; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierNetworkCredentialsPacketBuilderTests +{ + [Fact] + public void BuildPacket_EncodesInlineComAndEmptyCredentialLists() + { + var sharedKey = Enumerable.Range(1, 48).Select(static value => (byte)value).ToArray(); + var inlineCom = new byte[] { 0x7f, 0x01, 0x02, 0x03 }; + + var packet = ZeroTierNetworkCredentialsPacketBuilder.BuildPacket( + packetId: 0x0102030405060708, + destination: new NodeId(0x1111111111), + source: new NodeId(0x2222222222), + inlineCom: inlineCom, + sharedKey: sharedKey); + + Assert.True(ZeroTierPacketCrypto.Dearmor(packet, sharedKey)); + var payload = packet.AsSpan(ZeroTierPacketHeader.IndexPayload); + Assert.True(payload.Slice(0, inlineCom.Length).SequenceEqual(inlineCom)); + Assert.Equal(0, payload[inlineCom.Length]); + + var counts = payload.Slice(inlineCom.Length + 1); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(0, 2))); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(2, 2))); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(4, 2))); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(6, 2))); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 220ebdc..bbc3cd2 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -15,6 +15,8 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable { private const long DirectEndpointManagerTtlMs = 600_000; + private const long NetworkCredentialsBootstrapIntervalMs = 5_000; + private const long DirectHelloMinIntervalMs = 1_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -42,6 +44,8 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _directBootstrapTasks = new(); + private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); + private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); private readonly ConcurrentDictionary _pendingHelloProbes = new(); private long _lastPendingHelloCleanupMs; @@ -677,6 +681,7 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextRootHelloAt) >= 0) { await SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendNetworkCredentialsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); nextRootHelloAt = now + (long)TimeSpan.FromSeconds(1).TotalMilliseconds; } @@ -819,6 +824,55 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } + private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + if (_inlineCom.Length == 0) + { + return; + } + + var nowMs = Environment.TickCount64; + while (true) + { + if (_lastNetworkCredentialsBootstrapMs.TryGetValue(peerNodeId, out var lastSent) && + unchecked(nowMs - lastSent) < NetworkCredentialsBootstrapIntervalMs) + { + return; + } + + if (_lastNetworkCredentialsBootstrapMs.TryAdd(peerNodeId, nowMs) || + _lastNetworkCredentialsBootstrapMs.TryUpdate(peerNodeId, nowMs, lastSent)) + { + break; + } + } + + try + { + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var packet = ZeroTierNetworkCredentialsPacketBuilder.BuildPacket( + packetId: ZeroTierPacketIdGenerator.GeneratePacketId(), + destination: peerNodeId, + source: _localIdentity.NodeId, + inlineCom: _inlineCom, + sharedKey: ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion)); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX NETWORK_CREDENTIALS via root for {peerNodeId}."); + } + + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] NETWORK_CREDENTIALS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + private async Task SendHelloDirectAsync( NodeId peerNodeId, IPEndPoint[] endpoints, @@ -971,6 +1025,11 @@ private async Task SendHelloPacketAsync( byte[] sharedKey, CancellationToken cancellationToken) { + if (physicalDestination is not null && !ShouldSendDirectHello(peerNodeId, localSocketId, sendTo)) + { + return; + } + try { if (ZeroTierTrace.Enabled) @@ -1005,6 +1064,30 @@ private async Task SendHelloPacketAsync( } } + private bool ShouldSendDirectHello(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + { + var endpoint = remoteEndPoint.Address.IsIPv4MappedToIPv6 + ? new IPEndPoint(remoteEndPoint.Address.MapToIPv4(), remoteEndPoint.Port) + : remoteEndPoint; + var key = (peerNodeId, localSocketId, endpoint.ToString()); + var nowMs = Environment.TickCount64; + + while (true) + { + if (_lastDirectHelloSentMs.TryGetValue(key, out var lastSent) && + unchecked(nowMs - lastSent) < DirectHelloMinIntervalMs) + { + return false; + } + + if (_lastDirectHelloSentMs.TryAdd(key, nowMs) || + _lastDirectHelloSentMs.TryUpdate(key, nowMs, lastSent)) + { + return true; + } + } + } + private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSelectedPeerPath selected) { var observed = _peerPaths.GetSnapshot(peerNodeId); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierNetworkCredentialsPacketBuilder.cs b/ZTSharp/ZeroTier/Internal/ZeroTierNetworkCredentialsPacketBuilder.cs new file mode 100644 index 0000000..e0f6fab --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierNetworkCredentialsPacketBuilder.cs @@ -0,0 +1,45 @@ +using System.Buffers.Binary; +using ZTSharp.ZeroTier.Protocol; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierNetworkCredentialsPacketBuilder +{ + public static byte[] BuildPacket( + ulong packetId, + NodeId destination, + NodeId source, + ReadOnlySpan inlineCom, + ReadOnlySpan sharedKey) + { + if (inlineCom.IsEmpty) + { + throw new ArgumentException("Inline COM must not be empty.", nameof(inlineCom)); + } + + var payload = new byte[inlineCom.Length + 1 + (sizeof(ushort) * 4)]; + inlineCom.CopyTo(payload); + + var ptr = inlineCom.Length; + payload[ptr++] = 0; // End of COM array. + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + ptr += 2; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + ptr += 2; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + ptr += 2; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + + var header = new ZeroTierPacketHeader( + PacketId: packetId, + Destination: destination, + Source: source, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.NetworkCredentials); + + var packet = ZeroTierPacketCodec.Encode(header, payload); + ZeroTierPacketCrypto.Armor(packet, sharedKey, encryptPayload: true); + return packet; + } +} From 06632c28272d19d6a14b4ad417ca9493c5d44fda Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:22:19 +0100 Subject: [PATCH 149/296] Improve direct path confirmation and rendezvous punch --- ...oTierDirectEndpointManagerHopLimitTests.cs | 83 +++++++++++ ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs | 15 +- .../ZeroTierDataplanePeerDatagramProcessor.cs | 19 ++- .../Internal/ZeroTierDataplaneRuntime.cs | 39 ++++- .../Internal/ZeroTierDirectEndpointManager.cs | 23 ++- .../ZeroTierPeerPhysicalPathTracker.cs | 7 +- .../Transport/IZeroTierUdpTransport.cs | 7 + .../Transport/ZeroTierUdpMultiTransport.cs | 12 ++ .../Transport/ZeroTierUdpTransport.cs | 137 +++++++++++++++++- 9 files changed, 322 insertions(+), 20 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs new file mode 100644 index 0000000..44f7322 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs @@ -0,0 +1,83 @@ +using System.Buffers.Binary; +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointManagerHopLimitTests +{ + [Fact] + public async Task Rendezvous_UsesLowTtlHolePunch() + { + var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Loopback, 4242); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 7, + receivedVia: relay, + CancellationToken.None); + + var send = Assert.Single(udp.Sends); + Assert.Equal(7, send.LocalSocketId); + Assert.Equal(endpoint, send.RemoteEndPoint); + Assert.Equal(2, send.HopLimit); + } + + private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) + { + var addressBytes = endpoint.Address.GetAddressBytes(); + var payload = new byte[1 + 5 + 2 + 1 + addressBytes.Length]; + + payload[0] = 0; + ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, 5), with.Value); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(6, 2), (ushort)endpoint.Port); + payload[8] = (byte)addressBytes.Length; + addressBytes.CopyTo(payload.AsSpan(9)); + + return payload; + } + + private sealed class RecordingUdpTransport : IZeroTierUdpTransport + { + public List Sends { get; } = new(); + + public IReadOnlyList LocalSockets { get; } = + new[] { new ZeroTierUdpLocalSocket(Id: 7, LocalEndpoint: new IPEndPoint(IPAddress.Loopback, 0)) }; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => SendAsync(localSocketId: 7, remoteEndpoint, payload, cancellationToken); + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, HopLimit: null)); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, hopLimit)); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private readonly record struct SendCall(int LocalSocketId, IPEndPoint RemoteEndPoint, ReadOnlyMemory Payload, int? HopLimit); +} diff --git a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs index 434b4ae..b569061 100644 --- a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs @@ -98,12 +98,23 @@ public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, C public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) { - Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload)); + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, HopLimit: null)); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, hopLimit)); return Task.CompletedTask; } public ValueTask DisposeAsync() => ValueTask.CompletedTask; } - private readonly record struct SendCall(int LocalSocketId, IPEndPoint RemoteEndPoint, ReadOnlyMemory Payload); + private readonly record struct SendCall(int LocalSocketId, IPEndPoint RemoteEndPoint, ReadOnlyMemory Payload, int? HopLimit); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index 0d70784..e651331 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -19,6 +19,7 @@ internal sealed class ZeroTierDataplanePeerDatagramProcessor private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly bool _multipathEnabled; private readonly Action? _handleHelloOk; + private readonly Func? _handleUnknownDirectPathAsync; public ZeroTierDataplanePeerDatagramProcessor( NodeId localNodeId, @@ -30,7 +31,8 @@ public ZeroTierDataplanePeerDatagramProcessor( ZeroTierPeerQosManager peerQos, ZeroTierPeerPathNegotiationManager peerNegotiation, bool multipathEnabled, - Action? handleHelloOk = null) + Action? handleHelloOk = null, + Func? handleUnknownDirectPathAsync = null) { ArgumentNullException.ThrowIfNull(peerSecurity); ArgumentNullException.ThrowIfNull(peerPackets); @@ -50,6 +52,7 @@ public ZeroTierDataplanePeerDatagramProcessor( _peerNegotiation = peerNegotiation; _multipathEnabled = multipathEnabled; _handleHelloOk = handleHelloOk; + _handleUnknownDirectPathAsync = handleUnknownDirectPathAsync; } public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken cancellationToken) @@ -130,17 +133,29 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c $"[zerotier] RX peer decrypted: src={peerNodeId} hop={decoded.Header.HopCount} verb={decryptedVerb} socket={datagram.LocalSocketId} via {datagram.RemoteEndPoint}."); } + var observedNewHop0Path = false; + if (_multipathEnabled) { if (decoded.Header.HopCount == 0) { - _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + observedNewHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); await _peerEcho .TrySendEchoProbeAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, key, cancellationToken) .ConfigureAwait(false); } var verb = (ZeroTierVerb)(packetBytes[ZeroTierPacketHeader.IndexVerb] & 0x1F); + + if (decoded.Header.HopCount == 0 && + observedNewHop0Path && + verb != ZeroTierVerb.Ok && + _handleUnknownDirectPathAsync is not null) + { + await _handleUnknownDirectPathAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) + .ConfigureAwait(false); + } + var payload = packetBytes.AsMemory(ZeroTierPacketHeader.IndexPayload); var payloadSpan = payload.Span; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index bbc3cd2..6dee8ab 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -210,7 +210,8 @@ public ZeroTierDataplaneRuntime( _peerQos, _peerNegotiation, multipath.Enabled, - HandlePeerHelloOk); + HandlePeerHelloOk, + HandleUnknownDirectPathAsync); _rxLoops = new ZeroTierDataplaneRxLoops( _udp, _rootNodeId, @@ -1230,6 +1231,42 @@ pending.PhysicalDestination is { } confirmedPhysicalDestination && _peerEcho.ObserveHelloOkRtt(peerNodeId, receivedLocalSocketId, receivedVia, ok.TimestampEcho); } + private async ValueTask HandleUnknownDirectPathAsync( + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + CancellationToken cancellationToken) + { + if (!_multipath.Enabled) + { + return; + } + + byte[] sharedKey; + try + { + sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is InvalidOperationException or OperationCanceledException) + { + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Confirm unknown hop-0 path: peer={peerNodeId} endpoint={remoteEndPoint} socket={localSocketId}."); + } + + await SendHelloPacketAsync( + localSocketId, + peerNodeId, + remoteEndPoint, + remoteEndPoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } + private void TrackPendingHello( NodeId peerNodeId, int localSocketId, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 4e52ad0..5aee9a9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -1,5 +1,6 @@ using System.Net; using System.Collections.Concurrent; +using System.Globalization; using System.Net.Sockets; using System.Security.Cryptography; using System.Linq; @@ -11,6 +12,7 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDirectEndpointManager { private const int MaxEndpoints = 8; + private const int RendezvousHolePunchHopLimit = 2; private const long HolePunchMinIntervalMs = 5_000; private const long HolePunchCacheTtlMs = 60_000; private const int HolePunchCacheMaxEntries = 2048; @@ -102,7 +104,10 @@ public async ValueTask HandleRendezvousFromRootAsync( foreach (var endpoint in endpoints) { - TrySendHolePunch(endpoint, preferredLocalSocketIds: new[] { receivedLocalSocketId }); + TrySendHolePunch( + endpoint, + preferredLocalSocketIds: new[] { receivedLocalSocketId }, + hopLimit: RendezvousHolePunchHopLimit); if (_handleDirectEndpointHintAsync is not null) { await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); @@ -246,7 +251,7 @@ private bool RateGatePushDirectPaths(long nowMs) } } - private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketIds = null) + private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketIds = null, int? hopLimit = null) { var localSockets = _udp.LocalSockets; var now = Environment.TickCount64; @@ -265,7 +270,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId continue; } - TrySendHolePunchCore(socketId, endpoint, junk); + TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); } return; @@ -278,7 +283,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId return; } - TrySendHolePunchCore(localSocketId: 0, endpoint, junk); + TrySendHolePunchCore(localSocketId: 0, endpoint, junk, hopLimit); return; } @@ -290,17 +295,19 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId continue; } - TrySendHolePunchCore(socketId, endpoint, junk); + TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); } } - private void TrySendHolePunchCore(int localSocketId, IPEndPoint endpoint, byte[] junk) + private void TrySendHolePunchCore(int localSocketId, IPEndPoint endpoint, byte[] junk, int? hopLimit) { Task sendTask; try { - ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint} (socket={localSocketId})."); - sendTask = _udp.SendAsync(localSocketId, endpoint, junk, CancellationToken.None); + ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint} (socket={localSocketId}, hopLimit={(hopLimit?.ToString(CultureInfo.InvariantCulture) ?? "default")})."); + sendTask = hopLimit is int ttl + ? _udp.SendWithHopLimitAsync(localSocketId, endpoint, junk, ttl, CancellationToken.None) + : _udp.SendAsync(localSocketId, endpoint, junk, CancellationToken.None); } catch (Exception ex) when (ex is ObjectDisposedException or SocketException or OperationCanceledException) { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs index 13d0803..fec306a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs @@ -21,15 +21,18 @@ public ZeroTierPeerPhysicalPathTracker(TimeSpan ttl, Func? nowUnixMs = nul _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } - public void ObserveHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + public bool ObserveHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) { ArgumentNullException.ThrowIfNull(remoteEndPoint); var now = _nowUnixMs(); var peer = _peers.GetOrAdd(peerNodeId, _ => new PeerState()); - peer.Paths[new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)] = now; + var key = new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint); + var isNewPath = !peer.Paths.ContainsKey(key); + peer.Paths[key] = now; CleanupIfNeeded(now); + return isNewPath; } public ZeroTierPeerPhysicalPath[] GetSnapshot(NodeId peerNodeId) diff --git a/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs index 07f1493..baa66e9 100644 --- a/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs @@ -13,5 +13,12 @@ internal interface IZeroTierUdpTransport : IAsyncDisposable Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default); Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default); + + Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default); } diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs index 3b94ed1..f982e28 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs @@ -73,6 +73,18 @@ public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemo return socket.SendAsync(remoteEndpoint, payload, cancellationToken); } + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + var socket = GetSocket(localSocketId); + return socket.SendWithHopLimitAsync(localSocketId, remoteEndpoint, payload, hopLimit, cancellationToken); + } + public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) != 0) diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index 28ca8d4..e65ea8b 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -15,6 +15,7 @@ internal sealed class ZeroTierUdpTransport : IZeroTierUdpTransport private readonly CancellationTokenSource _cts = new(); private readonly Task _receiverLoop; private readonly int _localSocketId; + private readonly SemaphoreSlim _sendGate = new(1, 1); private long _incomingBackpressureCount; private int _disposed; @@ -64,11 +65,26 @@ public async ValueTask ReceiveAsync( } public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => SendCoreAsync(remoteEndpoint, payload, hopLimit: null, cancellationToken); + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(remoteEndpoint); - ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); - UdpEndpointNormalization.ValidateRemoteEndpoint(remoteEndpoint, nameof(remoteEndpoint)); - return _udp.SendAsync(payload, remoteEndpoint, cancellationToken).AsTask(); + if (localSocketId != _localSocketId) + { + throw new ArgumentOutOfRangeException(nameof(localSocketId), localSocketId, $"Invalid local socket id. This transport exposes only id {_localSocketId}."); + } + + if (hopLimit <= 0) + { + throw new ArgumentOutOfRangeException(nameof(hopLimit), hopLimit, "Hop limit must be positive."); + } + + return SendCoreAsync(remoteEndpoint, payload, hopLimit, cancellationToken); } public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) @@ -78,7 +94,42 @@ public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemo throw new ArgumentOutOfRangeException(nameof(localSocketId), localSocketId, $"Invalid local socket id. This transport exposes only id {_localSocketId}."); } - return SendAsync(remoteEndpoint, payload, cancellationToken); + return SendCoreAsync(remoteEndpoint, payload, hopLimit: null, cancellationToken); + } + + private async Task SendCoreAsync( + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int? hopLimit, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(remoteEndpoint); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + UdpEndpointNormalization.ValidateRemoteEndpoint(remoteEndpoint, nameof(remoteEndpoint)); + + await _sendGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + SocketOptionState? optionState = null; + if (hopLimit is int limit) + { + optionState = TryCreateSocketOptionState(remoteEndpoint, limit); + optionState?.Apply(_udp.Client); + } + + try + { + await _udp.SendAsync(payload, remoteEndpoint, cancellationToken).ConfigureAwait(false); + } + finally + { + optionState?.Restore(_udp.Client); + } + } + finally + { + _sendGate.Release(); + } } public async ValueTask DisposeAsync() @@ -124,6 +175,7 @@ public async ValueTask DisposeAsync() finally { _cts.Dispose(); + _sendGate.Dispose(); } try @@ -198,4 +250,79 @@ private void Log(string message) { _log?.Invoke(message); } + + private SocketOptionState? TryCreateSocketOptionState(IPEndPoint remoteEndpoint, int hopLimit) + { + var socket = _udp.Client; + var addressFamily = remoteEndpoint.AddressFamily == AddressFamily.InterNetworkV6 && remoteEndpoint.Address.IsIPv4MappedToIPv6 + ? AddressFamily.InterNetwork + : remoteEndpoint.AddressFamily; + + try + { + return addressFamily switch + { + AddressFamily.InterNetwork => TryCreateSocketOptionState( + socket, + SocketOptionLevel.IP, + SocketOptionName.IpTimeToLive, + hopLimit), + AddressFamily.InterNetworkV6 => TryCreateSocketOptionState( + socket, + SocketOptionLevel.IPv6, + SocketOptionName.HopLimit, + hopLimit), + _ => null + }; + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or NotSupportedException) + { + return null; + } + } + + private static SocketOptionState? TryCreateSocketOptionState( + Socket socket, + SocketOptionLevel level, + SocketOptionName name, + int hopLimit) + { + var option = socket.GetSocketOption(level, name); + if (option is not int originalValue) + { + return null; + } + + return new SocketOptionState(level, name, originalValue, hopLimit); + } + + private sealed class SocketOptionState + { + private readonly SocketOptionLevel _level; + private readonly SocketOptionName _name; + private readonly int _originalValue; + private readonly int _temporaryValue; + + public SocketOptionState(SocketOptionLevel level, SocketOptionName name, int originalValue, int temporaryValue) + { + _level = level; + _name = name; + _originalValue = originalValue; + _temporaryValue = temporaryValue; + } + + public void Apply(Socket socket) + => socket.SetSocketOption(_level, _name, _temporaryValue); + + public void Restore(Socket socket) + { + try + { + socket.SetSocketOption(_level, _name, _originalValue); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or NotSupportedException) + { + } + } + } } From e233345709002ee2386e0fb0b5a055232bd24bc9 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:28:17 +0100 Subject: [PATCH 150/296] Reduce direct hint fanout during bootstrap --- .../Internal/ZeroTierDataplaneRuntime.cs | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 6dee8ab..e7d84f7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -691,7 +691,10 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (hinted.Length > 0) { await SendEchoDirectAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); - await SendHelloDirectAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); + if (!CanUseEchoForDirectBootstrap(peerNodeId)) + { + await SendHelloDirectAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); + } } if (unchecked(Environment.TickCount64 - deadline) >= 0) @@ -932,6 +935,11 @@ private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint } } + if (preferred.Count != 0) + { + return preferred.ToArray(); + } + for (var i = 0; i < localSockets.Count; i++) { var socketId = localSockets[i].Id; @@ -980,17 +988,23 @@ private async ValueTask HandleDirectEndpointHintAsync( { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) .ConfigureAwait(false); - await SendHelloPacketAsync( - localSocketIds[i], - peerNodeId, - endpoint, - endpoint, - sharedKey, - cancellationToken) - .ConfigureAwait(false); + if (!CanUseEchoForDirectBootstrap(peerNodeId)) + { + await SendHelloPacketAsync( + localSocketIds[i], + peerNodeId, + endpoint, + endpoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } } } + private bool CanUseEchoForDirectBootstrap(NodeId peerNodeId) + => _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId) >= 5; + private async Task TrySendEchoDirectProbeAsync( NodeId peerNodeId, int localSocketId, @@ -1219,6 +1233,11 @@ pending.PhysicalDestination is { } physicalDestination && $"[zerotier] RX OK(HELLO): peer={peerNodeId} via={receivedVia} hop={hopCount} inRe=0x{ok.InRePacketId:x16} localSocket={observedLocalSocketId} surface={surfaceText} matched={matchedPending} trustedSurface={trustedSurface} {probeText}."); } + if (!matchedPending) + { + return; + } + if (matchedPending && hopCount == 0 && pending.PhysicalDestination is { } confirmedPhysicalDestination && From e5987e82018f14dcca27bc8b098b969a8fcf1423 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:34:02 +0100 Subject: [PATCH 151/296] Filter incompatible private direct path hints --- ...TierDirectEndpointManagerPushFlagsTests.cs | 21 ++++ .../Internal/ZeroTierDataplaneRuntime.cs | 107 +++++++++++++++++- .../Internal/ZeroTierDirectEndpointManager.cs | 20 +++- .../ZeroTierDirectEndpointSelection.cs | 23 +++- 4 files changed, 165 insertions(+), 6 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 1d3d7f7..6572f81 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -32,6 +32,27 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.DoesNotContain(manager.Endpoints, ep => ep.Equals(endpoint)); } + [Fact] + public async Task PushDirectPaths_RejectedEndpoint_IsIgnored() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + shouldAcceptEndpoint: static endpoint => !endpoint.Address.ToString().StartsWith("172.", StringComparison.Ordinal)); + + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993), flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.Empty(manager.Endpoints); + } + private const byte ZtPushDirectPathsFlagForgetPath = 0x01; private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flags) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index e7d84f7..7ad37d2 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1620,7 +1620,112 @@ private ValueTask HandlePeerControlPacketAsync( private ZeroTierDirectEndpointManager GetOrCreateDirectEndpointManager(NodeId peerNodeId) { _directEndpointLastUsedMs[peerNodeId] = Environment.TickCount64; - return _directEndpoints.GetOrAdd(peerNodeId, id => new ZeroTierDirectEndpointManager(_udp, _rootEndpoint, id, HandleDirectEndpointHintAsync)); + return _directEndpoints.GetOrAdd( + peerNodeId, + id => new ZeroTierDirectEndpointManager( + _udp, + _rootEndpoint, + id, + HandleDirectEndpointHintAsync, + ShouldAcceptDirectEndpoint)); + } + + private bool ShouldAcceptDirectEndpoint(IPEndPoint endpoint) + { + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + return true; + } + + var localAdvertisements = GetLocalDirectPathAdvertisements(); + for (var i = 0; i < localAdvertisements.Length; i++) + { + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(localAdvertisements[i])) + { + continue; + } + + if (HaveCompatiblePrivateScope(localAdvertisements[i], endpoint)) + { + return true; + } + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Drop hinted private endpoint: {endpoint}."); + } + + return false; + } + + private static bool HaveCompatiblePrivateScope(IPEndPoint localEndpoint, IPEndPoint remoteEndpoint) + { + var localAddress = localEndpoint.Address; + var remoteAddress = remoteEndpoint.Address; + if (localAddress.AddressFamily == AddressFamily.InterNetworkV6 && localAddress.IsIPv4MappedToIPv6) + { + localAddress = localAddress.MapToIPv4(); + } + + if (remoteAddress.AddressFamily == AddressFamily.InterNetworkV6 && remoteAddress.IsIPv4MappedToIPv6) + { + remoteAddress = remoteAddress.MapToIPv4(); + } + + if (localAddress.AddressFamily != remoteAddress.AddressFamily) + { + return false; + } + + if (localAddress.AddressFamily == AddressFamily.InterNetwork) + { + var localBytes = localAddress.GetAddressBytes(); + var remoteBytes = remoteAddress.GetAddressBytes(); + + if (localBytes[0] == 10 && remoteBytes[0] == 10) + { + return true; + } + + if (localBytes[0] == 172 && + localBytes[1] is >= 16 and <= 31 && + remoteBytes[0] == 172 && + remoteBytes[1] is >= 16 and <= 31) + { + return true; + } + + if (localBytes[0] == 192 && + localBytes[1] == 168 && + remoteBytes[0] == 192 && + remoteBytes[1] == 168) + { + return true; + } + + if (localBytes[0] == 100 && + localBytes[1] is >= 64 and <= 127 && + remoteBytes[0] == 100 && + remoteBytes[1] is >= 64 and <= 127) + { + return true; + } + + return false; + } + + if (localAddress.AddressFamily == AddressFamily.InterNetworkV6) + { + var localBytes = localAddress.GetAddressBytes(); + var remoteBytes = remoteAddress.GetAddressBytes(); + return localBytes.Length == 16 && + remoteBytes.Length == 16 && + (localBytes[0] & 0xFE) == 0xFC && + (remoteBytes[0] & 0xFE) == 0xFC; + } + + return false; } private void CleanupDirectEndpointManagers(long nowMs) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 5aee9a9..5020a0e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -26,6 +26,7 @@ internal sealed class ZeroTierDirectEndpointManager private readonly IPEndPoint _relayEndpoint; private readonly NodeId _remoteNodeId; private readonly Func? _handleDirectEndpointHintAsync; + private readonly Func? _shouldAcceptEndpoint; private readonly object _lock = new(); private IPEndPoint[] _directEndpoints = Array.Empty(); @@ -39,7 +40,8 @@ public ZeroTierDirectEndpointManager( IZeroTierUdpTransport udp, IPEndPoint relayEndpoint, NodeId remoteNodeId, - Func? handleDirectEndpointHintAsync = null) + Func? handleDirectEndpointHintAsync = null, + Func? shouldAcceptEndpoint = null) { ArgumentNullException.ThrowIfNull(udp); ArgumentNullException.ThrowIfNull(relayEndpoint); @@ -48,6 +50,7 @@ public ZeroTierDirectEndpointManager( _relayEndpoint = relayEndpoint; _remoteNodeId = remoteNodeId; _handleDirectEndpointHintAsync = handleDirectEndpointHintAsync; + _shouldAcceptEndpoint = shouldAcceptEndpoint; } public IPEndPoint[] Endpoints => _directEndpoints; @@ -90,7 +93,11 @@ public async ValueTask HandleRendezvousFromRootAsync( if (ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) && rendezvous.With == _remoteNodeId) { - var endpoints = ZeroTierDirectEndpointSelection.Normalize([rendezvous.Endpoint], _relayEndpoint, maxEndpoints: MaxEndpoints); + var endpoints = ZeroTierDirectEndpointSelection.Normalize( + [rendezvous.Endpoint], + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint); if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); @@ -179,7 +186,11 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( .Concat(redirect) .Concat(add); - endpoints = ZeroTierDirectEndpointSelection.Normalize(merged, _relayEndpoint, maxEndpoints: MaxEndpoints); + endpoints = ZeroTierDirectEndpointSelection.Normalize( + merged, + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint); _directEndpoints = endpoints; PrunePreferredLocalSockets_NoLock(endpoints); RememberPreferredLocalSockets_NoLock(redirect.Concat(add), receivedLocalSocketId); @@ -217,7 +228,8 @@ public void SeedEndpoints(IEnumerable endpoints) normalized = ZeroTierDirectEndpointSelection.Normalize( _directEndpoints.Concat(endpoints), _relayEndpoint, - maxEndpoints: MaxEndpoints); + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint); _directEndpoints = normalized; PrunePreferredLocalSockets_NoLock(normalized); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs index 3e41d17..23728ec 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs @@ -6,7 +6,11 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierDirectEndpointSelection { - public static IPEndPoint[] Normalize(IEnumerable endpoints, IPEndPoint relayEndpoint, int maxEndpoints) + public static IPEndPoint[] Normalize( + IEnumerable endpoints, + IPEndPoint relayEndpoint, + int maxEndpoints, + Func? shouldInclude = null) { ArgumentNullException.ThrowIfNull(endpoints); ArgumentNullException.ThrowIfNull(relayEndpoint); @@ -31,6 +35,11 @@ public static IPEndPoint[] Normalize(IEnumerable endpoints, IPEndPoi continue; } + if (shouldInclude is not null && !shouldInclude(canonical)) + { + continue; + } + if (canonical.Equals(relayEndpoint)) { continue; @@ -97,6 +106,18 @@ private static IPEndPoint Canonicalize(IPEndPoint endpoint) return endpoint; } + public static bool IsPublicEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + return IsPublicAddress(Canonicalize(endpoint).Address); + } + + public static bool IsPrivateEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + return !IsPublicEndpoint(endpoint); + } + private static bool IsPublicAddress(IPAddress address) { if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) From 3bb95b813e813ac3eb723e18ce3641b8f38443e9 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:39:02 +0100 Subject: [PATCH 152/296] Avoid duplicate direct hint rebootstrap --- ...TierDirectEndpointManagerPushFlagsTests.cs | 59 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 5 ++ .../Internal/ZeroTierDirectEndpointManager.cs | 19 +++++- 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 6572f81..c5a91a8 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -53,6 +53,34 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Empty(manager.Endpoints); } + [Fact] + public async Task PushDirectPaths_KnownEndpoint_IsNotRebootstrapped() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new List(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, _, endpoint, _) => + { + hintedEndpoints.Add(endpoint); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + var payload = BuildPushDirectPathsPayload(endpoint, flags: 0); + + await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); + + Assert.Single(hintedEndpoints); + Assert.Single(udp.Sends); + } + private const byte ZtPushDirectPathsFlagForgetPath = 0x01; private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flags) @@ -87,5 +115,36 @@ private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flag return payload; } + + private sealed class RecordingUdpTransport : IZeroTierUdpTransport + { + public List<(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit)> Sends { get; } = new(); + + public IReadOnlyList LocalSockets { get; } = + new[] { new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Loopback, 0)) }; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => SendAsync(0, remoteEndpoint, payload, cancellationToken); + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + Sends.Add((localSocketId, remoteEndpoint, null)); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) + { + Sends.Add((localSocketId, remoteEndpoint, hopLimit)); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 7ad37d2..bbec0ae 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1632,6 +1632,11 @@ private ZeroTierDirectEndpointManager GetOrCreateDirectEndpointManager(NodeId pe private bool ShouldAcceptDirectEndpoint(IPEndPoint endpoint) { + if (IPAddress.IsLoopback(endpoint.Address)) + { + return true; + } + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) { return true; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 5020a0e..3e2f740 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -155,6 +155,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( var forget = new HashSet(StringComparer.Ordinal); var redirect = new List(); var add = new List(); + var redirectKeys = new HashSet(StringComparer.Ordinal); for (var i = 0; i < paths.Length; i++) { @@ -171,6 +172,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( if ((flags & PushDirectPathsFlagClusterRedirect) != 0) { redirect.Add(endpoint); + redirectKeys.Add(key); } else { @@ -179,8 +181,12 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( } IPEndPoint[] endpoints; + IPEndPoint[] endpointsToProbe; lock (_lock) { + var previousKeys = _directEndpoints + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); var merged = _directEndpoints .Where(ep => !forget.Contains(FormatEndpointKey(ep))) .Concat(redirect) @@ -191,12 +197,19 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( _relayEndpoint, maxEndpoints: MaxEndpoints, _shouldAcceptEndpoint); + endpointsToProbe = endpoints + .Where(endpoint => + { + var key = FormatEndpointKey(endpoint); + return redirectKeys.Contains(key) || !previousKeys.Contains(key); + }) + .ToArray(); _directEndpoints = endpoints; PrunePreferredLocalSockets_NoLock(endpoints); - RememberPreferredLocalSockets_NoLock(redirect.Concat(add), receivedLocalSocketId); + RememberPreferredLocalSockets_NoLock(endpointsToProbe, receivedLocalSocketId); } - if (endpoints.Length == 0) + if (endpoints.Length == 0 || endpointsToProbe.Length == 0) { return; } @@ -206,7 +219,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( ZeroTierTrace.WriteLine($"[zerotier] RX PUSH_DIRECT_PATHS: endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} (candidates: {paths.Length})."); } - foreach (var endpoint in endpoints) + foreach (var endpoint in endpointsToProbe) { TrySendHolePunch(endpoint, preferredLocalSocketIds: new[] { receivedLocalSocketId }); if (_handleDirectEndpointHintAsync is not null) From 3b80ce23fc869123cb28912655be3987ae7234a0 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:47:25 +0100 Subject: [PATCH 153/296] Tune direct bootstrap probe cadence --- .../DirectBootstrapProbeCursorTests.cs | 37 +++++ .../Internal/ZeroTierDataplaneRuntime.cs | 137 ++++++++++++++++-- .../Internal/ZeroTierDirectEndpointManager.cs | 6 +- 3 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 ZTSharp.Tests/DirectBootstrapProbeCursorTests.cs diff --git a/ZTSharp.Tests/DirectBootstrapProbeCursorTests.cs b/ZTSharp.Tests/DirectBootstrapProbeCursorTests.cs new file mode 100644 index 0000000..31f8983 --- /dev/null +++ b/ZTSharp.Tests/DirectBootstrapProbeCursorTests.cs @@ -0,0 +1,37 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class DirectBootstrapProbeCursorTests +{ + [Fact] + public void TakeNextEndpoints_RotatesAcrossCandidates() + { + var cursor = new DirectBootstrapProbeCursor(); + var endpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 59827) + }; + + var first = cursor.TakeNextEndpoints(endpoints, budget: 2); + var second = cursor.TakeNextEndpoints(endpoints, budget: 2); + + Assert.Equal(new[] { endpoints[0], endpoints[1] }, first); + Assert.Equal(new[] { endpoints[2], endpoints[0] }, second); + } + + [Fact] + public void TakeNextSocketIndex_RotatesPerEndpoint() + { + var cursor = new DirectBootstrapProbeCursor(); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + + Assert.Equal(0, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + Assert.Equal(1, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + Assert.Equal(2, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + Assert.Equal(0, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index bbec0ae..68b0606 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -17,6 +17,11 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const long DirectEndpointManagerTtlMs = 600_000; private const long NetworkCredentialsBootstrapIntervalMs = 5_000; private const long DirectHelloMinIntervalMs = 1_000; + private const long DirectBootstrapRootHelloIntervalMs = 5_000; + private const long DirectBootstrapPathPushIntervalMs = 15_000; + private const long DirectBootstrapPathPushIntervalHavePathMs = 120_000; + private const long DirectBootstrapHintProbeIntervalMs = 1_000; + private const int DirectBootstrapHintProbeBudget = 2; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -44,6 +49,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _directBootstrapTasks = new(); + private readonly ConcurrentDictionary _directBootstrapProbeCursors = new(); private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); private readonly ConcurrentDictionary _pendingHelloProbes = new(); @@ -672,29 +678,34 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared ? TimeSpan.FromSeconds(5) : TimeSpan.FromSeconds(35); var deadline = Environment.TickCount64 + (long)bootstrapTimeout.TotalMilliseconds; - var nextRootHelloAt = Environment.TickCount64; + var now = Environment.TickCount64; + var nextRootHelloAt = now; + var nextDirectPathPushAt = now; + var nextHintProbeAt = now; while (!HasSufficientDirectPath(peerNodeId)) { cancellationToken.ThrowIfCancellationRequested(); - var now = Environment.TickCount64; + now = Environment.TickCount64; if (unchecked(now - nextRootHelloAt) >= 0) { await SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); await SendNetworkCredentialsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + nextRootHelloAt = now + DirectBootstrapRootHelloIntervalMs; + } + + if (unchecked(now - nextDirectPathPushAt) >= 0) + { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - nextRootHelloAt = now + (long)TimeSpan.FromSeconds(1).TotalMilliseconds; + nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; - if (hinted.Length > 0) + if (hinted.Length > 0 && unchecked(now - nextHintProbeAt) >= 0) { - await SendEchoDirectAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); - if (!CanUseEchoForDirectBootstrap(peerNodeId)) - { - await SendHelloDirectAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); - } + await ProbeHintedDirectEndpointsAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); + nextHintProbeAt = now + DirectBootstrapHintProbeIntervalMs; } if (unchecked(Environment.TickCount64 - deadline) >= 0) @@ -706,6 +717,46 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared } } + private long GetDirectPathPushBootstrapIntervalMs(NodeId peerNodeId) + => HasConfirmedDirectPath(peerNodeId) + ? DirectBootstrapPathPushIntervalHavePathMs + : DirectBootstrapPathPushIntervalMs; + + private async Task ProbeHintedDirectEndpointsAsync( + NodeId peerNodeId, + IPEndPoint[] hinted, + byte[] sharedKey, + CancellationToken cancellationToken) + { + if (hinted.Length == 0) + { + return; + } + + var cursor = _directBootstrapProbeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); + var endpointsToProbe = cursor.TakeNextEndpoints(hinted, DirectBootstrapHintProbeBudget); + for (var i = 0; i < endpointsToProbe.Length; i++) + { + var localSocketIds = GetRotatingDirectBootstrapSocketIds(peerNodeId, endpointsToProbe[i], cursor); + for (var s = 0; s < localSocketIds.Length; s++) + { + await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpointsToProbe[i], sharedKey, cancellationToken) + .ConfigureAwait(false); + if (!CanUseEchoForDirectBootstrap(peerNodeId)) + { + await SendHelloPacketAsync( + localSocketIds[s], + peerNodeId, + endpointsToProbe[i], + endpointsToProbe[i], + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } + } + } + } + private bool HasDirectPathCandidate(NodeId peerNodeId) => _peerPaths.GetSnapshot(peerNodeId).Length != 0 || GetOrCreateDirectEndpointManager(peerNodeId).Endpoints.Length != 0; @@ -957,6 +1008,21 @@ private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint return localSockets.Select(static socket => socket.Id).ToArray(); } + private int[] GetRotatingDirectBootstrapSocketIds( + NodeId peerNodeId, + IPEndPoint endpoint, + DirectBootstrapProbeCursor cursor) + { + var preferred = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoint); + if (preferred.Length <= 1) + { + return preferred; + } + + var socketIndex = cursor.TakeNextSocketIndex(endpoint, preferred.Length); + return new[] { preferred[socketIndex] }; + } + private async ValueTask HandleDirectEndpointHintAsync( NodeId peerNodeId, int receivedLocalSocketId, @@ -983,7 +1049,8 @@ private async ValueTask HandleDirectEndpointHintAsync( ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId}."); } - var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoint); + var cursor = _directBootstrapProbeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); + var localSocketIds = GetRotatingDirectBootstrapSocketIds(peerNodeId, endpoint, cursor); for (var i = 0; i < localSocketIds.Length; i++) { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) @@ -1752,12 +1819,62 @@ private void CleanupDirectEndpointManagers(long nowMs) { _directEndpointLastUsedMs.TryRemove(pair.Key, out _); _directEndpoints.TryRemove(pair.Key, out _); + _directBootstrapProbeCursors.TryRemove(pair.Key, out _); } } } } +internal sealed class DirectBootstrapProbeCursor +{ + private readonly object _lock = new(); + private int _nextEndpointIndex; + private readonly Dictionary _nextSocketIndexByEndpoint = new(StringComparer.Ordinal); + + public IPEndPoint[] TakeNextEndpoints(IPEndPoint[] endpoints, int budget) + { + ArgumentNullException.ThrowIfNull(endpoints); + + if (endpoints.Length == 0 || budget <= 0) + { + return Array.Empty(); + } + + lock (_lock) + { + var count = Math.Min(budget, endpoints.Length); + var selected = new IPEndPoint[count]; + for (var i = 0; i < count; i++) + { + var index = (_nextEndpointIndex + i) % endpoints.Length; + selected[i] = endpoints[index]; + } + + _nextEndpointIndex = (_nextEndpointIndex + count) % endpoints.Length; + return selected; + } + } + + public int TakeNextSocketIndex(IPEndPoint endpoint, int socketCount) + { + ArgumentNullException.ThrowIfNull(endpoint); + + if (socketCount <= 1) + { + return 0; + } + + lock (_lock) + { + var key = endpoint.ToString(); + _nextSocketIndexByEndpoint.TryGetValue(key, out var nextIndex); + _nextSocketIndexByEndpoint[key] = (nextIndex + 1) % socketCount; + return nextIndex % socketCount; + } + } +} + internal readonly record struct PendingHelloProbe( NodeId PeerNodeId, int LocalSocketId, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 3e2f740..b9b70dd 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -182,6 +182,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( IPEndPoint[] endpoints; IPEndPoint[] endpointsToProbe; + IPEndPoint[] preferredSocketEndpoints; lock (_lock) { var previousKeys = _directEndpoints @@ -204,9 +205,12 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( return redirectKeys.Contains(key) || !previousKeys.Contains(key); }) .ToArray(); + preferredSocketEndpoints = endpointsToProbe + .Where(endpoint => redirectKeys.Contains(FormatEndpointKey(endpoint))) + .ToArray(); _directEndpoints = endpoints; PrunePreferredLocalSockets_NoLock(endpoints); - RememberPreferredLocalSockets_NoLock(endpointsToProbe, receivedLocalSocketId); + RememberPreferredLocalSockets_NoLock(preferredSocketEndpoints, receivedLocalSocketId); } if (endpoints.Length == 0 || endpointsToProbe.Length == 0) From d93578d396baa3dc89a2dc783549fd3b5327495e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:55:05 +0100 Subject: [PATCH 154/296] Refine push-direct bootstrap policy --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 13 +++--- ...TierDirectEndpointManagerPushFlagsTests.cs | 33 +++++++++++---- .../Internal/ZeroTierDataplaneRuntime.cs | 41 ++++++++----------- .../Internal/ZeroTierDirectEndpointManager.cs | 1 - 4 files changed, 50 insertions(+), 38 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 1c61b7a..46958de 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -1,6 +1,7 @@ using System.Buffers.Binary; using System.Net; using System.Security.Cryptography; +using ZTSharp.ZeroTier; using ZTSharp.ZeroTier.Internal; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -56,7 +57,7 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() } [Fact] - public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsHolePunch() + public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() { await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); @@ -81,7 +82,8 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsHolePunch() networkId: 0x9ad07d01093a69e3UL, localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, localManagedIpsV6: Array.Empty(), - inlineCom: Array.Empty()); + inlineCom: Array.Empty(), + multipath: new ZeroTierMultipathOptions { Enabled = true }); var runtimeEndpoint = TestUdpEndpoints.ToLoopback(runtime.LocalUdp); @@ -127,8 +129,9 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsHolePunch() await rootUdp.SendAsync(runtimeEndpoint, pushPacket); - var holePunch = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - Assert.Equal(4, holePunch.Payload.Length); + var probe = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); + Assert.True(probe.Payload.Length > 4); + Assert.True(ZeroTierPacketCodec.TryDecode(probe.Payload, out _)); } private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) @@ -177,4 +180,4 @@ private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint) return payload; } -} +} \ No newline at end of file diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index c5a91a8..b0e16f3 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -12,16 +12,17 @@ public sealed class ZeroTierDirectEndpointManagerPushFlagsTests public async Task PushDirectPaths_ForgetFlag_RemovesEndpoint() { await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var receiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); var relay = new IPEndPoint(IPAddress.Loopback, 9999); var peerNodeId = new NodeId(0x1111111111); var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); - var endpoint = TestUdpEndpoints.ToLoopback(receiver.LocalEndpoint); - await manager.HandlePushDirectPathsFromRemoteAsync(BuildPushDirectPathsPayload(endpoint, flags: 0), receivedLocalSocketId: 0, CancellationToken.None); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); - _ = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); Assert.Contains(manager.Endpoints, ep => ep.Equals(endpoint)); await manager.HandlePushDirectPathsFromRemoteAsync( @@ -78,7 +79,26 @@ public async Task PushDirectPaths_KnownEndpoint_IsNotRebootstrapped() await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); Assert.Single(hintedEndpoints); - Assert.Single(udp.Sends); + Assert.Empty(udp.Sends); + } + + [Fact] + public async Task PushDirectPaths_NewEndpoint_DoesNotRememberReceivingSocketAffinity() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(udp.Sends); } private const byte ZtPushDirectPathsFlagForgetPath = 0x01; @@ -146,5 +166,4 @@ public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, public ValueTask DisposeAsync() => ValueTask.CompletedTask; } -} - +} \ No newline at end of file diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 68b0606..d045aa7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -742,17 +742,14 @@ private async Task ProbeHintedDirectEndpointsAsync( { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpointsToProbe[i], sharedKey, cancellationToken) .ConfigureAwait(false); - if (!CanUseEchoForDirectBootstrap(peerNodeId)) - { - await SendHelloPacketAsync( - localSocketIds[s], - peerNodeId, - endpointsToProbe[i], - endpointsToProbe[i], - sharedKey, - cancellationToken) - .ConfigureAwait(false); - } + await SendHelloPacketAsync( + localSocketIds[s], + peerNodeId, + endpointsToProbe[i], + endpointsToProbe[i], + sharedKey, + cancellationToken) + .ConfigureAwait(false); } } } @@ -1055,23 +1052,17 @@ private async ValueTask HandleDirectEndpointHintAsync( { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) .ConfigureAwait(false); - if (!CanUseEchoForDirectBootstrap(peerNodeId)) - { - await SendHelloPacketAsync( - localSocketIds[i], - peerNodeId, - endpoint, - endpoint, - sharedKey, - cancellationToken) - .ConfigureAwait(false); - } + await SendHelloPacketAsync( + localSocketIds[i], + peerNodeId, + endpoint, + endpoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); } } - private bool CanUseEchoForDirectBootstrap(NodeId peerNodeId) - => _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId) >= 5; - private async Task TrySendEchoDirectProbeAsync( NodeId peerNodeId, int localSocketId, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index b9b70dd..2c25720 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -225,7 +225,6 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( foreach (var endpoint in endpointsToProbe) { - TrySendHolePunch(endpoint, preferredLocalSocketIds: new[] { receivedLocalSocketId }); if (_handleDirectEndpointHintAsync is not null) { await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); From 40cb39d891e805e77bc55793e5ca584bd4965cd2 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:05:59 +0100 Subject: [PATCH 155/296] Support upstream empty echo direct probes --- ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs | 44 +++++++++++++++-- .../Internal/ZeroTierDataplaneRuntime.cs | 47 ++++++++++++------- .../Internal/ZeroTierPeerEchoManager.cs | 24 ++-------- 3 files changed, 75 insertions(+), 40 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs index b569061..b6b6940 100644 --- a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs @@ -26,12 +26,16 @@ public async Task EchoProbe_UsesOnWirePacketId_AndUpdatesRttOnOk() Assert.Single(udp.Sends); Assert.Equal(1, udp.Sends[0].LocalSocketId); Assert.Equal(endpoint, udp.Sends[0].RemoteEndPoint); + var sentBytes = udp.Sends[0].Payload.ToArray(); + Assert.True(ZeroTierPacketCrypto.Dearmor(sentBytes, sharedKey)); + Assert.True(ZeroTierPacketCodec.TryDecode(sentBytes, out var sent)); + Assert.Equal(ZeroTierVerb.Echo, sent.Header.Verb); + Assert.Empty(sent.Payload.ToArray()); var onWireEchoPacketId = BinaryPrimitives.ReadUInt64BigEndian(udp.Sends[0].Payload.Span.Slice(0, 8)); now += 25; - Span okTail = stackalloc byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(okTail, 1_000UL); + Span okTail = stackalloc byte[0]; mgr.HandleEchoOk(peerNodeId, localSocketId: 1, endpoint, onWireEchoPacketId, okTail); Assert.True(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, endpoint, out var rttMs)); @@ -42,7 +46,7 @@ public async Task EchoProbe_UsesOnWirePacketId_AndUpdatesRttOnOk() } [Fact] - public async Task HandleEchoRequest_SendsOkEchoWithTimestampEcho() + public async Task HandleEchoRequest_SendsOkEchoWithPayloadPassthrough() { var udp = new RecordingUdpTransport(); var localNodeId = new NodeId(0x2222222222); @@ -74,12 +78,44 @@ await mgr.HandleEchoRequestAsync( Assert.Equal(ZeroTierVerb.Ok, verb); var payload = decrypted.AsSpan(ZeroTierPacketHeader.IndexPayload); - Assert.True(payload.Length >= 1 + 8 + 8); + Assert.True(payload.Length >= 1 + 8); Assert.Equal(ZeroTierVerb.Echo, (ZeroTierVerb)(payload[0] & 0x1F)); Assert.Equal(123UL, BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(1, 8))); Assert.Equal(777UL, BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(1 + 8, 8))); } + [Fact] + public async Task HandleEchoRequest_SendsOkEchoForEmptyPayload() + { + var udp = new RecordingUdpTransport(); + var localNodeId = new NodeId(0x2222222222); + var peerNodeId = new NodeId(0x1111111111); + var mgr = new ZeroTierPeerEchoManager(udp, localNodeId, getPeerProtocolVersion: _ => 12, nowUnixMs: () => 1_000); + + var endpoint = new IPEndPoint(IPAddress.Loopback, 5555); + var sharedKey = Enumerable.Repeat((byte)7, 48).ToArray(); + + await mgr.HandleEchoRequestAsync( + peerNodeId, + localSocketId: 2, + endpoint, + inRePacketId: 123, + ReadOnlyMemory.Empty, + sharedKey, + CancellationToken.None); + + Assert.Single(udp.Sends); + + var decrypted = udp.Sends[0].Payload.ToArray(); + Assert.True(ZeroTierPacketCrypto.Dearmor(decrypted, sharedKey)); + + var payload = decrypted.AsSpan(ZeroTierPacketHeader.IndexPayload); + Assert.True(payload.Length >= 1 + 8); + Assert.Equal(ZeroTierVerb.Echo, (ZeroTierVerb)(payload[0] & 0x1F)); + Assert.Equal(123UL, BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(1, 8))); + Assert.Equal(1 + 8, payload.Length); + } + private sealed class RecordingUdpTransport : IZeroTierUdpTransport { public List Sends { get; } = new(); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d045aa7..61833d3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -742,14 +742,17 @@ private async Task ProbeHintedDirectEndpointsAsync( { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpointsToProbe[i], sharedKey, cancellationToken) .ConfigureAwait(false); - await SendHelloPacketAsync( - localSocketIds[s], - peerNodeId, - endpointsToProbe[i], - endpointsToProbe[i], - sharedKey, - cancellationToken) - .ConfigureAwait(false); + if (!CanUseEchoForDirectBootstrap(peerNodeId)) + { + await SendHelloPacketAsync( + localSocketIds[s], + peerNodeId, + endpointsToProbe[i], + endpointsToProbe[i], + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } } } } @@ -1038,6 +1041,12 @@ private async ValueTask HandleDirectEndpointHintAsync( } catch (Exception ex) when (ex is InvalidOperationException or OperationCanceledException) { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap skipped: peer={peerNodeId} endpoint={endpoint} reason={ex.GetType().Name}: {ex.Message}"); + } + return; } @@ -1052,17 +1061,23 @@ private async ValueTask HandleDirectEndpointHintAsync( { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) .ConfigureAwait(false); - await SendHelloPacketAsync( - localSocketIds[i], - peerNodeId, - endpoint, - endpoint, - sharedKey, - cancellationToken) - .ConfigureAwait(false); + if (!CanUseEchoForDirectBootstrap(peerNodeId)) + { + await SendHelloPacketAsync( + localSocketIds[i], + peerNodeId, + endpoint, + endpoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } } } + private bool CanUseEchoForDirectBootstrap(NodeId peerNodeId) + => _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId) >= 5; + private async Task TrySendEchoDirectProbeAsync( NodeId peerNodeId, int localSocketId, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs index c04d7f4..d300431 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs @@ -91,9 +91,6 @@ public async ValueTask TrySendEchoProbeAsync( _lastEchoSentUnixMs[pathKey] = now; - var payload = new byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(payload, (ulong)now); - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); var header = new ZeroTierPacketHeader( PacketId: packetId, @@ -103,7 +100,7 @@ public async ValueTask TrySendEchoProbeAsync( Mac: 0, VerbRaw: (byte)ZeroTierVerb.Echo); - var packet = ZeroTierPacketCodec.Encode(header, payload); + var packet = ZeroTierPacketCodec.Encode(header, ReadOnlySpan.Empty); var peerProtocolVersion = _getPeerProtocolVersion(peerNodeId); ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, peerProtocolVersion), encryptPayload: true); var onWirePacketId = BinaryPrimitives.ReadUInt64BigEndian(packet.AsSpan(0, 8)); @@ -134,17 +131,10 @@ public async ValueTask HandleEchoRequestAsync( cancellationToken.ThrowIfCancellationRequested(); var payloadSpan = payload.Span; - if (payloadSpan.Length < 8) - { - return; - } - - var timestampEcho = BinaryPrimitives.ReadUInt64BigEndian(payloadSpan.Slice(0, 8)); - - var okPayload = new byte[1 + 8 + 8]; + var okPayload = new byte[1 + 8 + payloadSpan.Length]; okPayload[0] = (byte)ZeroTierVerb.Echo; BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1, 8), inRePacketId); - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1 + 8, 8), timestampEcho); + payloadSpan.CopyTo(okPayload.AsSpan(1 + 8)); var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); var header = new ZeroTierPacketHeader( @@ -182,14 +172,8 @@ public void HandleEchoOk( return; } - if (okPayloadTail.Length < 8) - { - return; - } - - var timestampEcho = BinaryPrimitives.ReadUInt64BigEndian(okPayloadTail.Slice(0, 8)); var now = _nowUnixMs(); - var rtt = unchecked((long)now - (long)timestampEcho); + var rtt = unchecked(now - pending.TimestampUnixMs); if (rtt < 0 || rtt > int.MaxValue) { return; From 273014045311b32ac3906bf02ebdeed3f1502b0b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:12:57 +0100 Subject: [PATCH 156/296] Keep hinted direct paths alive during maintenance --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 83 ++++++++++++- .../Internal/ZeroTierDataplaneRuntime.cs | 116 +++++++++++++++++- 2 files changed, 192 insertions(+), 7 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 46958de..4e5a058 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -134,6 +134,87 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() Assert.True(ZeroTierPacketCodec.TryDecode(probe.Payload, out _)); } + [Fact] + public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var punchReceiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = new byte[48]; + RandomNumberGenerator.Fill(rootKey); + + await using var runtime = new ZeroTierDataplaneRuntime( + udp, + rootNodeId: rootNodeId, + rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), + rootKey: rootKey, + rootProtocolVersion: 12, + localIdentity, + networkId: 0x9ad07d01093a69e3UL, + localManagedIpsV4: Array.Empty(), + localManagedIpsV6: Array.Empty(), + inlineCom: Array.Empty(), + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var planet = new ZeroTierWorld( + id: 1, + type: ZeroTierWorldType.Planet, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: Array.Empty()); + + var runtimeEndpoint = TestUdpEndpoints.ToLoopback(udp.LocalEndpoint); + var helloPacket = ZeroTierHelloPacketBuilder.BuildPacket( + peerIdentity, + destination: localIdentity.NodeId, + physicalDestination: runtimeEndpoint, + planet, + timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + sharedKey, + advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, + advertisedMajorVersion: ZeroTierHelloClient.AdvertisedMajorVersion, + advertisedMinorVersion: ZeroTierHelloClient.AdvertisedMinorVersion, + advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, + out _); + + await rootUdp.SendAsync(runtimeEndpoint, helloPacket); + _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + + var hintedEndpoint = TestUdpEndpoints.ToLoopback(punchReceiver.LocalEndpoint); + var pushPayload = BuildPushDirectPathsPayload(hintedEndpoint); + var pushPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 2, + Destination: localIdentity.NodeId, + Source: peerIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.PushDirectPaths), + pushPayload); + + ZeroTierPacketCrypto.Armor(pushPacket, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); + await rootUdp.SendAsync(runtimeEndpoint, pushPacket); + _ = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); + + var candidates = runtime.GetHintedDirectCandidatesForMaintenance(peerIdentity.NodeId); + var candidate = Assert.Single(candidates); + Assert.Equal(hintedEndpoint, candidate.RemoteEndPoint); + Assert.Equal(0, candidate.LocalSocketId); + } + private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) { var addressBytes = endpoint.Address.GetAddressBytes(); @@ -180,4 +261,4 @@ private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint) return payload; } -} \ No newline at end of file +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 61833d3..98a8315 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -22,6 +22,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const long DirectBootstrapPathPushIntervalHavePathMs = 120_000; private const long DirectBootstrapHintProbeIntervalMs = 1_000; private const int DirectBootstrapHintProbeBudget = 2; + private const long DirectHintFullHelloIntervalMs = 60_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -676,7 +677,7 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared var bootstrapTimeout = _multipath.AllowRootRelayFallback ? TimeSpan.FromSeconds(5) - : TimeSpan.FromSeconds(35); + : TimeSpan.FromSeconds(75); var deadline = Environment.TickCount64 + (long)bootstrapTimeout.TotalMilliseconds; var now = Environment.TickCount64; var nextRootHelloAt = now; @@ -1111,9 +1112,10 @@ private async Task SendHelloPacketAsync( IPEndPoint? physicalDestination, IPEndPoint sendTo, byte[] sharedKey, + long minIntervalMs, CancellationToken cancellationToken) { - if (physicalDestination is not null && !ShouldSendDirectHello(peerNodeId, localSocketId, sendTo)) + if (physicalDestination is not null && !ShouldSendDirectHello(peerNodeId, localSocketId, sendTo, minIntervalMs)) { return; } @@ -1152,7 +1154,23 @@ private async Task SendHelloPacketAsync( } } - private bool ShouldSendDirectHello(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + private Task SendHelloPacketAsync( + int localSocketId, + NodeId peerNodeId, + IPEndPoint? physicalDestination, + IPEndPoint sendTo, + byte[] sharedKey, + CancellationToken cancellationToken) + => SendHelloPacketAsync( + localSocketId, + peerNodeId, + physicalDestination, + sendTo, + sharedKey, + DirectHelloMinIntervalMs, + cancellationToken); + + private bool ShouldSendDirectHello(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, long minIntervalMs) { var endpoint = remoteEndPoint.Address.IsIPv4MappedToIPv6 ? new IPEndPoint(remoteEndPoint.Address.MapToIPv4(), remoteEndPoint.Port) @@ -1163,7 +1181,7 @@ private bool ShouldSendDirectHello(NodeId peerNodeId, int localSocketId, IPEndPo while (true) { if (_lastDirectHelloSentMs.TryGetValue(key, out var lastSent) && - unchecked(nowMs - lastSent) < DirectHelloMinIntervalMs) + unchecked(nowMs - lastSent) < minIntervalMs) { return false; } @@ -1441,7 +1459,7 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati _bondEngine.MaintenanceTick(); CleanupDirectEndpointManagers(nowMs); - var peers = _peerPaths.GetPeersSnapshot(); + var peers = GetPeersForMultipathMaintenance(); if (peers.Length == 0) { return; @@ -1457,11 +1475,31 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); var paths = _peerPaths.GetSnapshot(peerNodeId); - if (paths.Length == 0) + var hintedCandidates = GetHintedDirectCandidates(peerNodeId, paths); + if (paths.Length == 0 && hintedCandidates.Length == 0) { continue; } + for (var c = 0; c < hintedCandidates.Length; c++) + { + var candidate = hintedCandidates[c]; + + await _peerEcho + .TrySendEchoProbeAsync(peerNodeId, candidate.LocalSocketId, candidate.RemoteEndPoint, key, cancellationToken) + .ConfigureAwait(false); + + await SendHelloPacketAsync( + candidate.LocalSocketId, + peerNodeId, + candidate.RemoteEndPoint, + candidate.RemoteEndPoint, + key, + DirectHintFullHelloIntervalMs, + cancellationToken) + .ConfigureAwait(false); + } + for (var p = 0; p < paths.Length; p++) { var path = paths[p]; @@ -1830,6 +1868,72 @@ private void CleanupDirectEndpointManagers(long nowMs) } } + internal ZeroTierSelectedPeerPath[] GetHintedDirectCandidatesForMaintenance(NodeId peerNodeId) + => GetHintedDirectCandidates(peerNodeId, _peerPaths.GetSnapshot(peerNodeId)); + + private NodeId[] GetPeersForMultipathMaintenance() + { + var peers = _peerPaths.GetPeersSnapshot(); + if (_directEndpoints.IsEmpty) + { + return peers; + } + + var set = peers.ToHashSet(); + foreach (var peerNodeId in _directEndpoints.Keys) + { + set.Add(peerNodeId); + } + + return set.ToArray(); + } + + private ZeroTierSelectedPeerPath[] GetHintedDirectCandidates(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths) + { + if (!_directEndpoints.TryGetValue(peerNodeId, out var directEndpoints)) + { + return Array.Empty(); + } + + var hinted = directEndpoints.Endpoints; + if (hinted.Length == 0) + { + return Array.Empty(); + } + + HashSet? observed = null; + if (observedPaths.Length > 0) + { + observed = observedPaths + .Select(static path => new ZeroTierPeerPhysicalPathKey(path.LocalSocketId, path.RemoteEndPoint)) + .ToHashSet(); + } + + var unique = new HashSet(); + var candidates = new List(hinted.Length); + for (var i = 0; i < hinted.Length; i++) + { + var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, hinted[i]); + for (var s = 0; s < localSocketIds.Length; s++) + { + var key = new ZeroTierPeerPhysicalPathKey(localSocketIds[s], hinted[i]); + if (observed is not null && observed.Contains(key)) + { + continue; + } + + if (!unique.Add(key)) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(localSocketIds[s], hinted[i])); + } + } + + return candidates.ToArray(); + } + } internal sealed class DirectBootstrapProbeCursor From 4f18e903493dd56fef61e0a918a05311e160bb64 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:18:06 +0100 Subject: [PATCH 157/296] Refactor internal classes for improved organization and maintainability --- .../ZeroTierSizeCapHardeningTests.cs | 1 + .../Internal/ZeroTierDataplanePeerSecurity.cs | 39 +++++++++--- .../ZeroTierDataplaneRuntimeFactory.cs | 62 ++++++++++++++++++- .../Internal/ZeroTierHelloPacketBuilder.cs | 8 +-- .../Protocol/ZeroTierInetAddressCodec.cs | 20 +++++- 5 files changed, 112 insertions(+), 18 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs b/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs index 3068708..7cc21be 100644 --- a/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs +++ b/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs @@ -57,6 +57,7 @@ public async Task PeerSecurity_DropsOversizedHello_WithoutCachingKeys() var packetBytes = new byte[ZeroTierProtocolLimits.MaxPacketBytes + 1]; await peerSecurity.HandleHelloAsync( peerNodeId: new NodeId(0xaaaaaaaaaa), + localSocketId: 0, helloPacketId: 1, packetBytes: packetBytes, remoteEndPoint: new IPEndPoint(IPAddress.Loopback, 12345), diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs index 40a4b52..2dbfdbd 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs @@ -8,6 +8,11 @@ namespace ZTSharp.ZeroTier.Internal; +internal readonly record struct ZeroTierInboundHelloPayload( + ulong Timestamp, + byte PeerProtocolVersion, + IPEndPoint? ReportedLocalSurfaceAddress); + internal sealed class ZeroTierDataplanePeerSecurity : IDisposable { private const int HelloPayloadMinLength = 13 + (5 + 1 + ZeroTierIdentity.PublicKeyLength + 1); @@ -107,8 +112,9 @@ public async Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken c return await task.WaitAsync(cancellationToken).ConfigureAwait(false); } - public async ValueTask HandleHelloAsync( + public async ValueTask HandleHelloAsync( NodeId peerNodeId, + int localSocketId, ulong helloPacketId, byte[] packetBytes, IPEndPoint remoteEndPoint, @@ -118,18 +124,18 @@ public async ValueTask HandleHelloAsync( if (packetBytes.Length > ZeroTierProtocolLimits.MaxPacketBytes) { - return; + return null; } if (packetBytes.Length < ZeroTierPacketHeader.Length + HelloPayloadMinLength) { - return; + return null; } var payload = packetBytes.AsSpan(ZeroTierPacketHeader.IndexPayload); if (payload.Length < HelloPayloadMinLength) { - return; + return null; } var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(5, 8)); @@ -141,12 +147,20 @@ public async ValueTask HandleHelloAsync( } catch (FormatException) { - return; + return null; } if (identity.NodeId != peerNodeId) { - return; + return null; + } + + var identityLength = ZeroTierIdentityCodec.GetSerializedLength(identity, includePrivate: false); + IPEndPoint? reportedLocalSurface = null; + if (payload.Length > 13 + identityLength && + ZeroTierInetAddressCodec.TryDeserialize(payload.Slice(13 + identityLength), out var parsedSurface, out _)) + { + reportedLocalSurface = parsedSurface; } var sharedKey = new byte[48]; @@ -156,7 +170,7 @@ public async ValueTask HandleHelloAsync( } catch (CryptographicException) { - return; + return null; } if (!ZeroTierPacketCrypto.Dearmor(packetBytes, sharedKey)) @@ -166,12 +180,12 @@ public async ValueTask HandleHelloAsync( ZeroTierTrace.WriteLine($"[zerotier] Drop: failed to authenticate HELLO from {peerNodeId} via {remoteEndPoint}."); } - return; + return null; } if (!identity.LocallyValidate()) { - return; + return null; } CachePeerKey(peerNodeId, sharedKey, nowMs: Environment.TickCount64); @@ -189,11 +203,16 @@ public async ValueTask HandleHelloAsync( try { - await _udp.SendAsync(remoteEndPoint, okPacket, cancellationToken).ConfigureAwait(false); + await _udp.SendAsync(localSocketId, remoteEndPoint, okPacket, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is SocketException or ObjectDisposedException) { } + + return new ZeroTierInboundHelloPayload( + Timestamp: helloTimestamp, + PeerProtocolVersion: peerProtocolVersion, + ReportedLocalSurfaceAddress: reportedLocalSurface); } public void Dispose() diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs index cbe7d2b..1291d49 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs @@ -28,12 +28,37 @@ internal static class ZeroTierDataplaneRuntimeFactory ArgumentNullException.ThrowIfNull(inlineCom); ArgumentNullException.ThrowIfNull(multipath); + var requiredSurfaceObservations = Math.Max(udp.LocalSockets.Count, 1); ZeroTierHelloOk helloOk; byte[] rootKey; if (cachedRoot is { } cachedHelloOk && cachedRootKey is not null) { - helloOk = cachedHelloOk; - rootKey = cachedRootKey; + var needsSurfaceRefresh = multipath.Enabled && + cachedHelloOk.ExternalSurfaceObservations.Count < requiredSurfaceObservations; + + if (!needsSurfaceRefresh) + { + helloOk = cachedHelloOk; + rootKey = cachedRootKey; + } + else + { + try + { + helloOk = await ZeroTierHelloClient + .HelloRootsAsync(udp, localIdentity, planet, timeout: TimeSpan.FromSeconds(10), cancellationToken) + .ConfigureAwait(false); + + rootKey = ComputeRootKey(localIdentity, planet, helloOk.RootNodeId); + } +#pragma warning disable CA1031 // Best-effort refresh should fall back to the cached root state. + catch +#pragma warning restore CA1031 + { + helloOk = cachedHelloOk; + rootKey = cachedRootKey; + } + } } else { @@ -44,6 +69,36 @@ internal static class ZeroTierDataplaneRuntimeFactory rootKey = ComputeRootKey(localIdentity, planet, helloOk.RootNodeId); } + if (multipath.Enabled && helloOk.ExternalSurfaceObservations.Count < requiredSurfaceObservations) + { + var refreshedObservations = await ZeroTierHelloClient + .ProbeRootExternalSurfacesAsync( + udp, + localIdentity, + planet, + helloOk.RootNodeId, + helloOk.RootEndpoint, + rootKey, + helloOk.ExternalSurfaceObservations, + timeout: TimeSpan.FromSeconds(10), + cancellationToken) + .ConfigureAwait(false); + + if (refreshedObservations.Count != helloOk.ExternalSurfaceObservations.Count) + { + var primarySurface = refreshedObservations + .OrderBy(static observation => observation.LocalSocketId) + .Select(static observation => observation.SurfaceAddress) + .FirstOrDefault() ?? helloOk.ExternalSurfaceAddress; + + helloOk = helloOk with + { + ExternalSurfaceAddress = primarySurface, + ExternalSurfaceObservations = refreshedObservations + }; + } + } + var runtime = new ZeroTierDataplaneRuntime( udp, rootNodeId: helloOk.RootNodeId, @@ -58,7 +113,8 @@ internal static class ZeroTierDataplaneRuntimeFactory multipath: multipath, planetId: planet.Id, planetTimestamp: planet.Timestamp, - localExternalSurfaceAddress: helloOk.ExternalSurfaceAddress); + localExternalSurfaceAddress: helloOk.ExternalSurfaceAddress, + initialExternalSurfaceObservations: helloOk.ExternalSurfaceObservations); await TrySubscribeForAddressResolutionAsync( udp, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs index ec7e11b..7ad3749 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs @@ -10,7 +10,7 @@ internal static class ZeroTierHelloPacketBuilder public static byte[] BuildPacket( ZeroTierIdentity localIdentity, NodeId destination, - IPEndPoint physicalDestination, + IPEndPoint? physicalDestination, ZeroTierWorld planet, ulong timestamp, ReadOnlySpan sharedKey, @@ -36,7 +36,7 @@ public static byte[] BuildPacket( public static byte[] BuildPacket( ZeroTierIdentity localIdentity, NodeId destination, - IPEndPoint physicalDestination, + IPEndPoint? physicalDestination, ulong timestamp, ulong planetId, ulong planetTimestamp, @@ -52,7 +52,7 @@ public static byte[] BuildPacket( packetId = BinaryPrimitives.ReadUInt64BigEndian(iv); var identityLength = ZeroTierIdentityCodec.GetSerializedLength(localIdentity, includePrivate: false); - var inetLength = ZeroTierInetAddressCodec.GetSerializedLength(physicalDestination); + var inetLength = ZeroTierInetAddressCodec.GetSerializedLengthNullable(physicalDestination); var payloadFixedLength = 1 + // protocol version @@ -80,7 +80,7 @@ public static byte[] BuildPacket( p += 8; p += ZeroTierIdentityCodec.Serialize(localIdentity, payload.AsSpan(p), includePrivate: false); - p += ZeroTierInetAddressCodec.Serialize(physicalDestination, payload.AsSpan(p)); + p += ZeroTierInetAddressCodec.SerializeNullable(physicalDestination, payload.AsSpan(p)); BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planetId); p += 8; diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs index 9a4e13f..912f564 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs @@ -5,6 +5,9 @@ namespace ZTSharp.ZeroTier.Protocol; internal static class ZeroTierInetAddressCodec { + public static int GetSerializedLengthNullable(IPEndPoint? endpoint) + => endpoint is null ? 1 : GetSerializedLength(endpoint); + public static int GetSerializedLength(IPEndPoint endpoint) { ArgumentNullException.ThrowIfNull(endpoint); @@ -17,6 +20,22 @@ public static int GetSerializedLength(IPEndPoint endpoint) }; } + public static int SerializeNullable(IPEndPoint? endpoint, Span destination) + { + if (endpoint is null) + { + if (destination.IsEmpty) + { + throw new ArgumentException("Destination must be at least 1 byte.", nameof(destination)); + } + + destination[0] = 0; + return 1; + } + + return Serialize(endpoint, destination); + } + public static int Serialize(IPEndPoint endpoint, Span destination) { ArgumentNullException.ThrowIfNull(endpoint); @@ -127,4 +146,3 @@ public static bool TryDeserialize(ReadOnlySpan data, out IPEndPoint? endpo } } } - From 9b9410c50b2cef4480b7aa97ca420c2372859e96 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:34:25 +0100 Subject: [PATCH 158/296] Maintain relayed peers in direct-path maintenance --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 88 +++++++++- ...TierDirectEndpointManagerPushFlagsTests.cs | 11 +- .../ZeroTierDirectEndpointSelectionTests.cs | 16 +- .../ZeroTierDataplanePeerDatagramProcessor.cs | 10 ++ .../Internal/ZeroTierDataplanePeerSecurity.cs | 17 ++ .../Internal/ZeroTierDataplaneRuntime.cs | 152 ++++++++---------- .../Internal/ZeroTierDirectEndpointManager.cs | 3 + .../ZeroTierDirectEndpointSelection.cs | 139 ++++++++-------- 8 files changed, 272 insertions(+), 164 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 4e5a058..b042209 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -129,9 +129,18 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() await rootUdp.SendAsync(runtimeEndpoint, pushPacket); - var probe = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - Assert.True(probe.Payload.Length > 4); - Assert.True(ZeroTierPacketCodec.TryDecode(probe.Payload, out _)); + var sawPacketProbe = false; + for (var attempt = 0; attempt < 4; attempt++) + { + var probe = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); + if (probe.Payload.Length > 4 && ZeroTierPacketCodec.TryDecode(probe.Payload, out _)) + { + sawPacketProbe = true; + break; + } + } + + Assert.True(sawPacketProbe); } [Fact] @@ -158,7 +167,7 @@ public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBefore rootProtocolVersion: 12, localIdentity, networkId: 0x9ad07d01093a69e3UL, - localManagedIpsV4: Array.Empty(), + localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, localManagedIpsV6: Array.Empty(), inlineCom: Array.Empty(), multipath: new ZeroTierMultipathOptions { Enabled = true }, @@ -215,6 +224,76 @@ public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBefore Assert.Equal(0, candidate.LocalSocketId); } + [Fact] + public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = new byte[48]; + RandomNumberGenerator.Fill(rootKey); + + var publicSurface = new IPEndPoint(IPAddress.Parse("203.0.113.10"), 54321); + await using var runtime = new ZeroTierDataplaneRuntime( + udp, + rootNodeId: rootNodeId, + rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), + rootKey: rootKey, + rootProtocolVersion: 12, + localIdentity, + networkId: 0x9ad07d01093a69e3UL, + localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, + localManagedIpsV6: Array.Empty(), + inlineCom: Array.Empty(), + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: new[] + { + new ZeroTierExternalSurfaceObservation(LocalSocketId: 0, SurfaceAddress: publicSurface) + }); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var planet = new ZeroTierWorld( + id: 1, + type: ZeroTierWorldType.Planet, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: Array.Empty()); + + var runtimeEndpoint = TestUdpEndpoints.ToLoopback(udp.LocalEndpoint); + var helloPacket = ZeroTierHelloPacketBuilder.BuildPacket( + peerIdentity, + destination: localIdentity.NodeId, + physicalDestination: runtimeEndpoint, + planet, + timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + sharedKey, + advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, + advertisedMajorVersion: ZeroTierHelloClient.AdvertisedMajorVersion, + advertisedMinorVersion: ZeroTierHelloClient.AdvertisedMinorVersion, + advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, + out _); + + await rootUdp.SendAsync(runtimeEndpoint, helloPacket); + _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + + var maintenancePeers = runtime.GetPeersForMultipathMaintenanceForTests(); + Assert.Contains(peerIdentity.NodeId, maintenancePeers); + + var advertisements = runtime.GetLocalDirectPathAdvertisementsForTests(); + Assert.Contains(publicSurface, advertisements); + } + private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) { var addressBytes = endpoint.Address.GetAddressBytes(); @@ -262,3 +341,4 @@ private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint) return payload; } } + diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index b0e16f3..2c7b6c9 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -79,7 +79,8 @@ public async Task PushDirectPaths_KnownEndpoint_IsNotRebootstrapped() await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); Assert.Single(hintedEndpoints); - Assert.Empty(udp.Sends); + Assert.Single(udp.Sends); + Assert.Equal(endpoint, udp.Sends[0].RemoteEndPoint); } [Fact] @@ -98,7 +99,10 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); - Assert.Empty(udp.Sends); + var send = Assert.Single(udp.Sends); + Assert.Equal(0, send.LocalSocketId); + Assert.Equal(endpoint, send.RemoteEndPoint); + Assert.Null(send.HopLimit); } private const byte ZtPushDirectPathsFlagForgetPath = 0x01; @@ -166,4 +170,5 @@ public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, public ValueTask DisposeAsync() => ValueTask.CompletedTask; } -} \ No newline at end of file +} + diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs index a1ea9a0..068eb90 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs @@ -7,7 +7,7 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectEndpointSelectionTests { [Fact] - public void Normalize_CanonicalizesIpv4MappedIpv6_ForUniquenessAndOrdering() + public void Normalize_CanonicalizesIpv4MappedIpv6_AndPreservesFirstSeenOrder() { var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); var endpoints = new[] @@ -30,5 +30,19 @@ public void Normalize_CanonicalizesIpv4MappedIpv6_ForUniquenessAndOrdering() Assert.Equal(new IPEndPoint(IPAddress.Parse("8.8.8.8"), 9993), normalized[0]); Assert.Equal(new IPEndPoint(IPAddress.Parse("10.0.0.1"), 9993), normalized[1]); } + + [Theory] + [InlineData("10.0.0.1", true)] + [InlineData("100.85.196.109", true)] + [InlineData("172.17.0.1", true)] + [InlineData("192.168.0.1", true)] + [InlineData("176.66.90.119", true)] + [InlineData("169.254.1.2", false)] + [InlineData("127.0.0.1", false)] + public void IsUsablePathEndpoint_FollowsUpstreamPathScopeRules(string ip, bool expected) + { + var endpoint = new IPEndPoint(IPAddress.Parse(ip), 9993); + Assert.Equal(expected, ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index e651331..ad793cd 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -18,6 +18,7 @@ internal sealed class ZeroTierDataplanePeerDatagramProcessor private readonly ZeroTierPeerQosManager _peerQos; private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly bool _multipathEnabled; + private readonly Action? _handleAuthenticatedPeer; private readonly Action? _handleHelloOk; private readonly Func? _handleUnknownDirectPathAsync; @@ -31,6 +32,7 @@ public ZeroTierDataplanePeerDatagramProcessor( ZeroTierPeerQosManager peerQos, ZeroTierPeerPathNegotiationManager peerNegotiation, bool multipathEnabled, + Action? handleAuthenticatedPeer = null, Action? handleHelloOk = null, Func? handleUnknownDirectPathAsync = null) { @@ -51,6 +53,7 @@ public ZeroTierDataplanePeerDatagramProcessor( _peerQos = peerQos; _peerNegotiation = peerNegotiation; _multipathEnabled = multipathEnabled; + _handleAuthenticatedPeer = handleAuthenticatedPeer; _handleHelloOk = handleHelloOk; _handleUnknownDirectPathAsync = handleUnknownDirectPathAsync; } @@ -101,6 +104,11 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c } } + if (inboundHello is not null) + { + _handleAuthenticatedPeer?.Invoke(peerNodeId); + } + return; } @@ -115,6 +123,8 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c return; } + _handleAuthenticatedPeer?.Invoke(peerNodeId); + if ((packetBytes[ZeroTierPacketHeader.IndexVerb] & ZeroTierPacketHeader.VerbFlagCompressed) != 0) { if (!ZeroTierPacketCompression.TryUncompress(packetBytes, out var uncompressed)) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs index 2dbfdbd..7005805 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs @@ -221,6 +221,23 @@ public void Dispose() _cts.Dispose(); } + internal NodeId[] GetTrustedPeersSnapshot() + { + var nowMs = Environment.TickCount64; + var peers = new List(); + foreach (var (peerNodeId, entry) in _peerKeys) + { + if (entry.Key is null || entry.ExpiresAtUnixMs <= nowMs) + { + continue; + } + + peers.Add(peerNodeId); + } + + return peers.ToArray(); + } + private Task StartPeerKeyFetch(NodeId peerNodeId) { var task = FetchAndCachePeerKeyAsync(peerNodeId); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 98a8315..614c9b8 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -49,9 +49,11 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierMultipathOptions _multipath; + private readonly ConcurrentDictionary _authenticatedPeers = new(); private readonly ConcurrentDictionary _directBootstrapTasks = new(); private readonly ConcurrentDictionary _directBootstrapProbeCursors = new(); private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); + private readonly ConcurrentDictionary _lastDirectPathPushSentMs = new(); private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); private readonly ConcurrentDictionary _pendingHelloProbes = new(); private long _lastPendingHelloCleanupMs; @@ -217,6 +219,7 @@ public ZeroTierDataplaneRuntime( _peerQos, _peerNegotiation, multipath.Enabled, + HandleAuthenticatedPeer, HandlePeerHelloOk, HandleUnknownDirectPathAsync); _rxLoops = new ZeroTierDataplaneRxLoops( @@ -723,6 +726,31 @@ private long GetDirectPathPushBootstrapIntervalMs(NodeId peerNodeId) ? DirectBootstrapPathPushIntervalHavePathMs : DirectBootstrapPathPushIntervalMs; + private async Task TrySendPeriodicDirectPathPushAsync( + NodeId peerNodeId, + byte[] sharedKey, + CancellationToken cancellationToken) + { + var nowMs = Environment.TickCount64; + var minIntervalMs = GetDirectPathPushBootstrapIntervalMs(peerNodeId); + while (true) + { + if (_lastDirectPathPushSentMs.TryGetValue(peerNodeId, out var lastSent) && + unchecked(nowMs - lastSent) < minIntervalMs) + { + return; + } + + if (_lastDirectPathPushSentMs.TryAdd(peerNodeId, nowMs) || + _lastDirectPathPushSentMs.TryUpdate(peerNodeId, nowMs, lastSent)) + { + break; + } + } + + await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + } + private async Task ProbeHintedDirectEndpointsAsync( NodeId peerNodeId, IPEndPoint[] hinted, @@ -1377,6 +1405,11 @@ await SendHelloPacketAsync( .ConfigureAwait(false); } + private void HandleAuthenticatedPeer(NodeId peerNodeId) + { + _authenticatedPeers[peerNodeId] = 0; + } + private void TrackPendingHello( NodeId peerNodeId, int localSocketId, @@ -1473,6 +1506,8 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati continue; } + await TrySendPeriodicDirectPathPushAsync(peerNodeId, key, cancellationToken).ConfigureAwait(false); + var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); var paths = _peerPaths.GetSnapshot(peerNodeId); var hintedCandidates = GetHintedDirectCandidates(peerNodeId, paths); @@ -1748,97 +1783,14 @@ private bool ShouldAcceptDirectEndpoint(IPEndPoint endpoint) return true; } - if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + if (ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)) { return true; } - var localAdvertisements = GetLocalDirectPathAdvertisements(); - for (var i = 0; i < localAdvertisements.Length; i++) - { - if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(localAdvertisements[i])) - { - continue; - } - - if (HaveCompatiblePrivateScope(localAdvertisements[i], endpoint)) - { - return true; - } - } - if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] Drop hinted private endpoint: {endpoint}."); - } - - return false; - } - - private static bool HaveCompatiblePrivateScope(IPEndPoint localEndpoint, IPEndPoint remoteEndpoint) - { - var localAddress = localEndpoint.Address; - var remoteAddress = remoteEndpoint.Address; - if (localAddress.AddressFamily == AddressFamily.InterNetworkV6 && localAddress.IsIPv4MappedToIPv6) - { - localAddress = localAddress.MapToIPv4(); - } - - if (remoteAddress.AddressFamily == AddressFamily.InterNetworkV6 && remoteAddress.IsIPv4MappedToIPv6) - { - remoteAddress = remoteAddress.MapToIPv4(); - } - - if (localAddress.AddressFamily != remoteAddress.AddressFamily) - { - return false; - } - - if (localAddress.AddressFamily == AddressFamily.InterNetwork) - { - var localBytes = localAddress.GetAddressBytes(); - var remoteBytes = remoteAddress.GetAddressBytes(); - - if (localBytes[0] == 10 && remoteBytes[0] == 10) - { - return true; - } - - if (localBytes[0] == 172 && - localBytes[1] is >= 16 and <= 31 && - remoteBytes[0] == 172 && - remoteBytes[1] is >= 16 and <= 31) - { - return true; - } - - if (localBytes[0] == 192 && - localBytes[1] == 168 && - remoteBytes[0] == 192 && - remoteBytes[1] == 168) - { - return true; - } - - if (localBytes[0] == 100 && - localBytes[1] is >= 64 and <= 127 && - remoteBytes[0] == 100 && - remoteBytes[1] is >= 64 and <= 127) - { - return true; - } - - return false; - } - - if (localAddress.AddressFamily == AddressFamily.InterNetworkV6) - { - var localBytes = localAddress.GetAddressBytes(); - var remoteBytes = remoteAddress.GetAddressBytes(); - return localBytes.Length == 16 && - remoteBytes.Length == 16 && - (localBytes[0] & 0xFE) == 0xFC && - (remoteBytes[0] & 0xFE) == 0xFC; + ZeroTierTrace.WriteLine($"[zerotier] Drop unusable hinted endpoint: {endpoint}."); } return false; @@ -1871,15 +1823,40 @@ private void CleanupDirectEndpointManagers(long nowMs) internal ZeroTierSelectedPeerPath[] GetHintedDirectCandidatesForMaintenance(NodeId peerNodeId) => GetHintedDirectCandidates(peerNodeId, _peerPaths.GetSnapshot(peerNodeId)); + internal Task RunMultipathMaintenanceOnceForTestsAsync(CancellationToken cancellationToken = default) + => RunMultipathMaintenanceOnceAsync(cancellationToken); + + internal NodeId[] GetPeersForMultipathMaintenanceForTests() + => GetPeersForMultipathMaintenance(); + + internal IPEndPoint[] GetLocalDirectPathAdvertisementsForTests() + => GetLocalDirectPathAdvertisements(); + private NodeId[] GetPeersForMultipathMaintenance() { var peers = _peerPaths.GetPeersSnapshot(); - if (_directEndpoints.IsEmpty) + if (_directEndpoints.IsEmpty && _authenticatedPeers.IsEmpty) { - return peers; + var trustedPeers = _peerSecurity.GetTrustedPeersSnapshot(); + if (trustedPeers.Length == 0) + { + return peers; + } + + return peers.Concat(trustedPeers).Distinct().ToArray(); } var set = peers.ToHashSet(); + foreach (var peerNodeId in _peerSecurity.GetTrustedPeersSnapshot()) + { + set.Add(peerNodeId); + } + + foreach (var peerNodeId in _authenticatedPeers.Keys) + { + set.Add(peerNodeId); + } + foreach (var peerNodeId in _directEndpoints.Keys) { set.Add(peerNodeId); @@ -1991,3 +1968,4 @@ internal readonly record struct PendingHelloProbe( IPEndPoint SendTo, IPEndPoint? PhysicalDestination, long SentAtMs); + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 2c25720..bac2a58 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -225,6 +225,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( foreach (var endpoint in endpointsToProbe) { + TrySendHolePunch(endpoint); if (_handleDirectEndpointHintAsync is not null) { await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); @@ -467,3 +468,5 @@ private static string FormatEndpointKey(IPEndPoint endpoint) return $"{address}:{endpoint.Port}"; } } + + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs index 23728ec..917e1f1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs @@ -17,10 +17,8 @@ public static IPEndPoint[] Normalize( relayEndpoint = Canonicalize(relayEndpoint); - var publicV4 = new List(); - var publicV6 = new List(); - var privateV4 = new List(); - var privateV6 = new List(); + var unique = new List(); + var seen = new HashSet(StringComparer.Ordinal); foreach (var endpoint in endpoints) { @@ -50,33 +48,13 @@ public static IPEndPoint[] Normalize( continue; } - var isPublic = IsPublicAddress(canonical.Address); - if (canonical.AddressFamily == AddressFamily.InterNetwork) - { - (isPublic ? publicV4 : privateV4).Add(canonical); - } - else if (canonical.AddressFamily == AddressFamily.InterNetworkV6) - { - (isPublic ? publicV6 : privateV6).Add(canonical); - } - } - - var ordered = publicV4 - .Concat(publicV6) - .Concat(privateV4) - .Concat(privateV6); - - var unique = new List(); - var seen = new HashSet(StringComparer.Ordinal); - foreach (var endpoint in ordered) - { - var key = endpoint.Address + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); + var key = canonical.Address + ":" + canonical.Port.ToString(CultureInfo.InvariantCulture); if (!seen.Add(key)) { continue; } - unique.Add(endpoint); + unique.Add(canonical); if (unique.Count >= maxEndpoints) { break; @@ -118,7 +96,21 @@ public static bool IsPrivateEndpoint(IPEndPoint endpoint) return !IsPublicEndpoint(endpoint); } + public static bool IsUsablePathEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + endpoint = Canonicalize(endpoint); + return GetScope(endpoint.Address) is + EndpointScope.Private or + EndpointScope.Shared or + EndpointScope.Pseudoprivate or + EndpointScope.Global; + } + private static bool IsPublicAddress(IPAddress address) + => GetScope(address) == EndpointScope.Global; + + private static EndpointScope GetScope(IPAddress address) { if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) { @@ -127,7 +119,7 @@ private static bool IsPublicAddress(IPAddress address) if (IPAddress.IsLoopback(address)) { - return false; + return EndpointScope.Loopback; } if (address.AddressFamily == AddressFamily.InterNetwork) @@ -135,47 +127,34 @@ private static bool IsPublicAddress(IPAddress address) var bytes = address.GetAddressBytes(); if (bytes.Length != 4) { - return false; - } - - if (bytes[0] == 10) - { - return false; - } - - if (bytes[0] == 172 && bytes[1] is >= 16 and <= 31) - { - return false; - } - - if (bytes[0] == 192 && bytes[1] == 168) - { - return false; - } - - if (bytes[0] == 169 && bytes[1] == 254) - { - return false; - } - - if (bytes[0] == 100 && bytes[1] is >= 64 and <= 127) - { - return false; - } - - if (bytes[0] == 0 || bytes[0] >= 224) - { - return false; - } - - return true; + return EndpointScope.None; + } + + return bytes[0] switch + { + 0 => EndpointScope.None, + 6 or 11 or 21 or 22 or 25 or 26 or 28 or 29 or 30 or 51 or 55 or 56 => EndpointScope.Pseudoprivate, + 10 => EndpointScope.Private, + 100 when bytes[1] is >= 64 and <= 127 => EndpointScope.Shared, + 127 => EndpointScope.Loopback, + 169 when bytes[1] == 254 => EndpointScope.LinkLocal, + 172 when bytes[1] is >= 16 and <= 31 => EndpointScope.Private, + 192 when bytes[1] == 168 => EndpointScope.Private, + 192 when bytes[1] == 0 && bytes[2] == 2 => EndpointScope.Private, + 198 when bytes[1] is 18 or 19 => EndpointScope.Private, + 198 when bytes[1] == 51 && bytes[2] == 100 => EndpointScope.Private, + 203 when bytes[1] == 0 && bytes[2] == 113 => EndpointScope.Private, + >= 224 and <= 239 => EndpointScope.Multicast, + >= 240 => EndpointScope.Pseudoprivate, + _ => EndpointScope.Global + }; } if (address.AddressFamily == AddressFamily.InterNetworkV6) { if (address.Equals(IPAddress.IPv6Any) || address.Equals(IPAddress.IPv6None)) { - return false; + return EndpointScope.None; } if (address.IsIPv6LinkLocal || @@ -183,25 +162,47 @@ private static bool IsPublicAddress(IPAddress address) address.IsIPv6SiteLocal || address.Equals(IPAddress.IPv6Loopback)) { - return false; + return address.IsIPv6Multicast + ? EndpointScope.Multicast + : address.Equals(IPAddress.IPv6Loopback) + ? EndpointScope.Loopback + : EndpointScope.LinkLocal; } var bytes = address.GetAddressBytes(); if (bytes.Length != 16) { - return false; + return EndpointScope.None; } - // fc00::/7 Unique Local Address (ULA) if ((bytes[0] & 0xFE) == 0xFC) { - return false; + return EndpointScope.Private; } - return true; + if (bytes[0] == 0x20 && + bytes[1] == 0x01 && + bytes[2] == 0x04 && + bytes[3] == 0x70) + { + return EndpointScope.None; + } + + return EndpointScope.Global; } - return false; + return EndpointScope.None; } -} + private enum EndpointScope + { + None, + Loopback, + LinkLocal, + Private, + Shared, + Pseudoprivate, + Global, + Multicast + } +} From 5ed7ebcd98c1d403a0c426117e688bef7506a389 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:37:47 +0100 Subject: [PATCH 159/296] Prioritize reachable hinted path scopes --- .../ZeroTierDirectEndpointSelectionTests.cs | 21 ++++++++++- .../ZeroTierDirectEndpointSelection.cs | 37 +++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs index 068eb90..9179265 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs @@ -31,7 +31,25 @@ public void Normalize_CanonicalizesIpv4MappedIpv6_AndPreservesFirstSeenOrder() Assert.Equal(new IPEndPoint(IPAddress.Parse("10.0.0.1"), 9993), normalized[1]); } - [Theory] + + [Fact] + public void Normalize_PrioritizesGlobalCandidatesBeforePrivateCandidates() + { + var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); + var endpoints = new[] + { + new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993), + new IPEndPoint(IPAddress.Parse("172.18.0.1"), 9993), + new IPEndPoint(IPAddress.Parse("10.22.10.94"), 9993), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 48705), + }; + + var normalized = ZeroTierDirectEndpointSelection.Normalize(endpoints, relay, maxEndpoints: 3); + + Assert.Equal(new IPEndPoint(IPAddress.Parse("176.66.90.119"), 48705), normalized[0]); + Assert.Contains(normalized, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993))); + } [Theory] [InlineData("10.0.0.1", true)] [InlineData("100.85.196.109", true)] [InlineData("172.17.0.1", true)] @@ -46,3 +64,4 @@ public void IsUsablePathEndpoint_FollowsUpstreamPathScopeRules(string ip, bool e } } + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs index 917e1f1..02da240 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs @@ -17,8 +17,10 @@ public static IPEndPoint[] Normalize( relayEndpoint = Canonicalize(relayEndpoint); - var unique = new List(); - var seen = new HashSet(StringComparer.Ordinal); + var global = new List(); + var shared = new List(); + var pseudoprivate = new List(); + var privateEndpoints = new List(); foreach (var endpoint in endpoints) { @@ -48,13 +50,39 @@ public static IPEndPoint[] Normalize( continue; } - var key = canonical.Address + ":" + canonical.Port.ToString(CultureInfo.InvariantCulture); + switch (GetScope(canonical.Address)) + { + case EndpointScope.Global: + global.Add(canonical); + break; + case EndpointScope.Shared: + shared.Add(canonical); + break; + case EndpointScope.Pseudoprivate: + pseudoprivate.Add(canonical); + break; + case EndpointScope.Private: + privateEndpoints.Add(canonical); + break; + } + } + + var ordered = global + .Concat(shared) + .Concat(pseudoprivate) + .Concat(privateEndpoints); + + var unique = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var endpoint in ordered) + { + var key = endpoint.Address + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); if (!seen.Add(key)) { continue; } - unique.Add(canonical); + unique.Add(endpoint); if (unique.Count >= maxEndpoints) { break; @@ -206,3 +234,4 @@ private enum EndpointScope Multicast } } + From 01a823e3c02f8a2f05f0e2cd627364021ed1a19a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:38:37 +0100 Subject: [PATCH 160/296] Probe hinted direct paths across fallback sockets --- .../Internal/ZeroTierDataplaneRuntime.cs | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 614c9b8..d7f2c89 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -766,7 +766,11 @@ private async Task ProbeHintedDirectEndpointsAsync( var endpointsToProbe = cursor.TakeNextEndpoints(hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { - var localSocketIds = GetRotatingDirectBootstrapSocketIds(peerNodeId, endpointsToProbe[i], cursor); + var localSocketIds = GetRotatingDirectBootstrapSocketIds( + peerNodeId, + endpointsToProbe[i], + cursor, + includeFallbackLocalSockets: true); for (var s = 0; s < localSocketIds.Length; s++) { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpointsToProbe[i], sharedKey, cancellationToken) @@ -965,7 +969,7 @@ private async Task SendHelloDirectAsync( { for (var i = 0; i < endpoints.Length; i++) { - var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoints[i]); + var localSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, endpoints[i]); for (var s = 0; s < localSocketIds.Length; s++) { await SendHelloPacketAsync( @@ -988,7 +992,7 @@ private async Task SendEchoDirectAsync( { for (var i = 0; i < endpoints.Length; i++) { - var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoints[i]); + var localSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, endpoints[i]); for (var s = 0; s < localSocketIds.Length; s++) { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpoints[i], sharedKey, cancellationToken) @@ -998,6 +1002,15 @@ await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpoints[i], s } private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + => GetDirectBootstrapSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: false); + + private int[] GetPreferredAndFallbackDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + => GetDirectBootstrapSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true); + + private int[] GetDirectBootstrapSocketIds( + NodeId peerNodeId, + IPEndPoint endpoint, + bool includeFallbackLocalSockets) { var localSockets = _udp.LocalSockets; if (localSockets.Count == 0) @@ -1015,17 +1028,15 @@ private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint } } - if (preferred.Count != 0) + if (includeFallbackLocalSockets || preferred.Count == 0) { - return preferred.ToArray(); - } - - for (var i = 0; i < localSockets.Count; i++) - { - var socketId = localSockets[i].Id; - if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && !preferred.Contains(socketId)) + for (var i = 0; i < localSockets.Count; i++) { - preferred.Add(socketId); + var socketId = localSockets[i].Id; + if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && !preferred.Contains(socketId)) + { + preferred.Add(socketId); + } } } @@ -1040,9 +1051,12 @@ private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint private int[] GetRotatingDirectBootstrapSocketIds( NodeId peerNodeId, IPEndPoint endpoint, - DirectBootstrapProbeCursor cursor) + DirectBootstrapProbeCursor cursor, + bool includeFallbackLocalSockets) { - var preferred = GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoint); + var preferred = includeFallbackLocalSockets + ? GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, endpoint) + : GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoint); if (preferred.Length <= 1) { return preferred; @@ -1085,7 +1099,11 @@ private async ValueTask HandleDirectEndpointHintAsync( } var cursor = _directBootstrapProbeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); - var localSocketIds = GetRotatingDirectBootstrapSocketIds(peerNodeId, endpoint, cursor); + var localSocketIds = GetRotatingDirectBootstrapSocketIds( + peerNodeId, + endpoint, + cursor, + includeFallbackLocalSockets: false); for (var i = 0; i < localSocketIds.Length; i++) { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) @@ -1250,7 +1268,7 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel if (hinted.Length > 0) { var endpointIndex = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); - var preferredLocalSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, hinted[endpointIndex]); + var preferredLocalSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, hinted[endpointIndex]); var socketIndex = preferredLocalSocketIds.Length <= 1 ? 0 : (int)((flowId / (uint)hinted.Length) % (uint)preferredLocalSocketIds.Length); @@ -1280,7 +1298,7 @@ private bool TrySelectConfirmedHintedDirectPath( var bestRttMs = int.MaxValue; for (var i = 0; i < hinted.Length; i++) { - var socketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, hinted[i]); + var socketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, hinted[i]); for (var s = 0; s < socketIds.Length; s++) { if (!_peerEcho.TryGetLastRttMs(peerNodeId, socketIds[s], hinted[i], out var rttMs)) @@ -1890,7 +1908,7 @@ private ZeroTierSelectedPeerPath[] GetHintedDirectCandidates(NodeId peerNodeId, var candidates = new List(hinted.Length); for (var i = 0; i < hinted.Length; i++) { - var localSocketIds = GetPreferredDirectBootstrapSocketIds(peerNodeId, hinted[i]); + var localSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, hinted[i]); for (var s = 0; s < localSocketIds.Length; s++) { var key = new ZeroTierPeerPhysicalPathKey(localSocketIds[s], hinted[i]); From 61d107cbf662950dfb8feb33939ab109862f0177 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:50:32 +0100 Subject: [PATCH 161/296] Modularize direct hint path planning --- .../Internal/ZeroTierDataplaneRuntime.cs | 187 ++---------------- .../Internal/ZeroTierDirectHintPathPlanner.cs | 182 +++++++++++++++++ 2 files changed, 194 insertions(+), 175 deletions(-) create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d7f2c89..9161204 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -48,10 +48,10 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerQosManager _peerQos; private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly ZeroTierPeerBondPolicyEngine _bondEngine; + private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _authenticatedPeers = new(); private readonly ConcurrentDictionary _directBootstrapTasks = new(); - private readonly ConcurrentDictionary _directBootstrapProbeCursors = new(); private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); private readonly ConcurrentDictionary _lastDirectPathPushSentMs = new(); private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); @@ -196,6 +196,7 @@ public ZeroTierDataplaneRuntime( _peerQos = new ZeroTierPeerQosManager(); _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); + _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); var ip = new ZeroTierDataplaneIpHandler( @@ -762,14 +763,12 @@ private async Task ProbeHintedDirectEndpointsAsync( return; } - var cursor = _directBootstrapProbeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); - var endpointsToProbe = cursor.TakeNextEndpoints(hinted, DirectBootstrapHintProbeBudget); + var endpointsToProbe = _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { - var localSocketIds = GetRotatingDirectBootstrapSocketIds( + var localSocketIds = _directHintPlanner.GetRotatingSocketIds( peerNodeId, endpointsToProbe[i], - cursor, includeFallbackLocalSockets: true); for (var s = 0; s < localSocketIds.Length; s++) { @@ -969,7 +968,7 @@ private async Task SendHelloDirectAsync( { for (var i = 0; i < endpoints.Length; i++) { - var localSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, endpoints[i]); + var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoints[i]); for (var s = 0; s < localSocketIds.Length; s++) { await SendHelloPacketAsync( @@ -992,7 +991,7 @@ private async Task SendEchoDirectAsync( { for (var i = 0; i < endpoints.Length; i++) { - var localSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, endpoints[i]); + var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoints[i]); for (var s = 0; s < localSocketIds.Length; s++) { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpoints[i], sharedKey, cancellationToken) @@ -1001,71 +1000,6 @@ await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpoints[i], s } } - private int[] GetPreferredDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint endpoint) - => GetDirectBootstrapSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: false); - - private int[] GetPreferredAndFallbackDirectBootstrapSocketIds(NodeId peerNodeId, IPEndPoint endpoint) - => GetDirectBootstrapSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true); - - private int[] GetDirectBootstrapSocketIds( - NodeId peerNodeId, - IPEndPoint endpoint, - bool includeFallbackLocalSockets) - { - var localSockets = _udp.LocalSockets; - if (localSockets.Count == 0) - { - return new[] { 0 }; - } - - var preferred = new List(localSockets.Count); - var preferredFromHints = GetOrCreateDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); - for (var i = 0; i < preferredFromHints.Length; i++) - { - if (!preferred.Contains(preferredFromHints[i])) - { - preferred.Add(preferredFromHints[i]); - } - } - - if (includeFallbackLocalSockets || preferred.Count == 0) - { - for (var i = 0; i < localSockets.Count; i++) - { - var socketId = localSockets[i].Id; - if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && !preferred.Contains(socketId)) - { - preferred.Add(socketId); - } - } - } - - if (preferred.Count != 0) - { - return preferred.ToArray(); - } - - return localSockets.Select(static socket => socket.Id).ToArray(); - } - - private int[] GetRotatingDirectBootstrapSocketIds( - NodeId peerNodeId, - IPEndPoint endpoint, - DirectBootstrapProbeCursor cursor, - bool includeFallbackLocalSockets) - { - var preferred = includeFallbackLocalSockets - ? GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, endpoint) - : GetPreferredDirectBootstrapSocketIds(peerNodeId, endpoint); - if (preferred.Length <= 1) - { - return preferred; - } - - var socketIndex = cursor.TakeNextSocketIndex(endpoint, preferred.Length); - return new[] { preferred[socketIndex] }; - } - private async ValueTask HandleDirectEndpointHintAsync( NodeId peerNodeId, int receivedLocalSocketId, @@ -1098,11 +1032,9 @@ private async ValueTask HandleDirectEndpointHintAsync( ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId}."); } - var cursor = _directBootstrapProbeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); - var localSocketIds = GetRotatingDirectBootstrapSocketIds( + var localSocketIds = _directHintPlanner.GetRotatingSocketIds( peerNodeId, endpoint, - cursor, includeFallbackLocalSockets: false); for (var i = 0; i < localSocketIds.Length; i++) { @@ -1268,7 +1200,7 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel if (hinted.Length > 0) { var endpointIndex = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); - var preferredLocalSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, hinted[endpointIndex]); + var preferredLocalSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[endpointIndex]); var socketIndex = preferredLocalSocketIds.Length <= 1 ? 0 : (int)((flowId / (uint)hinted.Length) % (uint)preferredLocalSocketIds.Length); @@ -1298,7 +1230,7 @@ private bool TrySelectConfirmedHintedDirectPath( var bestRttMs = int.MaxValue; for (var i = 0; i < hinted.Length; i++) { - var socketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, hinted[i]); + var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); for (var s = 0; s < socketIds.Length; s++) { if (!_peerEcho.TryGetLastRttMs(peerNodeId, socketIds[s], hinted[i], out var rttMs)) @@ -1528,7 +1460,7 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); var paths = _peerPaths.GetSnapshot(peerNodeId); - var hintedCandidates = GetHintedDirectCandidates(peerNodeId, paths); + var hintedCandidates = _directHintPlanner.GetHintedCandidates(peerNodeId, paths); if (paths.Length == 0 && hintedCandidates.Length == 0) { continue; @@ -1833,13 +1765,13 @@ private void CleanupDirectEndpointManagers(long nowMs) { _directEndpointLastUsedMs.TryRemove(pair.Key, out _); _directEndpoints.TryRemove(pair.Key, out _); - _directBootstrapProbeCursors.TryRemove(pair.Key, out _); + _directHintPlanner.RemovePeer(pair.Key); } } } internal ZeroTierSelectedPeerPath[] GetHintedDirectCandidatesForMaintenance(NodeId peerNodeId) - => GetHintedDirectCandidates(peerNodeId, _peerPaths.GetSnapshot(peerNodeId)); + => _directHintPlanner.GetHintedCandidates(peerNodeId, _peerPaths.GetSnapshot(peerNodeId)); internal Task RunMultipathMaintenanceOnceForTestsAsync(CancellationToken cancellationToken = default) => RunMultipathMaintenanceOnceAsync(cancellationToken); @@ -1883,101 +1815,6 @@ private NodeId[] GetPeersForMultipathMaintenance() return set.ToArray(); } - private ZeroTierSelectedPeerPath[] GetHintedDirectCandidates(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths) - { - if (!_directEndpoints.TryGetValue(peerNodeId, out var directEndpoints)) - { - return Array.Empty(); - } - - var hinted = directEndpoints.Endpoints; - if (hinted.Length == 0) - { - return Array.Empty(); - } - - HashSet? observed = null; - if (observedPaths.Length > 0) - { - observed = observedPaths - .Select(static path => new ZeroTierPeerPhysicalPathKey(path.LocalSocketId, path.RemoteEndPoint)) - .ToHashSet(); - } - - var unique = new HashSet(); - var candidates = new List(hinted.Length); - for (var i = 0; i < hinted.Length; i++) - { - var localSocketIds = GetPreferredAndFallbackDirectBootstrapSocketIds(peerNodeId, hinted[i]); - for (var s = 0; s < localSocketIds.Length; s++) - { - var key = new ZeroTierPeerPhysicalPathKey(localSocketIds[s], hinted[i]); - if (observed is not null && observed.Contains(key)) - { - continue; - } - - if (!unique.Add(key)) - { - continue; - } - - candidates.Add(new ZeroTierSelectedPeerPath(localSocketIds[s], hinted[i])); - } - } - - return candidates.ToArray(); - } - -} - -internal sealed class DirectBootstrapProbeCursor -{ - private readonly object _lock = new(); - private int _nextEndpointIndex; - private readonly Dictionary _nextSocketIndexByEndpoint = new(StringComparer.Ordinal); - - public IPEndPoint[] TakeNextEndpoints(IPEndPoint[] endpoints, int budget) - { - ArgumentNullException.ThrowIfNull(endpoints); - - if (endpoints.Length == 0 || budget <= 0) - { - return Array.Empty(); - } - - lock (_lock) - { - var count = Math.Min(budget, endpoints.Length); - var selected = new IPEndPoint[count]; - for (var i = 0; i < count; i++) - { - var index = (_nextEndpointIndex + i) % endpoints.Length; - selected[i] = endpoints[index]; - } - - _nextEndpointIndex = (_nextEndpointIndex + count) % endpoints.Length; - return selected; - } - } - - public int TakeNextSocketIndex(IPEndPoint endpoint, int socketCount) - { - ArgumentNullException.ThrowIfNull(endpoint); - - if (socketCount <= 1) - { - return 0; - } - - lock (_lock) - { - var key = endpoint.ToString(); - _nextSocketIndexByEndpoint.TryGetValue(key, out var nextIndex); - _nextSocketIndexByEndpoint[key] = (nextIndex + 1) % socketCount; - return nextIndex % socketCount; - } - } } internal readonly record struct PendingHelloProbe( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs new file mode 100644 index 0000000..f01d89e --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -0,0 +1,182 @@ +using System.Collections.Concurrent; +using System.Net; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectHintPathPlanner +{ + private readonly IZeroTierUdpTransport _udp; + private readonly ZeroTierExternalSurfaceAddressTracker _surfaceAddresses; + private readonly Func _getDirectEndpointManager; + private readonly ConcurrentDictionary _probeCursors = new(); + + public ZeroTierDirectHintPathPlanner( + IZeroTierUdpTransport udp, + ZeroTierExternalSurfaceAddressTracker surfaceAddresses, + Func getDirectEndpointManager) + { + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(surfaceAddresses); + ArgumentNullException.ThrowIfNull(getDirectEndpointManager); + + _udp = udp; + _surfaceAddresses = surfaceAddresses; + _getDirectEndpointManager = getDirectEndpointManager; + } + + public IPEndPoint[] TakeNextHintedEndpoints(NodeId peerNodeId, IPEndPoint[] hinted, int budget) + => GetOrCreateCursor(peerNodeId).TakeNextEndpoints(hinted, budget); + + public int[] GetRotatingSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeFallbackLocalSockets) + { + var socketIds = GetSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets); + if (socketIds.Length <= 1) + { + return socketIds; + } + + var cursor = GetOrCreateCursor(peerNodeId); + var socketIndex = cursor.TakeNextSocketIndex(endpoint, socketIds.Length); + return new[] { socketIds[socketIndex] }; + } + + public int[] GetPreferredSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + => GetSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: false); + + public int[] GetPreferredAndFallbackSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + => GetSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true); + + public ZeroTierSelectedPeerPath[] GetHintedCandidates(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths) + { + var hinted = _getDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length == 0) + { + return Array.Empty(); + } + + HashSet? observed = null; + if (observedPaths.Length > 0) + { + observed = observedPaths + .Select(static path => new ZeroTierPeerPhysicalPathKey(path.LocalSocketId, path.RemoteEndPoint)) + .ToHashSet(); + } + + var unique = new HashSet(); + var candidates = new List(hinted.Length); + for (var i = 0; i < hinted.Length; i++) + { + var localSocketIds = GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); + for (var s = 0; s < localSocketIds.Length; s++) + { + var key = new ZeroTierPeerPhysicalPathKey(localSocketIds[s], hinted[i]); + if (observed is not null && observed.Contains(key)) + { + continue; + } + + if (!unique.Add(key)) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(localSocketIds[s], hinted[i])); + } + } + + return candidates.ToArray(); + } + + public void RemovePeer(NodeId peerNodeId) + { + _probeCursors.TryRemove(peerNodeId, out _); + } + + private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeFallbackLocalSockets) + { + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return new[] { 0 }; + } + + var preferred = new List(localSockets.Count); + var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); + for (var i = 0; i < preferredFromHints.Length; i++) + { + if (!preferred.Contains(preferredFromHints[i])) + { + preferred.Add(preferredFromHints[i]); + } + } + + if (includeFallbackLocalSockets || preferred.Count == 0) + { + for (var i = 0; i < localSockets.Count; i++) + { + var socketId = localSockets[i].Id; + if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && !preferred.Contains(socketId)) + { + preferred.Add(socketId); + } + } + } + + if (preferred.Count != 0) + { + return preferred.ToArray(); + } + + return localSockets.Select(static socket => socket.Id).ToArray(); + } + + private DirectBootstrapProbeCursor GetOrCreateCursor(NodeId peerNodeId) + => _probeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); +} + +internal sealed class DirectBootstrapProbeCursor +{ + private readonly object _lock = new(); + private int _nextEndpointIndex; + private readonly Dictionary _nextSocketIndexByEndpoint = new(StringComparer.Ordinal); + + public IPEndPoint[] TakeNextEndpoints(IPEndPoint[] endpoints, int budget) + { + ArgumentNullException.ThrowIfNull(endpoints); + + if (endpoints.Length == 0 || budget <= 0) + { + return Array.Empty(); + } + + lock (_lock) + { + var count = Math.Min(budget, endpoints.Length); + var selected = new IPEndPoint[count]; + for (var i = 0; i < count; i++) + { + selected[i] = endpoints[(_nextEndpointIndex + i) % endpoints.Length]; + } + + _nextEndpointIndex = (_nextEndpointIndex + count) % endpoints.Length; + return selected; + } + } + + public int TakeNextSocketIndex(IPEndPoint endpoint, int socketCount) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(socketCount); + + var key = endpoint.ToString(); + lock (_lock) + { + var index = _nextSocketIndexByEndpoint.TryGetValue(key, out var current) + ? current + : 0; + _nextSocketIndexByEndpoint[key] = (index + 1) % socketCount; + return index; + } + } +} From 462b5d16b2a088c1b88e349501406ec7fd04b02c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:56:22 +0100 Subject: [PATCH 162/296] Keep direct candidates per scope and family --- .../ZeroTierDirectEndpointSelectionTests.cs | 25 ++++- .../Internal/ZeroTierDirectEndpointManager.cs | 12 ++- .../ZeroTierDirectEndpointSelection.cs | 94 +++++++++++++------ 3 files changed, 96 insertions(+), 35 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs index 9179265..55c164b 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs @@ -49,7 +49,30 @@ public void Normalize_PrioritizesGlobalCandidatesBeforePrivateCandidates() Assert.Equal(new IPEndPoint(IPAddress.Parse("176.66.90.119"), 48705), normalized[0]); Assert.Contains(normalized, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993))); - } [Theory] + } + + [Fact] + public void Normalize_KeepsUpToPerScopeAndFamilyBudget() + { + var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); + var endpoints = Enumerable.Range(0, 10) + .Select(port => new IPEndPoint(IPAddress.Parse("176.66.90.119"), 40000 + port)) + .Concat(Enumerable.Range(0, 10) + .Select(port => new IPEndPoint(IPAddress.Parse("2001:4860:4860::8888"), 50000 + port))) + .ToArray(); + + var normalized = ZeroTierDirectEndpointSelection.Normalize( + endpoints, + relay, + maxEndpoints: 32, + maxPerScopeAndFamily: 8); + + Assert.Equal(16, normalized.Length); + Assert.Equal(8, normalized.Count(endpoint => endpoint.AddressFamily == AddressFamily.InterNetwork)); + Assert.Equal(8, normalized.Count(endpoint => endpoint.AddressFamily == AddressFamily.InterNetworkV6)); + } + + [Theory] [InlineData("10.0.0.1", true)] [InlineData("100.85.196.109", true)] [InlineData("172.17.0.1", true)] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index bac2a58..6a623fc 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -11,7 +11,8 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDirectEndpointManager { - private const int MaxEndpoints = 8; + private const int MaxEndpoints = 64; + private const int MaxEndpointsPerScopeAndFamily = 8; private const int RendezvousHolePunchHopLimit = 2; private const long HolePunchMinIntervalMs = 5_000; private const long HolePunchCacheTtlMs = 60_000; @@ -97,7 +98,8 @@ public async ValueTask HandleRendezvousFromRootAsync( [rendezvous.Endpoint], _relayEndpoint, maxEndpoints: MaxEndpoints, - _shouldAcceptEndpoint); + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); @@ -197,7 +199,8 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( merged, _relayEndpoint, maxEndpoints: MaxEndpoints, - _shouldAcceptEndpoint); + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); endpointsToProbe = endpoints .Where(endpoint => { @@ -246,7 +249,8 @@ public void SeedEndpoints(IEnumerable endpoints) _directEndpoints.Concat(endpoints), _relayEndpoint, maxEndpoints: MaxEndpoints, - _shouldAcceptEndpoint); + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); _directEndpoints = normalized; PrunePreferredLocalSockets_NoLock(normalized); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs index 02da240..36959f6 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs @@ -1,6 +1,7 @@ using System.Net; using System.Net.Sockets; using System.Globalization; +using System.Runtime.InteropServices; namespace ZTSharp.ZeroTier.Internal; @@ -10,17 +11,17 @@ public static IPEndPoint[] Normalize( IEnumerable endpoints, IPEndPoint relayEndpoint, int maxEndpoints, - Func? shouldInclude = null) + Func? shouldInclude = null, + int maxPerScopeAndFamily = int.MaxValue) { ArgumentNullException.ThrowIfNull(endpoints); ArgumentNullException.ThrowIfNull(relayEndpoint); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxEndpoints); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxPerScopeAndFamily); relayEndpoint = Canonicalize(relayEndpoint); - var global = new List(); - var shared = new List(); - var pseudoprivate = new List(); - var privateEndpoints = new List(); + var buckets = new Dictionary<(EndpointScope Scope, AddressFamily Family), List>(); foreach (var endpoint in endpoints) { @@ -50,42 +51,39 @@ public static IPEndPoint[] Normalize( continue; } - switch (GetScope(canonical.Address)) + var scope = GetScope(canonical.Address); + if (scope is not (EndpointScope.Global or EndpointScope.Shared or EndpointScope.Pseudoprivate or EndpointScope.Private)) { - case EndpointScope.Global: - global.Add(canonical); - break; - case EndpointScope.Shared: - shared.Add(canonical); - break; - case EndpointScope.Pseudoprivate: - pseudoprivate.Add(canonical); - break; - case EndpointScope.Private: - privateEndpoints.Add(canonical); - break; + continue; } - } - var ordered = global - .Concat(shared) - .Concat(pseudoprivate) - .Concat(privateEndpoints); + var key = (scope, canonical.AddressFamily); + ref var bucket = ref CollectionsMarshal.GetValueRefOrAddDefault(buckets, key, out _); + bucket ??= new List(); + bucket.Add(canonical); + } var unique = new List(); var seen = new HashSet(StringComparer.Ordinal); - foreach (var endpoint in ordered) + foreach (var scope in ScopePriority) { - var key = endpoint.Address + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); - if (!seen.Add(key)) + if (unique.Count >= maxEndpoints) { - continue; + break; } - unique.Add(endpoint); - if (unique.Count >= maxEndpoints) + for (var i = 0; i < FamilyPriority.Length; i++) { - break; + if (!buckets.TryGetValue((scope, FamilyPriority[i]), out var bucket)) + { + continue; + } + + AddUniqueEndpoints(bucket, unique, seen, maxEndpoints, maxPerScopeAndFamily); + if (unique.Count >= maxEndpoints) + { + break; + } } } @@ -112,6 +110,28 @@ private static IPEndPoint Canonicalize(IPEndPoint endpoint) return endpoint; } + private static void AddUniqueEndpoints( + List bucket, + List unique, + HashSet seen, + int maxEndpoints, + int maxPerScopeAndFamily) + { + var kept = 0; + for (var i = 0; i < bucket.Count && unique.Count < maxEndpoints && kept < maxPerScopeAndFamily; i++) + { + var endpoint = bucket[i]; + var key = endpoint.Address + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); + if (!seen.Add(key)) + { + continue; + } + + unique.Add(endpoint); + kept++; + } + } + public static bool IsPublicEndpoint(IPEndPoint endpoint) { ArgumentNullException.ThrowIfNull(endpoint); @@ -233,5 +253,19 @@ private enum EndpointScope Global, Multicast } + + private static readonly EndpointScope[] ScopePriority = + [ + EndpointScope.Global, + EndpointScope.Shared, + EndpointScope.Pseudoprivate, + EndpointScope.Private + ]; + + private static readonly AddressFamily[] FamilyPriority = + [ + AddressFamily.InterNetwork, + AddressFamily.InterNetworkV6 + ]; } From baccb793895aa2361a6a12e2daa0b6c290f97eb7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 00:59:46 +0100 Subject: [PATCH 163/296] Align push-direct probing with upstream --- .../ZeroTierDirectEndpointManagerPushFlagsTests.cs | 10 +++------- .../ZeroTier/Internal/ZeroTierDirectEndpointManager.cs | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 2c7b6c9..0c08583 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -79,12 +79,11 @@ public async Task PushDirectPaths_KnownEndpoint_IsNotRebootstrapped() await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); Assert.Single(hintedEndpoints); - Assert.Single(udp.Sends); - Assert.Equal(endpoint, udp.Sends[0].RemoteEndPoint); + Assert.Empty(udp.Sends); } [Fact] - public async Task PushDirectPaths_NewEndpoint_DoesNotRememberReceivingSocketAffinity() + public async Task PushDirectPaths_NewEndpoint_DoesNotHolePunchOrRememberReceivingSocketAffinity() { var udp = new RecordingUdpTransport(); @@ -99,10 +98,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); - var send = Assert.Single(udp.Sends); - Assert.Equal(0, send.LocalSocketId); - Assert.Equal(endpoint, send.RemoteEndPoint); - Assert.Null(send.HopLimit); + Assert.Empty(udp.Sends); } private const byte ZtPushDirectPathsFlagForgetPath = 0x01; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 6a623fc..8c0be62 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -228,7 +228,6 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( foreach (var endpoint in endpointsToProbe) { - TrySendHolePunch(endpoint); if (_handleDirectEndpointHintAsync is not null) { await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); From 48c34b1ad300285be60a9cf8d542c7ecb2c7f4f4 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:02:41 +0100 Subject: [PATCH 164/296] Gate private direct hints by local networks --- .../ZeroTierDirectEndpointPolicyTests.cs | 39 +++++ .../Internal/ZeroTierDataplaneRuntime.cs | 9 +- .../Internal/ZeroTierDirectEndpointPolicy.cs | 165 ++++++++++++++++++ 3 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs new file mode 100644 index 0000000..3b55016 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs @@ -0,0 +1,39 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointPolicyTests +{ + [Fact] + public void ShouldAccept_AcceptsPublicEndpoint() + { + var policy = new ZeroTierDirectEndpointPolicy(getLocalNetworks: static () => Array.Empty()); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_RejectsPrivateEndpointWithoutMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("192.168.1.10"), 24)]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsPrivateEndpointWithinMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("172.17.0.22"), 16)]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); + + Assert.True(accepted); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 9161204..49ec6b3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -49,6 +49,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; + private readonly ZeroTierDirectEndpointPolicy _directEndpointPolicy; private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _authenticatedPeers = new(); private readonly ConcurrentDictionary _directBootstrapTasks = new(); @@ -196,6 +197,7 @@ public ZeroTierDataplaneRuntime( _peerQos = new ZeroTierPeerQosManager(); _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); + _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); @@ -1728,12 +1730,7 @@ private ZeroTierDirectEndpointManager GetOrCreateDirectEndpointManager(NodeId pe private bool ShouldAcceptDirectEndpoint(IPEndPoint endpoint) { - if (IPAddress.IsLoopback(endpoint.Address)) - { - return true; - } - - if (ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)) + if (_directEndpointPolicy.ShouldAccept(endpoint)) { return true; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs new file mode 100644 index 0000000..a038520 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs @@ -0,0 +1,165 @@ +using System.Net; +using System.Net.NetworkInformation; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectEndpointPolicy +{ + private static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(30); + + private readonly Func _getLocalNetworks; + private readonly Func _nowMs; + private readonly object _lock = new(); + + private NetworkAddress[] _cachedLocalNetworks = Array.Empty(); + private long _nextRefreshMs; + + public ZeroTierDirectEndpointPolicy( + Func? getLocalNetworks = null, + Func? nowMs = null) + { + _getLocalNetworks = getLocalNetworks ?? GetLocalNetworks; + _nowMs = nowMs ?? (() => Environment.TickCount64); + } + + public bool ShouldAccept(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + if (IPAddress.IsLoopback(endpoint.Address)) + { + return true; + } + + if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)) + { + return false; + } + + if (!ZeroTierDirectEndpointSelection.IsPrivateEndpoint(endpoint)) + { + return true; + } + + var localNetworks = GetCachedLocalNetworks(); + for (var i = 0; i < localNetworks.Length; i++) + { + if (Contains(localNetworks[i], endpoint.Address)) + { + return true; + } + } + + return false; + } + + private NetworkAddress[] GetCachedLocalNetworks() + { + var now = _nowMs(); + lock (_lock) + { + if (_cachedLocalNetworks.Length == 0 || unchecked(now - _nextRefreshMs) >= 0) + { + _cachedLocalNetworks = _getLocalNetworks(); + _nextRefreshMs = now + (long)RefreshInterval.TotalMilliseconds; + } + + return _cachedLocalNetworks; + } + } + + private static NetworkAddress[] GetLocalNetworks() + { + var networks = new List(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + { + if (nic.OperationalStatus != OperationalStatus.Up) + { + continue; + } + + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + continue; + } + + foreach (var unicast in properties.UnicastAddresses) + { + if (unicast.Address is not { } address || IPAddress.IsLoopback(address)) + { + continue; + } + + var endpoint = new IPEndPoint(address, 1); + if (!ZeroTierDirectEndpointSelection.IsPrivateEndpoint(endpoint)) + { + continue; + } + + var prefixLength = GetPrefixLength(unicast); + if (prefixLength <= 0) + { + continue; + } + + networks.Add(new NetworkAddress(address, prefixLength)); + } + } + + return networks.ToArray(); + } + + private static int GetPrefixLength(UnicastIPAddressInformation unicast) + { + if (unicast.Address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + if (unicast.IPv4Mask is not { } mask) + { + return unicast.PrefixLength; + } + + return mask.GetAddressBytes().Sum(static octet => byte.PopCount(octet)); + } + + return unicast.PrefixLength; + } + + private static bool Contains(NetworkAddress network, IPAddress address) + { + if (network.Address.AddressFamily != address.AddressFamily) + { + return false; + } + + var networkBytes = network.Address.GetAddressBytes(); + var addressBytes = address.GetAddressBytes(); + if (networkBytes.Length != addressBytes.Length) + { + return false; + } + + var fullBytes = network.PrefixLength / 8; + for (var i = 0; i < fullBytes; i++) + { + if (networkBytes[i] != addressBytes[i]) + { + return false; + } + } + + var remainingBits = network.PrefixLength % 8; + if (remainingBits == 0) + { + return true; + } + + var mask = (byte)(0xFF << (8 - remainingBits)); + return (networkBytes[fullBytes] & mask) == (addressBytes[fullBytes] & mask); + } +} From 3e3ef8936df276a2fd04b4e11fae654d34a135b1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:05:59 +0100 Subject: [PATCH 165/296] Allow shared direct hints during bootstrap --- .../ZeroTierDirectEndpointPolicyTests.cs | 22 +++++++++++ .../Internal/ZeroTierDirectEndpointPolicy.cs | 38 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs index 3b55016..c4eab8d 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs @@ -26,6 +26,17 @@ public void ShouldAccept_RejectsPrivateEndpointWithoutMatchingLocalNetwork() Assert.False(accepted); } + [Fact] + public void ShouldAccept_AcceptsSharedEndpointWithoutMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("192.168.1.10"), 24)]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.True(accepted); + } + [Fact] public void ShouldAccept_AcceptsPrivateEndpointWithinMatchingLocalNetwork() { @@ -36,4 +47,15 @@ public void ShouldAccept_AcceptsPrivateEndpointWithinMatchingLocalNetwork() Assert.True(accepted); } + + [Fact] + public void ShouldAccept_RejectsUlaEndpointWithoutMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("2001:db8::1"), 64)]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993)); + + Assert.False(accepted); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs index a038520..e41b81c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs @@ -26,6 +26,7 @@ public ZeroTierDirectEndpointPolicy( public bool ShouldAccept(IPEndPoint endpoint) { ArgumentNullException.ThrowIfNull(endpoint); + endpoint = Canonicalize(endpoint); if (IPAddress.IsLoopback(endpoint.Address)) { @@ -37,7 +38,7 @@ public bool ShouldAccept(IPEndPoint endpoint) return false; } - if (!ZeroTierDirectEndpointSelection.IsPrivateEndpoint(endpoint)) + if (!RequiresLocalNetworkReachabilityCheck(endpoint.Address)) { return true; } @@ -97,7 +98,7 @@ private static NetworkAddress[] GetLocalNetworks() } var endpoint = new IPEndPoint(address, 1); - if (!ZeroTierDirectEndpointSelection.IsPrivateEndpoint(endpoint)) + if (!RequiresLocalNetworkReachabilityCheck(endpoint.Address)) { continue; } @@ -162,4 +163,37 @@ private static bool Contains(NetworkAddress network, IPAddress address) var mask = (byte)(0xFF << (8 - remainingBits)); return (networkBytes[fullBytes] & mask) == (addressBytes[fullBytes] & mask); } + + private static bool RequiresLocalNetworkReachabilityCheck(IPAddress address) + { + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + var bytes = address.GetAddressBytes(); + return bytes[0] switch + { + 10 => true, + 172 when bytes[1] is >= 16 and <= 31 => true, + 192 when bytes[1] == 168 => true, + _ => false + }; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + var bytes = address.GetAddressBytes(); + return bytes.Length == 16 && (bytes[0] & 0xFE) == 0xFC; + } + + return false; + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && endpoint.Address.IsIPv4MappedToIPv6 + ? new IPEndPoint(endpoint.Address.MapToIPv4(), endpoint.Port) + : endpoint; } From c021254dec9bbfa5f54545b13b0b61e5618bdac5 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:14:21 +0100 Subject: [PATCH 166/296] Trace pre-decode direct inbound traffic --- .../ZeroTierDataplanePeerDatagramProcessor.cs | 6 + .../Internal/ZeroTierDataplaneRuntime.cs | 3 + .../Internal/ZeroTierDataplaneRxLoops.cs | 9 ++ .../ZeroTierInboundDatagramDiagnostics.cs | 115 ++++++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierInboundDatagramDiagnostics.cs diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index ad793cd..462e4c8 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -17,6 +17,7 @@ internal sealed class ZeroTierDataplanePeerDatagramProcessor private readonly ZeroTierExternalSurfaceAddressTracker _surfaceAddresses; private readonly ZeroTierPeerQosManager _peerQos; private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; + private readonly ZeroTierInboundDatagramDiagnostics _diagnostics; private readonly bool _multipathEnabled; private readonly Action? _handleAuthenticatedPeer; private readonly Action? _handleHelloOk; @@ -31,6 +32,7 @@ public ZeroTierDataplanePeerDatagramProcessor( ZeroTierExternalSurfaceAddressTracker surfaceAddresses, ZeroTierPeerQosManager peerQos, ZeroTierPeerPathNegotiationManager peerNegotiation, + ZeroTierInboundDatagramDiagnostics diagnostics, bool multipathEnabled, Action? handleAuthenticatedPeer = null, Action? handleHelloOk = null, @@ -43,6 +45,7 @@ public ZeroTierDataplanePeerDatagramProcessor( ArgumentNullException.ThrowIfNull(surfaceAddresses); ArgumentNullException.ThrowIfNull(peerQos); ArgumentNullException.ThrowIfNull(peerNegotiation); + ArgumentNullException.ThrowIfNull(diagnostics); _localNodeId = localNodeId; _peerSecurity = peerSecurity; @@ -52,6 +55,7 @@ public ZeroTierDataplanePeerDatagramProcessor( _surfaceAddresses = surfaceAddresses; _peerQos = peerQos; _peerNegotiation = peerNegotiation; + _diagnostics = diagnostics; _multipathEnabled = multipathEnabled; _handleAuthenticatedPeer = handleAuthenticatedPeer; _handleHelloOk = handleHelloOk; @@ -114,12 +118,14 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c if (!_peerSecurity.TryGetPeerKey(peerNodeId, out var key)) { + _diagnostics.TracePeerKeyMissing(peerNodeId, datagram, decoded.Header.HopCount); _peerSecurity.EnsurePeerKeyAsync(peerNodeId); return; } if (!ZeroTierPacketCrypto.Dearmor(packetBytes, key)) { + _diagnostics.TraceDearmorFailure(peerNodeId, datagram, decoded.Header.HopCount); return; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 49ec6b3..c623446 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -199,6 +199,7 @@ public ZeroTierDataplaneRuntime( _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); + var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); var ip = new ZeroTierDataplaneIpHandler( @@ -221,6 +222,7 @@ public ZeroTierDataplaneRuntime( _surfaceAddresses, _peerQos, _peerNegotiation, + inboundDiagnostics, multipath.Enabled, HandleAuthenticatedPeer, HandlePeerHelloOk, @@ -233,6 +235,7 @@ public ZeroTierDataplaneRuntime( _localIdentity.NodeId, _rootClient, _peerDatagrams, + inboundDiagnostics, acceptDirectPeerDatagrams: multipath.Enabled, handleRootControlAsync: HandleRootControlPacketAsync, onPeerQueueDrop: () => diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index 66d8539..9640946 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -14,6 +14,7 @@ internal sealed class ZeroTierDataplaneRxLoops private readonly NodeId _localNodeId; private readonly ZeroTierDataplaneRootClient _rootClient; private readonly IZeroTierDataplanePeerDatagramProcessor _peerDatagrams; + private readonly ZeroTierInboundDatagramDiagnostics _diagnostics; private readonly Func, int, IPEndPoint, CancellationToken, ValueTask>? _handleRootControlAsync; private readonly Action? _onPeerQueueDrop; private readonly bool _acceptDirectPeerDatagrams; @@ -28,6 +29,7 @@ public ZeroTierDataplaneRxLoops( NodeId localNodeId, ZeroTierDataplaneRootClient rootClient, IZeroTierDataplanePeerDatagramProcessor peerDatagrams, + ZeroTierInboundDatagramDiagnostics? diagnostics = null, bool acceptDirectPeerDatagrams = false, Func, int, IPEndPoint, CancellationToken, ValueTask>? handleRootControlAsync = null, Action? onPeerQueueDrop = null) @@ -45,6 +47,7 @@ public ZeroTierDataplaneRxLoops( _localNodeId = localNodeId; _rootClient = rootClient; _peerDatagrams = peerDatagrams; + _diagnostics = diagnostics ?? new ZeroTierInboundDatagramDiagnostics(localNodeId, rootEndpoint, rawBudget: 0, dropBudget: 0); _acceptDirectPeerDatagrams = acceptDirectPeerDatagrams; _handleRootControlAsync = handleRootControlAsync; _onPeerQueueDrop = onPeerQueueDrop; @@ -74,6 +77,8 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca return; } + _diagnostics.TraceInboundRaw(datagram); + if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) { var peek = datagram.Payload.AsSpan(); @@ -94,14 +99,18 @@ public async Task DispatcherLoopAsync(Channel peerQueue, Ca var packetBytes = datagram.Payload; if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var decoded)) { + _diagnostics.TraceDecodeFailure(datagram); continue; } if (decoded.Header.Destination != _localNodeId) { + _diagnostics.TraceDestinationMismatch(datagram, decoded); continue; } + _diagnostics.TraceDecodedInbound(datagram, decoded); + if (ZeroTierTrace.Enabled && _traceRxRemaining > 0) { _traceRxRemaining--; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierInboundDatagramDiagnostics.cs b/ZTSharp/ZeroTier/Internal/ZeroTierInboundDatagramDiagnostics.cs new file mode 100644 index 0000000..99088ed --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierInboundDatagramDiagnostics.cs @@ -0,0 +1,115 @@ +using System.Net; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierInboundDatagramDiagnostics +{ + private readonly NodeId _localNodeId; + private readonly IPEndPoint _rootEndpoint; + private int _rawRemaining; + private int _dropRemaining; + + public ZeroTierInboundDatagramDiagnostics(NodeId localNodeId, IPEndPoint rootEndpoint, int rawBudget = 200, int dropBudget = 100) + { + ArgumentNullException.ThrowIfNull(rootEndpoint); + _localNodeId = localNodeId; + _rootEndpoint = rootEndpoint; + _rawRemaining = rawBudget; + _dropRemaining = dropBudget; + } + + public void TraceInboundRaw(ZeroTierUdpDatagram datagram) + { + if (!ShouldTraceRaw(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] RX direct raw: socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint} bytes={datagram.Payload.Length}."); + } + + public void TraceDecodeFailure(ZeroTierUdpDatagram datagram) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct raw: decode failed on socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint} bytes={datagram.Payload.Length}."); + } + + public void TraceDestinationMismatch(ZeroTierUdpDatagram datagram, in ZeroTierPacketView decoded) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct raw: destination mismatch src={decoded.Header.Source} dst={decoded.Header.Destination} local={_localNodeId} hop={decoded.Header.HopCount} via={datagram.RemoteEndPoint}."); + } + + public void TraceDecodedInbound(ZeroTierUdpDatagram datagram, in ZeroTierPacketView decoded) + { + if (!ShouldTraceRaw(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] RX direct decoded: src={decoded.Header.Source} dst={decoded.Header.Destination} hop={decoded.Header.HopCount} cipher={decoded.Header.CipherSuite} verbRaw=0x{decoded.Header.VerbRaw:x2} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint}."); + } + + public void TracePeerKeyMissing(NodeId peerNodeId, ZeroTierUdpDatagram datagram, byte hopCount) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct peer: missing shared key peer={peerNodeId} hop={hopCount} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint}."); + } + + public void TraceDearmorFailure(NodeId peerNodeId, ZeroTierUdpDatagram datagram, byte hopCount) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct peer: failed dearmor peer={peerNodeId} hop={hopCount} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint}."); + } + + private bool ShouldTraceRaw(IPEndPoint remoteEndPoint) + => ZeroTierTrace.Enabled && + !remoteEndPoint.Equals(_rootEndpoint) && + Consume(ref _rawRemaining); + + private bool ShouldTraceDrop(IPEndPoint remoteEndPoint) + => ZeroTierTrace.Enabled && + !remoteEndPoint.Equals(_rootEndpoint) && + Consume(ref _dropRemaining); + + private static bool Consume(ref int budget) + { + while (true) + { + var current = Volatile.Read(ref budget); + if (current <= 0) + { + return false; + } + + if (Interlocked.CompareExchange(ref budget, current - 1, current) == current) + { + return true; + } + } + } +} From 0e9709cb9fd8cae4c8f4eba9bf7858c1d3be10bb Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:21:09 +0100 Subject: [PATCH 167/296] Advertise local interface direct paths --- ...LocalDirectPathAdvertisementSourceTests.cs | 45 +++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 14 +- ...oTierLocalDirectPathAdvertisementSource.cs | 127 ++++++++++++++++++ 3 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs new file mode 100644 index 0000000..a6c5f4f --- /dev/null +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs @@ -0,0 +1,45 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierLocalDirectPathAdvertisementSourceTests +{ + [Fact] + public void GetSnapshot_CombinesAddressesWithSocketPorts() + { + var source = new ZeroTierLocalDirectPathAdvertisementSource( + getLocalAddresses: static () => + [ + IPAddress.Parse("10.0.0.112"), + IPAddress.Parse("100.74.185.14"), + IPAddress.Parse("2001:db8::1") + ]); + + var endpoints = source.GetSnapshot( + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 53585)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 53586)) + ]); + + Assert.Equal(6, endpoints.Length); + Assert.Contains(new IPEndPoint(IPAddress.Parse("10.0.0.112"), 53585), endpoints); + Assert.Contains(new IPEndPoint(IPAddress.Parse("100.74.185.14"), 53586), endpoints); + Assert.Contains(new IPEndPoint(IPAddress.Parse("2001:db8::1"), 53585), endpoints); + } + + [Fact] + public void GetSnapshot_ReturnsEmptyWhenNoSocketsHavePorts() + { + var source = new ZeroTierLocalDirectPathAdvertisementSource( + getLocalAddresses: static () => [IPAddress.Parse("10.0.0.112")]); + + var endpoints = source.GetSnapshot( + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 0)) + ]); + + Assert.Empty(endpoints); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c623446..107c9f5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -50,6 +50,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; private readonly ZeroTierDirectEndpointPolicy _directEndpointPolicy; + private readonly ZeroTierLocalDirectPathAdvertisementSource _localDirectPathAdvertisementsSource; private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _authenticatedPeers = new(); private readonly ConcurrentDictionary _directBootstrapTasks = new(); @@ -198,6 +199,7 @@ public ZeroTierDataplaneRuntime( _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); + _localDirectPathAdvertisementsSource = new ZeroTierLocalDirectPathAdvertisementSource(); _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); @@ -289,6 +291,7 @@ private IPEndPoint[] GetLocalDirectPathAdvertisements() { var endpoints = new List(_localDirectPathAdvertisements); var localSockets = _udp.LocalSockets; + endpoints.AddRange(_localDirectPathAdvertisementsSource.GetSnapshot(localSockets)); if (localSockets.Count == 0) { endpoints.AddRange(_surfaceAddresses.GetSnapshot(localSocketId: 0)); @@ -301,13 +304,10 @@ private IPEndPoint[] GetLocalDirectPathAdvertisements() } } - return endpoints - .Where(static endpoint => endpoint.Port != 0) - .Select(UdpEndpointNormalization.Normalize) - .Where(endpoint => !endpoint.Equals(_rootEndpoint)) - .Distinct() - .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) - .ToArray(); + return ZeroTierDirectEndpointSelection.Normalize( + endpoints.Where(static endpoint => endpoint.Port != 0).Select(UdpEndpointNormalization.Normalize), + _rootEndpoint, + maxEndpoints: ZeroTierProtocolLimits.MaxPushedDirectPaths); } private void SeedInitialExternalSurfaceObservations(IReadOnlyList? observations) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs new file mode 100644 index 0000000..0c5e4bc --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs @@ -0,0 +1,127 @@ +using System.Net; +using System.Net.NetworkInformation; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierLocalDirectPathAdvertisementSource +{ + private static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(30); + + private readonly Func _getLocalAddresses; + private readonly Func _nowMs; + private readonly object _lock = new(); + + private IPAddress[] _cachedLocalAddresses = Array.Empty(); + private long _nextRefreshMs; + + public ZeroTierLocalDirectPathAdvertisementSource( + Func? getLocalAddresses = null, + Func? nowMs = null) + { + _getLocalAddresses = getLocalAddresses ?? GetLocalAddresses; + _nowMs = nowMs ?? (() => Environment.TickCount64); + } + + public IPEndPoint[] GetSnapshot(IReadOnlyList localSockets) + { + ArgumentNullException.ThrowIfNull(localSockets); + + var localAddresses = GetCachedLocalAddresses(); + if (localAddresses.Length == 0 || localSockets.Count == 0) + { + return Array.Empty(); + } + + var endpoints = new List(localAddresses.Length * localSockets.Count); + for (var i = 0; i < localSockets.Count; i++) + { + var port = localSockets[i].LocalEndpoint.Port; + if (port == 0) + { + continue; + } + + for (var a = 0; a < localAddresses.Length; a++) + { + endpoints.Add(new IPEndPoint(localAddresses[a], port)); + } + } + + return endpoints + .Distinct() + .ToArray(); + } + + private IPAddress[] GetCachedLocalAddresses() + { + var now = _nowMs(); + lock (_lock) + { + if (_cachedLocalAddresses.Length == 0 || unchecked(now - _nextRefreshMs) >= 0) + { + _cachedLocalAddresses = _getLocalAddresses(); + _nextRefreshMs = now + (long)RefreshInterval.TotalMilliseconds; + } + + return _cachedLocalAddresses; + } + } + + private static IPAddress[] GetLocalAddresses() + { + var addresses = new List(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + { + if (nic.OperationalStatus != OperationalStatus.Up) + { + continue; + } + + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + continue; + } + + foreach (var unicast in properties.UnicastAddresses) + { + var address = Canonicalize(unicast.Address); + if (address is null || IPAddress.IsLoopback(address)) + { + continue; + } + + if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(new IPEndPoint(address, 1))) + { + continue; + } + + addresses.Add(address); + } + } + + return addresses + .Distinct() + .ToArray(); + } + + private static IPAddress? Canonicalize(IPAddress? address) + { + if (address is null) + { + return null; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + return address.MapToIPv4(); + } + + return address; + } +} From 55bccec78fcf08ed81556507ce8ac51e61e726bc Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:22:32 +0100 Subject: [PATCH 168/296] Fix local direct path source initialization --- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 107c9f5..81e084e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -194,12 +194,12 @@ public ZeroTierDataplaneRuntime( _peerPaths = new ZeroTierPeerPhysicalPathTracker(ttl: TimeSpan.FromSeconds(30)); _peerEcho = new ZeroTierPeerEchoManager(udp, localIdentity.NodeId, _peerSecurity.GetPeerProtocolVersionOrDefault); _surfaceAddresses = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromMinutes(30)); + _localDirectPathAdvertisementsSource = new ZeroTierLocalDirectPathAdvertisementSource(); SeedInitialExternalSurfaceObservations(initialExternalSurfaceObservations); _peerQos = new ZeroTierPeerQosManager(); _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); - _localDirectPathAdvertisementsSource = new ZeroTierLocalDirectPathAdvertisementSource(); _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); From 93519b6bb1f646501a27373ebe895a81c8df2fb6 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:27:26 +0100 Subject: [PATCH 169/296] Throttle hinted direct maintenance probes --- ...TierDirectEndpointManagerPushFlagsTests.cs | 38 +++++++++- .../ZeroTierDirectHintPathPlannerTests.cs | 69 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 11 ++- .../Internal/ZeroTierDirectEndpointManager.cs | 43 ++++++++++-- .../Internal/ZeroTierDirectHintPathPlanner.cs | 51 ++++++++++++++ 5 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 0c08583..e2e9562 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -66,7 +66,7 @@ public async Task PushDirectPaths_KnownEndpoint_IsNotRebootstrapped() udp, relay, peerNodeId, - handleDirectEndpointHintAsync: (_, _, endpoint, _) => + handleDirectEndpointHintAsync: (_, _, endpoint, _, _) => { hintedEndpoints.Add(endpoint); return ValueTask.CompletedTask; @@ -101,7 +101,38 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Empty(udp.Sends); } + [Fact] + public async Task PushDirectPaths_ClusterRedirect_UsesReceivingSocketAndForcesFullHello() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, _) => + { + hints.Add((endpoint, forceFullHello, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagClusterRedirect), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, true, 1), hints[0]); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(udp.Sends); + } + private const byte ZtPushDirectPathsFlagForgetPath = 0x01; + private const byte ZtPushDirectPathsFlagClusterRedirect = 0x02; private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flags) { @@ -141,7 +172,10 @@ private sealed class RecordingUdpTransport : IZeroTierUdpTransport public List<(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit)> Sends { get; } = new(); public IReadOnlyList LocalSockets { get; } = - new[] { new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Loopback, 0)) }; + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Loopback, 0)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Loopback, 0)) + ]; public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs new file mode 100644 index 0000000..35a77ed --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs @@ -0,0 +1,69 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectHintPathPlannerTests +{ + [Fact] + public void GetNextHintedCandidatesForMaintenance_RotatesSockets_AndSkipsObservedPaths() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint1 = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + var endpoint2 = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var endpoint3 = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + + var udp = new StubUdpTransport([0, 1]); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + manager.SeedEndpoints([endpoint1, endpoint2, endpoint3]); + + var planner = new ZeroTierDirectHintPathPlanner( + udp, + new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)), + _ => manager); + + var first = planner.GetNextHintedCandidatesForMaintenance( + peerNodeId, + [new ZeroTierPeerPhysicalPath(0, endpoint1, LastSeenUnixMs: 1)], + endpointBudget: 3); + var second = planner.GetNextHintedCandidatesForMaintenance(peerNodeId, Array.Empty(), endpointBudget: 2); + + Assert.Equal( + [new ZeroTierSelectedPeerPath(0, endpoint2), new ZeroTierSelectedPeerPath(0, endpoint3)], + first); + Assert.Equal( + [new ZeroTierSelectedPeerPath(1, endpoint1), new ZeroTierSelectedPeerPath(1, endpoint2)], + second); + } + + private sealed class StubUdpTransport : IZeroTierUdpTransport + { + public StubUdpTransport(int[] socketIds) + { + LocalSockets = socketIds + .Select(id => new ZeroTierUdpLocalSocket(id, new IPEndPoint(IPAddress.Loopback, 10000 + id))) + .ToArray(); + } + + public IReadOnlyList LocalSockets { get; } + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 81e084e..f427242 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -22,6 +22,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const long DirectBootstrapPathPushIntervalHavePathMs = 120_000; private const long DirectBootstrapHintProbeIntervalMs = 1_000; private const int DirectBootstrapHintProbeBudget = 2; + private const int DirectHintMaintenanceProbeBudget = 2; private const long DirectHintFullHelloIntervalMs = 60_000; private readonly IZeroTierUdpTransport _udp; @@ -1009,6 +1010,7 @@ private async ValueTask HandleDirectEndpointHintAsync( NodeId peerNodeId, int receivedLocalSocketId, IPEndPoint endpoint, + bool forceFullHello, CancellationToken cancellationToken) { if (!_multipath.Enabled) @@ -1034,7 +1036,7 @@ private async ValueTask HandleDirectEndpointHintAsync( if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId}."); + ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId} fullHello={forceFullHello}."); } var localSocketIds = _directHintPlanner.GetRotatingSocketIds( @@ -1045,7 +1047,7 @@ private async ValueTask HandleDirectEndpointHintAsync( { await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) .ConfigureAwait(false); - if (!CanUseEchoForDirectBootstrap(peerNodeId)) + if (forceFullHello || !CanUseEchoForDirectBootstrap(peerNodeId)) { await SendHelloPacketAsync( localSocketIds[i], @@ -1465,7 +1467,10 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); var paths = _peerPaths.GetSnapshot(peerNodeId); - var hintedCandidates = _directHintPlanner.GetHintedCandidates(peerNodeId, paths); + var hintedCandidates = _directHintPlanner.GetNextHintedCandidatesForMaintenance( + peerNodeId, + paths, + DirectHintMaintenanceProbeBudget); if (paths.Length == 0 && hintedCandidates.Length == 0) { continue; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 8c0be62..ea98721 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -26,7 +26,7 @@ internal sealed class ZeroTierDirectEndpointManager private readonly IZeroTierUdpTransport _udp; private readonly IPEndPoint _relayEndpoint; private readonly NodeId _remoteNodeId; - private readonly Func? _handleDirectEndpointHintAsync; + private readonly Func? _handleDirectEndpointHintAsync; private readonly Func? _shouldAcceptEndpoint; private readonly object _lock = new(); @@ -41,7 +41,7 @@ public ZeroTierDirectEndpointManager( IZeroTierUdpTransport udp, IPEndPoint relayEndpoint, NodeId remoteNodeId, - Func? handleDirectEndpointHintAsync = null, + Func? handleDirectEndpointHintAsync = null, Func? shouldAcceptEndpoint = null) { ArgumentNullException.ThrowIfNull(udp); @@ -119,7 +119,7 @@ public async ValueTask HandleRendezvousFromRootAsync( hopLimit: RendezvousHolePunchHopLimit); if (_handleDirectEndpointHintAsync is not null) { - await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); + await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, false, cancellationToken).ConfigureAwait(false); } } @@ -223,14 +223,15 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] RX PUSH_DIRECT_PATHS: endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} (candidates: {paths.Length})."); + ZeroTierTrace.WriteLine($"[zerotier] RX PUSH_DIRECT_PATHS: {FormatPaths(paths)} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)}."); } foreach (var endpoint in endpointsToProbe) { if (_handleDirectEndpointHintAsync is not null) { - await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, cancellationToken).ConfigureAwait(false); + var forceFullHello = redirectKeys.Contains(FormatEndpointKey(endpoint)); + await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, forceFullHello, cancellationToken).ConfigureAwait(false); } } @@ -283,6 +284,37 @@ private bool RateGatePushDirectPaths(long nowMs) } } + private static string FormatPaths(ZeroTierPushedDirectPath[] paths) + { + if (paths.Length == 0) + { + return ""; + } + + return string.Join(", ", paths.Select(static path => $"{FormatFlags(path.Flags)}{path.Endpoint}")); + } + + private static string FormatFlags(byte flags) + { + if (flags == 0) + { + return string.Empty; + } + + var parts = new List(2); + if ((flags & PushDirectPathsFlagForgetPath) != 0) + { + parts.Add("forget"); + } + + if ((flags & PushDirectPathsFlagClusterRedirect) != 0) + { + parts.Add("redirect"); + } + + return "[" + string.Join("|", parts) + "] "; + } + private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketIds = null, int? hopLimit = null) { var localSockets = _udp.LocalSockets; @@ -472,4 +504,3 @@ private static string FormatEndpointKey(IPEndPoint endpoint) } } - diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs index f01d89e..fea7720 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -88,6 +88,56 @@ public ZeroTierSelectedPeerPath[] GetHintedCandidates(NodeId peerNodeId, ZeroTie return candidates.ToArray(); } + public ZeroTierSelectedPeerPath[] GetNextHintedCandidatesForMaintenance( + NodeId peerNodeId, + ZeroTierPeerPhysicalPath[] observedPaths, + int endpointBudget) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(endpointBudget); + + var hinted = _getDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length == 0) + { + return Array.Empty(); + } + + HashSet? observed = null; + if (observedPaths.Length > 0) + { + observed = observedPaths + .Select(static path => new ZeroTierPeerPhysicalPathKey(path.LocalSocketId, path.RemoteEndPoint)) + .ToHashSet(); + } + + var selectedEndpoints = TakeNextHintedEndpoints(peerNodeId, hinted, endpointBudget); + var unique = new HashSet(); + var candidates = new List(selectedEndpoints.Length); + for (var i = 0; i < selectedEndpoints.Length; i++) + { + var localSocketIds = GetRotatingSocketIds( + peerNodeId, + selectedEndpoints[i], + includeFallbackLocalSockets: true); + for (var s = 0; s < localSocketIds.Length; s++) + { + var key = new ZeroTierPeerPhysicalPathKey(localSocketIds[s], selectedEndpoints[i]); + if (observed is not null && observed.Contains(key)) + { + continue; + } + + if (!unique.Add(key)) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(localSocketIds[s], selectedEndpoints[i])); + } + } + + return candidates.ToArray(); + } + public void RemovePeer(NodeId peerNodeId) { _probeCursors.TryRemove(peerNodeId, out _); @@ -180,3 +230,4 @@ public int TakeNextSocketIndex(IPEndPoint endpoint, int socketCount) } } } + From dea236bb2943fa51453788ad7c8104bbf0f7fc6d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:31:23 +0100 Subject: [PATCH 170/296] Filter local direct path advertisements --- ...LocalDirectPathAdvertisementSourceTests.cs | 39 ++++++---------- ...oTierLocalDirectPathAdvertisementSource.cs | 46 ++++++++++++++++++- 2 files changed, 60 insertions(+), 25 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs index a6c5f4f..7c4e2ef 100644 --- a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs @@ -7,39 +7,30 @@ namespace ZTSharp.Tests; public sealed class ZeroTierLocalDirectPathAdvertisementSourceTests { [Fact] - public void GetSnapshot_CombinesAddressesWithSocketPorts() + public void GetSnapshot_ExcludesPrivateAndUlaLocalAddresses() { var source = new ZeroTierLocalDirectPathAdvertisementSource( - getLocalAddresses: static () => + getLocalAddresses: () => [ - IPAddress.Parse("10.0.0.112"), + IPAddress.Parse("212.241.85.84"), IPAddress.Parse("100.74.185.14"), - IPAddress.Parse("2001:db8::1") + IPAddress.Parse("10.5.0.2"), + IPAddress.Parse("172.25.128.1"), + IPAddress.Parse("192.168.224.1"), + IPAddress.Parse("fd7a:115c:a1e0::5132:b90e") ]); var endpoints = source.GetSnapshot( [ - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 53585)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 53586)) + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 51158)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 51159)) ]); - Assert.Equal(6, endpoints.Length); - Assert.Contains(new IPEndPoint(IPAddress.Parse("10.0.0.112"), 53585), endpoints); - Assert.Contains(new IPEndPoint(IPAddress.Parse("100.74.185.14"), 53586), endpoints); - Assert.Contains(new IPEndPoint(IPAddress.Parse("2001:db8::1"), 53585), endpoints); - } - - [Fact] - public void GetSnapshot_ReturnsEmptyWhenNoSocketsHavePorts() - { - var source = new ZeroTierLocalDirectPathAdvertisementSource( - getLocalAddresses: static () => [IPAddress.Parse("10.0.0.112")]); - - var endpoints = source.GetSnapshot( - [ - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 0)) - ]); - - Assert.Empty(endpoints); + Assert.Contains(endpoints, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 51158))); + Assert.Contains(endpoints, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("100.74.185.14"), 51159))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("10.5.0.2"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("172.25.128.1"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("192.168.224.1"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("fd7a:115c:a1e0::5132:b90e"))); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs index 0c5e4bc..2a00495 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs @@ -44,6 +44,11 @@ public IPEndPoint[] GetSnapshot(IReadOnlyList localSocke for (var a = 0; a < localAddresses.Length; a++) { + if (!ShouldAdvertise(localAddresses[a])) + { + continue; + } + endpoints.Add(new IPEndPoint(localAddresses[a], port)); } } @@ -96,7 +101,7 @@ private static IPAddress[] GetLocalAddresses() continue; } - if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(new IPEndPoint(address, 1))) + if (!ShouldAdvertise(address)) { continue; } @@ -110,6 +115,44 @@ private static IPAddress[] GetLocalAddresses() .ToArray(); } + private static bool ShouldAdvertise(IPAddress address) + { + if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(new IPEndPoint(address, 1))) + { + return false; + } + + return !RequiresLocalNetworkReachabilityCheck(address); + } + + private static bool RequiresLocalNetworkReachabilityCheck(IPAddress address) + { + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + var bytes = address.GetAddressBytes(); + return bytes[0] switch + { + 10 => true, + 172 when bytes[1] is >= 16 and <= 31 => true, + 192 when bytes[1] == 168 => true, + _ => false + }; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + var bytes = address.GetAddressBytes(); + return bytes.Length == 16 && (bytes[0] & 0xFE) == 0xFC; + } + + return false; + } + private static IPAddress? Canonicalize(IPAddress? address) { if (address is null) @@ -125,3 +168,4 @@ private static IPAddress[] GetLocalAddresses() return address; } } + From 850d703aa605d87719fadd5921a4b930e9585c33 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:36:21 +0100 Subject: [PATCH 171/296] Loosen remote path hints and trim local ads --- .../ZeroTierDirectEndpointPolicyTests.cs | 28 ++- ...LocalDirectPathAdvertisementSourceTests.cs | 4 +- .../Internal/ZeroTierDirectEndpointPolicy.cs | 177 +----------------- ...oTierLocalDirectPathAdvertisementSource.cs | 37 +--- 4 files changed, 17 insertions(+), 229 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs index c4eab8d..f431592 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs @@ -8,7 +8,7 @@ public sealed class ZeroTierDirectEndpointPolicyTests [Fact] public void ShouldAccept_AcceptsPublicEndpoint() { - var policy = new ZeroTierDirectEndpointPolicy(getLocalNetworks: static () => Array.Empty()); + var policy = new ZeroTierDirectEndpointPolicy(); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); @@ -16,21 +16,19 @@ public void ShouldAccept_AcceptsPublicEndpoint() } [Fact] - public void ShouldAccept_RejectsPrivateEndpointWithoutMatchingLocalNetwork() + public void ShouldAccept_AcceptsPrivateEndpoint() { - var policy = new ZeroTierDirectEndpointPolicy( - getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("192.168.1.10"), 24)]); + var policy = new ZeroTierDirectEndpointPolicy(); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); - Assert.False(accepted); + Assert.True(accepted); } [Fact] - public void ShouldAccept_AcceptsSharedEndpointWithoutMatchingLocalNetwork() + public void ShouldAccept_AcceptsSharedEndpoint() { - var policy = new ZeroTierDirectEndpointPolicy( - getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("192.168.1.10"), 24)]); + var policy = new ZeroTierDirectEndpointPolicy(); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); @@ -38,23 +36,21 @@ public void ShouldAccept_AcceptsSharedEndpointWithoutMatchingLocalNetwork() } [Fact] - public void ShouldAccept_AcceptsPrivateEndpointWithinMatchingLocalNetwork() + public void ShouldAccept_AcceptsUlaEndpoint() { - var policy = new ZeroTierDirectEndpointPolicy( - getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("172.17.0.22"), 16)]); + var policy = new ZeroTierDirectEndpointPolicy(); - var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993)); Assert.True(accepted); } [Fact] - public void ShouldAccept_RejectsUlaEndpointWithoutMatchingLocalNetwork() + public void ShouldAccept_RejectsLinkLocalEndpoint() { - var policy = new ZeroTierDirectEndpointPolicy( - getLocalNetworks: static () => [new NetworkAddress(IPAddress.Parse("2001:db8::1"), 64)]); + var policy = new ZeroTierDirectEndpointPolicy(); - var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993)); + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fe80::1"), 9993)); Assert.False(accepted); } diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs index 7c4e2ef..4d395e1 100644 --- a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs @@ -7,7 +7,7 @@ namespace ZTSharp.Tests; public sealed class ZeroTierLocalDirectPathAdvertisementSourceTests { [Fact] - public void GetSnapshot_ExcludesPrivateAndUlaLocalAddresses() + public void GetSnapshot_ExcludesNonPublicLocalAddresses() { var source = new ZeroTierLocalDirectPathAdvertisementSource( getLocalAddresses: () => @@ -27,7 +27,7 @@ public void GetSnapshot_ExcludesPrivateAndUlaLocalAddresses() ]); Assert.Contains(endpoints, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 51158))); - Assert.Contains(endpoints, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("100.74.185.14"), 51159))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("100.74.185.14"))); Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("10.5.0.2"))); Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("172.25.128.1"))); Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("192.168.224.1"))); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs index e41b81c..5ea4a67 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs @@ -1,27 +1,10 @@ using System.Net; -using System.Net.NetworkInformation; -using ZTSharp.ZeroTier.Transport; namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDirectEndpointPolicy { - private static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(30); - - private readonly Func _getLocalNetworks; - private readonly Func _nowMs; - private readonly object _lock = new(); - - private NetworkAddress[] _cachedLocalNetworks = Array.Empty(); - private long _nextRefreshMs; - - public ZeroTierDirectEndpointPolicy( - Func? getLocalNetworks = null, - Func? nowMs = null) - { - _getLocalNetworks = getLocalNetworks ?? GetLocalNetworks; - _nowMs = nowMs ?? (() => Environment.TickCount64); - } + private readonly bool _acceptUsableEndpoints = true; public bool ShouldAccept(IPEndPoint endpoint) { @@ -33,163 +16,7 @@ public bool ShouldAccept(IPEndPoint endpoint) return true; } - if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)) - { - return false; - } - - if (!RequiresLocalNetworkReachabilityCheck(endpoint.Address)) - { - return true; - } - - var localNetworks = GetCachedLocalNetworks(); - for (var i = 0; i < localNetworks.Length; i++) - { - if (Contains(localNetworks[i], endpoint.Address)) - { - return true; - } - } - - return false; - } - - private NetworkAddress[] GetCachedLocalNetworks() - { - var now = _nowMs(); - lock (_lock) - { - if (_cachedLocalNetworks.Length == 0 || unchecked(now - _nextRefreshMs) >= 0) - { - _cachedLocalNetworks = _getLocalNetworks(); - _nextRefreshMs = now + (long)RefreshInterval.TotalMilliseconds; - } - - return _cachedLocalNetworks; - } - } - - private static NetworkAddress[] GetLocalNetworks() - { - var networks = new List(); - foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) - { - if (nic.OperationalStatus != OperationalStatus.Up) - { - continue; - } - - IPInterfaceProperties properties; - try - { - properties = nic.GetIPProperties(); - } - catch (NetworkInformationException) - { - continue; - } - - foreach (var unicast in properties.UnicastAddresses) - { - if (unicast.Address is not { } address || IPAddress.IsLoopback(address)) - { - continue; - } - - var endpoint = new IPEndPoint(address, 1); - if (!RequiresLocalNetworkReachabilityCheck(endpoint.Address)) - { - continue; - } - - var prefixLength = GetPrefixLength(unicast); - if (prefixLength <= 0) - { - continue; - } - - networks.Add(new NetworkAddress(address, prefixLength)); - } - } - - return networks.ToArray(); - } - - private static int GetPrefixLength(UnicastIPAddressInformation unicast) - { - if (unicast.Address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) - { - if (unicast.IPv4Mask is not { } mask) - { - return unicast.PrefixLength; - } - - return mask.GetAddressBytes().Sum(static octet => byte.PopCount(octet)); - } - - return unicast.PrefixLength; - } - - private static bool Contains(NetworkAddress network, IPAddress address) - { - if (network.Address.AddressFamily != address.AddressFamily) - { - return false; - } - - var networkBytes = network.Address.GetAddressBytes(); - var addressBytes = address.GetAddressBytes(); - if (networkBytes.Length != addressBytes.Length) - { - return false; - } - - var fullBytes = network.PrefixLength / 8; - for (var i = 0; i < fullBytes; i++) - { - if (networkBytes[i] != addressBytes[i]) - { - return false; - } - } - - var remainingBits = network.PrefixLength % 8; - if (remainingBits == 0) - { - return true; - } - - var mask = (byte)(0xFF << (8 - remainingBits)); - return (networkBytes[fullBytes] & mask) == (addressBytes[fullBytes] & mask); - } - - private static bool RequiresLocalNetworkReachabilityCheck(IPAddress address) - { - if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) - { - address = address.MapToIPv4(); - } - - if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) - { - var bytes = address.GetAddressBytes(); - return bytes[0] switch - { - 10 => true, - 172 when bytes[1] is >= 16 and <= 31 => true, - 192 when bytes[1] == 168 => true, - _ => false - }; - } - - if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) - { - var bytes = address.GetAddressBytes(); - return bytes.Length == 16 && (bytes[0] & 0xFE) == 0xFC; - } - - return false; + return _acceptUsableEndpoints && ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint); } private static IPEndPoint Canonicalize(IPEndPoint endpoint) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs index 2a00495..21a3fe3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs @@ -116,42 +116,7 @@ private static IPAddress[] GetLocalAddresses() } private static bool ShouldAdvertise(IPAddress address) - { - if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(new IPEndPoint(address, 1))) - { - return false; - } - - return !RequiresLocalNetworkReachabilityCheck(address); - } - - private static bool RequiresLocalNetworkReachabilityCheck(IPAddress address) - { - if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) - { - address = address.MapToIPv4(); - } - - if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) - { - var bytes = address.GetAddressBytes(); - return bytes[0] switch - { - 10 => true, - 172 when bytes[1] is >= 16 and <= 31 => true, - 192 when bytes[1] == 168 => true, - _ => false - }; - } - - if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) - { - var bytes = address.GetAddressBytes(); - return bytes.Length == 16 && (bytes[0] & 0xFE) == 0xFC; - } - - return false; - } + => ZeroTierDirectEndpointSelection.IsPublicEndpoint(new IPEndPoint(address, 1)); private static IPAddress? Canonicalize(IPAddress? address) { From a6d28b414f962ab9aa59370f1c99d5f0bd339d25 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:49:28 +0100 Subject: [PATCH 172/296] Improve direct bootstrap compatibility --- .../ZeroTierUdpMultiTransportTests.cs | 24 ++++++ .../ZeroTierDataplanePeerDatagramProcessor.cs | 7 +- .../Internal/ZeroTierDataplanePeerSecurity.cs | 46 ++++++++++-- .../Internal/ZeroTierDataplaneRuntime.cs | 75 +++++++++++-------- .../Transport/ZeroTierUdpMultiTransport.cs | 31 +++++++- 5 files changed, 141 insertions(+), 42 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs b/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs index 756c7f9..20e124e 100644 --- a/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs @@ -52,6 +52,30 @@ public async Task SendAsync_UsesSelectedSocket_AsUdpSourceEndpoint() Assert.Equal(socket2.LocalEndpoint.Port, datagram2.RemoteEndPoint.Port); } + [Fact] + public async Task SendAsync_WithoutSocketId_FansOutAcrossAllSockets() + { + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + await using var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + await using var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + await using var receiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + var receiverEndpoint = TestUdpEndpoints.ToLoopback(receiver.LocalEndpoint); + + await multi.SendAsync(receiverEndpoint, new byte[] { 0x33 }); + + var seenPorts = new HashSet(); + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(2); + while (seenPorts.Count < 2 && DateTimeOffset.UtcNow < deadline) + { + var datagram = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); + seenPorts.Add(datagram.RemoteEndPoint.Port); + } + + Assert.Contains(socket1.LocalEndpoint.Port, seenPorts); + Assert.Contains(socket2.LocalEndpoint.Port, seenPorts); + } + [Fact] public async Task DisposeAsync_DisposesUnderlyingSockets_AndMarksTransportDisposed() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index 462e4c8..27edb44 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -230,7 +230,12 @@ await _peerEcho { if (ZeroTierHelloOkParser.TryParseDecryptedOkHello(packetBytes, out var ok)) { - _peerSecurity.ObservePeerProtocolVersion(peerNodeId, ok.RemoteProtocolVersion); + _peerSecurity.ObservePeerVersion( + peerNodeId, + ok.RemoteProtocolVersion, + ok.RemoteMajorVersion, + ok.RemoteMinorVersion, + ok.RemoteRevision); if (_handleHelloOk is not null) { _handleHelloOk( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs index 7005805..d515d77 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs @@ -13,6 +13,15 @@ internal readonly record struct ZeroTierInboundHelloPayload( byte PeerProtocolVersion, IPEndPoint? ReportedLocalSurfaceAddress); +internal readonly record struct ZeroTierPeerVersion( + byte ProtocolVersion, + byte MajorVersion, + byte MinorVersion, + ushort Revision) +{ + public static readonly ZeroTierPeerVersion None; +} + internal sealed class ZeroTierDataplanePeerSecurity : IDisposable { private const int HelloPayloadMinLength = 13 + (5 + 1 + ZeroTierIdentity.PublicKeyLength + 1); @@ -27,7 +36,7 @@ internal sealed class ZeroTierDataplanePeerSecurity : IDisposable private readonly ConcurrentDictionary _peerKeys = new(); private readonly ConcurrentDictionary> _inflightKeys = new(); - private readonly ConcurrentDictionary _peerProtocolVersions = new(); + private readonly ConcurrentDictionary _peerVersions = new(); private readonly CancellationTokenSource _cts = new(); private int _trimPeerKeysInProgress; @@ -52,15 +61,22 @@ public ZeroTierDataplanePeerSecurity( } public byte GetPeerProtocolVersionOrDefault(NodeId peerNodeId) - => _peerProtocolVersions.TryGetValue(peerNodeId, out var version) ? version : (byte)0; + => GetPeerVersionOrDefault(peerNodeId).ProtocolVersion; - internal void ObservePeerProtocolVersion(NodeId peerNodeId, byte peerProtocolVersion) + public ZeroTierPeerVersion GetPeerVersionOrDefault(NodeId peerNodeId) + => _peerVersions.TryGetValue(peerNodeId, out var version) ? version : ZeroTierPeerVersion.None; + + internal void ObservePeerVersion(NodeId peerNodeId, byte peerProtocolVersion, byte peerMajorVersion, byte peerMinorVersion, ushort peerRevision) { peerProtocolVersion = peerProtocolVersion <= ZeroTierHelloClient.AdvertisedProtocolVersion ? peerProtocolVersion : ZeroTierHelloClient.AdvertisedProtocolVersion; - _peerProtocolVersions[peerNodeId] = peerProtocolVersion; + _peerVersions[peerNodeId] = new ZeroTierPeerVersion( + peerProtocolVersion, + peerMajorVersion, + peerMinorVersion, + peerRevision); } public bool TryGetPeerKey(NodeId peerNodeId, out byte[] key) @@ -139,6 +155,9 @@ public async Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken c } var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(5, 8)); + var peerMajorVersion = payload[1]; + var peerMinorVersion = payload[2]; + var peerRevision = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(3, 2)); ZeroTierIdentity identity; try @@ -190,7 +209,8 @@ public async Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken c CachePeerKey(peerNodeId, sharedKey, nowMs: Environment.TickCount64); var peerProtocolVersion = payload[0]; - ObservePeerProtocolVersion(peerNodeId, peerProtocolVersion); + ObservePeerVersion(peerNodeId, peerProtocolVersion, peerMajorVersion, peerMinorVersion, peerRevision); + var reportedRemoteSurface = NormalizeExternalSurfaceForPeer(remoteEndPoint, peerProtocolVersion); var okPacket = ZeroTierHelloOkPacketBuilder.BuildPacket( packetId: ZeroTierPacketIdGenerator.GeneratePacketId(), @@ -198,7 +218,7 @@ public async Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken c source: _localNodeId, inRePacketId: helloPacketId, helloTimestampEcho: helloTimestamp, - externalSurfaceAddress: remoteEndPoint, + externalSurfaceAddress: reportedRemoteSurface, sharedKey: ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, peerProtocolVersion)); try @@ -359,7 +379,7 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) if (entry.ExpiresAtUnixMs <= nowMs) { _peerKeys.TryRemove(peerNodeId, out _); - _peerProtocolVersions.TryRemove(peerNodeId, out _); + _peerVersions.TryRemove(peerNodeId, out _); } } @@ -378,7 +398,7 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) } _peerKeys.TryRemove(peerNodeId, out _); - _peerProtocolVersions.TryRemove(peerNodeId, out _); + _peerVersions.TryRemove(peerNodeId, out _); toRemove--; } } @@ -388,5 +408,15 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) } } + private static IPEndPoint NormalizeExternalSurfaceForPeer(IPEndPoint remoteEndPoint, byte peerProtocolVersion) + { + if (peerProtocolVersion >= 5 || remoteEndPoint.Port == 0) + { + return remoteEndPoint; + } + + return new IPEndPoint(remoteEndPoint.Address, 0); + } + private readonly record struct PeerKeyCacheEntry(byte[]? Key, long ExpiresAtUnixMs); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index f427242..4a64da7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -778,19 +778,15 @@ private async Task ProbeHintedDirectEndpointsAsync( includeFallbackLocalSockets: true); for (var s = 0; s < localSocketIds.Length; s++) { - await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpointsToProbe[i], sharedKey, cancellationToken) + await SendDirectBootstrapProbeAsync( + peerNodeId, + localSocketIds[s], + endpointsToProbe[i], + sharedKey, + forceFullHello: false, + DirectHelloMinIntervalMs, + cancellationToken) .ConfigureAwait(false); - if (!CanUseEchoForDirectBootstrap(peerNodeId)) - { - await SendHelloPacketAsync( - localSocketIds[s], - peerNodeId, - endpointsToProbe[i], - endpointsToProbe[i], - sharedKey, - cancellationToken) - .ConfigureAwait(false); - } } } } @@ -1045,24 +1041,43 @@ private async ValueTask HandleDirectEndpointHintAsync( includeFallbackLocalSockets: false); for (var i = 0; i < localSocketIds.Length; i++) { - await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[i], endpoint, sharedKey, cancellationToken) + await SendDirectBootstrapProbeAsync( + peerNodeId, + localSocketIds[i], + endpoint, + sharedKey, + forceFullHello, + DirectHelloMinIntervalMs, + cancellationToken) .ConfigureAwait(false); - if (forceFullHello || !CanUseEchoForDirectBootstrap(peerNodeId)) - { - await SendHelloPacketAsync( - localSocketIds[i], - peerNodeId, - endpoint, - endpoint, - sharedKey, - cancellationToken) - .ConfigureAwait(false); - } } } private bool CanUseEchoForDirectBootstrap(NodeId peerNodeId) - => _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId) >= 5; + { + var version = _peerSecurity.GetPeerVersionOrDefault(peerNodeId); + return version.ProtocolVersion >= 5 && + !(version.MajorVersion == 1 && version.MinorVersion == 1 && version.Revision == 0); + } + + private Task SendDirectBootstrapProbeAsync( + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + byte[] sharedKey, + bool forceFullHello, + long helloMinIntervalMs, + CancellationToken cancellationToken) + => forceFullHello || !CanUseEchoForDirectBootstrap(peerNodeId) + ? SendHelloPacketAsync( + localSocketId, + peerNodeId, + remoteEndPoint, + remoteEndPoint, + sharedKey, + helloMinIntervalMs, + cancellationToken) + : TrySendEchoDirectProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken); private async Task TrySendEchoDirectProbeAsync( NodeId peerNodeId, @@ -1480,16 +1495,12 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati { var candidate = hintedCandidates[c]; - await _peerEcho - .TrySendEchoProbeAsync(peerNodeId, candidate.LocalSocketId, candidate.RemoteEndPoint, key, cancellationToken) - .ConfigureAwait(false); - - await SendHelloPacketAsync( - candidate.LocalSocketId, + await SendDirectBootstrapProbeAsync( peerNodeId, - candidate.RemoteEndPoint, + candidate.LocalSocketId, candidate.RemoteEndPoint, key, + forceFullHello: false, DirectHintFullHelloIntervalMs, cancellationToken) .ConfigureAwait(false); diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs index f982e28..8bf9582 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs @@ -63,7 +63,7 @@ public async ValueTask ReceiveAsync(TimeSpan timeout, Cance public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); - return _sockets[0].SendAsync(remoteEndpoint, payload, cancellationToken); + return SendAllAsync(remoteEndpoint, payload, cancellationToken); } public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) @@ -165,6 +165,35 @@ private ZeroTierUdpTransport GetSocket(int localSocketId) throw new ArgumentOutOfRangeException(nameof(localSocketId), localSocketId, "Unknown local socket id."); } + private async Task SendAllAsync( + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + CancellationToken cancellationToken) + { + var sent = false; + for (var i = 0; i < _sockets.Count; i++) + { + try + { + await _sockets[i].SendAsync(remoteEndpoint, payload, cancellationToken).ConfigureAwait(false); + sent = true; + } + catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException) + { + throw; + } + catch (Exception ex) when (ex is System.Net.Sockets.SocketException or InvalidOperationException) + { + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] SendAsync failed on socket {i}: {ex}"); + } + } + + if (!sent) + { + throw new InvalidOperationException("All UDP socket sends failed."); + } + } + private async Task ForwardLoopAsync(ZeroTierUdpTransport socket, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) From 00de4cf4581399223ea770cee675dcac64c7ab5a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:55:20 +0100 Subject: [PATCH 173/296] Preserve push-path socket affinity --- ...irectEndpointManagerSocketAffinityTests.cs | 73 ++++++++++++++++--- .../Internal/ZeroTierDirectEndpointManager.cs | 6 +- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index 4c77644..b9e0dd1 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -11,14 +11,10 @@ public sealed class ZeroTierDirectEndpointManagerSocketAffinityTests [Fact] public async Task Rendezvous_UsesReceivingLocalSocket_ForInitialHolePunch() { - await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 0); - await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); - await using var udp = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); - await using var receiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - + await using var udp = new RecordingUdpTransport(); var relay = new IPEndPoint(IPAddress.Loopback, 9999); var peerNodeId = new NodeId(0x1111111111); - var endpoint = TestUdpEndpoints.ToLoopback(receiver.LocalEndpoint); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); await manager.HandleRendezvousFromRootAsync( @@ -27,12 +23,28 @@ await manager.HandleRendezvousFromRootAsync( receivedVia: relay, CancellationToken.None); - var datagram = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - Assert.Equal(4, datagram.Payload.Length); - Assert.Equal(socket1.LocalEndpoint.Port, datagram.RemoteEndPoint.Port); + var send = Assert.Single(udp.Sends); + Assert.Equal(1, send.LocalSocketId); + Assert.Equal(endpoint, send.RemoteEndPoint); + Assert.Equal(2, send.HopLimit); Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + } + + [Fact] + public async Task PushDirectPaths_UsesReceivingLocalSocket_ForNewEndpoints() + { + await using var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("203.0.113.40"), 4242); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandlePushDirectPathsFromRemoteAsync( + ZeroTierPushDirectPathsCodec.BuildPayload(new[] { endpoint }), + receivedLocalSocketId: 1, + CancellationToken.None); - await Assert.ThrowsAsync(async () => await receiver.ReceiveAsync(TimeSpan.FromMilliseconds(250))); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); } private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) @@ -48,4 +60,45 @@ private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) return payload; } + + private sealed class RecordingUdpTransport : IZeroTierUdpTransport + { + public IReadOnlyList LocalSockets { get; } = + [ + new(0, new IPEndPoint(IPAddress.Any, 10000)), + new(1, new IPEndPoint(IPAddress.Any, 10001)) + ]; + + public List Sends { get; } = new(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + Sends.Add(new RecordedSend(localSocketId, remoteEndpoint, HopLimit: null, payload.ToArray())); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + Sends.Add(new RecordedSend(localSocketId, remoteEndpoint, hopLimit, payload.ToArray())); + return Task.CompletedTask; + } + } + + private sealed record RecordedSend(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit, byte[] Payload); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index ea98721..8005535 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -184,7 +184,6 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( IPEndPoint[] endpoints; IPEndPoint[] endpointsToProbe; - IPEndPoint[] preferredSocketEndpoints; lock (_lock) { var previousKeys = _directEndpoints @@ -208,12 +207,9 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( return redirectKeys.Contains(key) || !previousKeys.Contains(key); }) .ToArray(); - preferredSocketEndpoints = endpointsToProbe - .Where(endpoint => redirectKeys.Contains(FormatEndpointKey(endpoint))) - .ToArray(); _directEndpoints = endpoints; PrunePreferredLocalSockets_NoLock(endpoints); - RememberPreferredLocalSockets_NoLock(preferredSocketEndpoints, receivedLocalSocketId); + RememberPreferredLocalSockets_NoLock(endpointsToProbe, receivedLocalSocketId); } if (endpoints.Length == 0 || endpointsToProbe.Length == 0) From a91dfca0eb792c72b7c11ab0ebac397572a30e5c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:00:08 +0100 Subject: [PATCH 174/296] Bind multipath sockets to local interfaces --- ZTSharp.Tests/OsUdpSocketFactoryTests.cs | 7 +- ...ketRuntimeBootstrapperUdpTransportTests.cs | 20 ++++ .../Transport/Internal/OsUdpSocketFactory.cs | 33 ++++- .../ZeroTierSocketRuntimeBootstrapper.cs | 104 +++++++++------- .../ZeroTierUdpLocalBindAddressSource.cs | 113 ++++++++++++++++++ .../Transport/ZeroTierUdpTransport.cs | 9 +- 6 files changed, 237 insertions(+), 49 deletions(-) create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs diff --git a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs index c5c564d..ba16ce4 100644 --- a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs +++ b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs @@ -48,7 +48,12 @@ public void CreateUdp6OnlyBound_SetsDualModeFalse() { Skip.IfNot(Socket.OSSupportsIPv6, "IPv6 not supported on this platform."); - var method = typeof(OsUdpSocketFactory).GetMethod("CreateUdp6OnlyBound", BindingFlags.NonPublic | BindingFlags.Static); + var method = typeof(OsUdpSocketFactory).GetMethod( + "CreateUdp6OnlyBound", + BindingFlags.NonPublic | BindingFlags.Static, + binder: null, + types: [typeof(int)], + modifiers: null); Assert.NotNull(method); UdpClient? udp; diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index a0a7e2b..0cc5d88 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -91,6 +91,26 @@ await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( enableIpv6: false)); } + [Fact] + public void CreateUdpSocketBindings_MultipathEnabled_UsesDiscoveredBindAddressesBeforeWildcard() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + getLocalBindAddresses: () => + [ + IPAddress.Parse("192.0.2.10"), + IPAddress.Parse("2001:db8::10") + ]); + + Assert.Collection( + bindings, + binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress), + binding => Assert.Equal(IPAddress.Parse("2001:db8::10"), binding.LocalAddress), + binding => Assert.Null(binding.LocalAddress), + binding => Assert.Null(binding.LocalAddress)); + } + private static int GetAvailableUdpPort() { using var udp = new UdpClient(new IPEndPoint(IPAddress.Loopback, 0)); diff --git a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs index b5de5a3..f5a4aa5 100644 --- a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs +++ b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs @@ -8,9 +8,11 @@ internal static class OsUdpSocketFactory { private const int WindowsSioUdpConnReset = unchecked((int)0x9800000C); - public static UdpClient Create(int localPort, bool enableIpv6, Action? log = null) + public static UdpClient Create(int localPort, bool enableIpv6, Action? log = null, IPAddress? localBindAddress = null) { - var udp = CreateSocketCore(localPort, enableIpv6); + var udp = localBindAddress is null + ? CreateSocketCore(localPort, enableIpv6) + : CreateBoundSocket(localBindAddress, localPort); TryDisableWindowsUdpConnReset(udp, log); return udp; } @@ -72,12 +74,27 @@ internal static UdpClient CreateSocketCore( throw lastException!; } + private static UdpClient CreateBoundSocket(IPAddress localBindAddress, int localPort) + { + ArgumentNullException.ThrowIfNull(localBindAddress); + + return localBindAddress.AddressFamily switch + { + AddressFamily.InterNetwork => CreateUdp4Bound(new IPEndPoint(localBindAddress, localPort)), + AddressFamily.InterNetworkV6 => CreateUdp6OnlyBound(new IPEndPoint(localBindAddress, localPort)), + _ => throw new NotSupportedException($"Unsupported UDP bind address family: {localBindAddress.AddressFamily}.") + }; + } + private static UdpClient CreateUdp4Bound(int localPort) + => CreateUdp4Bound(new IPEndPoint(IPAddress.Any, localPort)); + + private static UdpClient CreateUdp4Bound(IPEndPoint localEndPoint) { var udp4 = new UdpClient(AddressFamily.InterNetwork); try { - udp4.Client.Bind(new IPEndPoint(IPAddress.Any, localPort)); + udp4.Client.Bind(localEndPoint); return udp4; } catch @@ -88,12 +105,15 @@ private static UdpClient CreateUdp4Bound(int localPort) } private static UdpClient CreateUdp6DualModeBound(int localPort) + => CreateUdp6DualModeBound(new IPEndPoint(IPAddress.IPv6Any, localPort)); + + private static UdpClient CreateUdp6DualModeBound(IPEndPoint localEndPoint) { var udp6 = new UdpClient(AddressFamily.InterNetworkV6); try { udp6.Client.DualMode = true; - udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); + udp6.Client.Bind(localEndPoint); return udp6; } catch @@ -104,12 +124,15 @@ private static UdpClient CreateUdp6DualModeBound(int localPort) } private static UdpClient CreateUdp6OnlyBound(int localPort) + => CreateUdp6OnlyBound(new IPEndPoint(IPAddress.IPv6Any, localPort)); + + private static UdpClient CreateUdp6OnlyBound(IPEndPoint localEndPoint) { var udp6 = new UdpClient(AddressFamily.InterNetworkV6); try { udp6.Client.DualMode = false; - udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); + udp6.Client.Bind(localEndPoint); return udp6; } catch diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index e94ee87..31470b5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -9,6 +9,8 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierSocketRuntimeBootstrapper { + internal readonly record struct ZeroTierUdpSocketBinding(IPAddress? LocalAddress, int LocalPort, int LocalSocketId); + internal static async ValueTask CreateUdpTransportAsync(ZeroTierMultipathOptions multipath, bool enableIpv6) { ArgumentNullException.ThrowIfNull(multipath); @@ -18,31 +20,68 @@ internal static async ValueTask CreateUdpTransportAsync(Z return new ZeroTierUdpTransport(localPort: 0, enableIpv6: enableIpv6, localSocketId: 0); } - if (multipath.UdpSocketCount < 1) + var bindings = CreateUdpSocketBindings(multipath, enableIpv6); + if (bindings.Length == 1) { - throw new ArgumentOutOfRangeException(nameof(multipath), multipath.UdpSocketCount, "UdpSocketCount must be at least 1 when multipath is enabled."); + var binding = bindings[0]; + return new ZeroTierUdpTransport( + localPort: binding.LocalPort, + enableIpv6: enableIpv6, + localSocketId: binding.LocalSocketId, + localBindAddress: binding.LocalAddress); } - if (multipath.UdpSocketCount == 1) + var sockets = new List(bindings.Length); + var success = false; + try { - var port = 0; - var localPorts = multipath.LocalUdpPorts; - if (localPorts is not null) + for (var i = 0; i < bindings.Length; i++) { - if (localPorts.Count != 1) - { - throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts length must match UdpSocketCount."); - } - - port = localPorts[0]; + var binding = bindings[i]; + sockets.Add(new ZeroTierUdpTransport( + localPort: binding.LocalPort, + enableIpv6: enableIpv6, + localSocketId: binding.LocalSocketId, + localBindAddress: binding.LocalAddress)); } - if (port < 0 || port > 65535) + var transport = new ZeroTierUdpMultiTransport(sockets); + success = true; + return transport; + } + finally + { + if (!success) { - throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts entries must be in the range [0, 65535]."); + foreach (var socket in sockets) + { + try + { + await socket.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException or SocketException or InvalidOperationException) + { + } + } } + } + } + + internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( + ZeroTierMultipathOptions multipath, + bool enableIpv6, + Func? getLocalBindAddresses = null) + { + ArgumentNullException.ThrowIfNull(multipath); + + if (!multipath.Enabled) + { + return [new ZeroTierUdpSocketBinding(LocalAddress: null, LocalPort: 0, LocalSocketId: 0)]; + } - return new ZeroTierUdpTransport(localPort: port, enableIpv6: enableIpv6, localSocketId: 0); + if (multipath.UdpSocketCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(multipath), multipath.UdpSocketCount, "UdpSocketCount must be at least 1 when multipath is enabled."); } var ports = multipath.LocalUdpPorts; @@ -71,35 +110,18 @@ internal static async ValueTask CreateUdpTransportAsync(Z } } - var sockets = new List(multipath.UdpSocketCount); - var success = false; - try - { - for (var i = 0; i < multipath.UdpSocketCount; i++) - { - sockets.Add(new ZeroTierUdpTransport(localPort: ports[i], enableIpv6: enableIpv6, localSocketId: i)); - } + var bindAddresses = multipath.UdpSocketCount > 1 + ? (getLocalBindAddresses ?? (() => ZeroTierUdpLocalBindAddressSource.GetSnapshot(enableIpv6))).Invoke() + : Array.Empty(); - var transport = new ZeroTierUdpMultiTransport(sockets); - success = true; - return transport; - } - finally + var bindings = new ZeroTierUdpSocketBinding[multipath.UdpSocketCount]; + for (var i = 0; i < bindings.Length; i++) { - if (!success) - { - foreach (var socket in sockets) - { - try - { - await socket.DisposeAsync().ConfigureAwait(false); - } - catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException or SocketException or InvalidOperationException) - { - } - } - } + var localAddress = i < bindAddresses.Length ? bindAddresses[i] : null; + bindings[i] = new ZeroTierUdpSocketBinding(localAddress, ports[i], i); } + + return bindings; } [SuppressMessage( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs new file mode 100644 index 0000000..f4fec17 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs @@ -0,0 +1,113 @@ +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierUdpLocalBindAddressSource +{ + private static readonly string[] ExcludedAdapterNameTokens = + [ + "zerotier", + "tailscale", + "nord", + "wintun", + "wireguard", + "openvpn" + ]; + + public static IPAddress[] GetSnapshot(bool enableIpv6) + { + var addresses = new List(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + { + if (!ShouldUseInterface(nic)) + { + continue; + } + + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + continue; + } + + foreach (var unicast in properties.UnicastAddresses) + { + var address = Canonicalize(unicast.Address); + if (address is null || !ShouldUseAddress(address, enableIpv6)) + { + continue; + } + + addresses.Add(address); + } + } + + return addresses + .Distinct() + .OrderBy(static address => address.AddressFamily == AddressFamily.InterNetwork ? 0 : 1) + .ToArray(); + } + + private static bool ShouldUseInterface(NetworkInterface nic) + { + if (nic.OperationalStatus != OperationalStatus.Up) + { + return false; + } + + if (nic.NetworkInterfaceType is NetworkInterfaceType.Loopback or NetworkInterfaceType.Tunnel) + { + return false; + } + + var name = (nic.Name + " " + nic.Description).ToUpperInvariant(); + for (var i = 0; i < ExcludedAdapterNameTokens.Length; i++) + { + if (name.Contains(ExcludedAdapterNameTokens[i].ToUpperInvariant(), StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool ShouldUseAddress(IPAddress? address, bool enableIpv6) + { + if (address is null || IPAddress.IsLoopback(address)) + { + return false; + } + + return address.AddressFamily switch + { + AddressFamily.InterNetwork => !address.Equals(IPAddress.Any), + AddressFamily.InterNetworkV6 => enableIpv6 && + !address.Equals(IPAddress.IPv6Any) && + !address.IsIPv6LinkLocal && + !address.IsIPv6SiteLocal, + _ => false + }; + } + + private static IPAddress? Canonicalize(IPAddress? address) + { + if (address is null) + { + return null; + } + + if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + return address.MapToIPv4(); + } + + return address; + } +} diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index e65ea8b..f02cc7f 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -19,11 +19,16 @@ internal sealed class ZeroTierUdpTransport : IZeroTierUdpTransport private long _incomingBackpressureCount; private int _disposed; - public ZeroTierUdpTransport(int localPort = 0, bool enableIpv6 = true, Action? log = null, int localSocketId = 0) + public ZeroTierUdpTransport( + int localPort = 0, + bool enableIpv6 = true, + Action? log = null, + int localSocketId = 0, + IPAddress? localBindAddress = null) { _log = log; _localSocketId = localSocketId; - _udp = OsUdpSocketFactory.Create(localPort, enableIpv6, Log); + _udp = OsUdpSocketFactory.Create(localPort, enableIpv6, Log, localBindAddress); _incoming = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) { From 9b836ba701659378752c62a8d546c8bb15d467b3 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:03:28 +0100 Subject: [PATCH 175/296] Filter local bind addresses by gateway --- ...ketRuntimeBootstrapperUdpTransportTests.cs | 15 ++- .../ZeroTierUdpLocalBindAddressSourceTests.cs | 58 ++++++++++ .../ZeroTierSocketRuntimeBootstrapper.cs | 4 +- .../ZeroTierUdpLocalBindAddressSource.cs | 101 ++++++++++++++---- 4 files changed, 157 insertions(+), 21 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierUdpLocalBindAddressSourceTests.cs diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index 0cc5d88..4256d86 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -107,8 +107,19 @@ public void CreateUdpSocketBindings_MultipathEnabled_UsesDiscoveredBindAddresses bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress), binding => Assert.Equal(IPAddress.Parse("2001:db8::10"), binding.LocalAddress), - binding => Assert.Null(binding.LocalAddress), - binding => Assert.Null(binding.LocalAddress)); + binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress), + binding => Assert.Equal(IPAddress.Parse("2001:db8::10"), binding.LocalAddress)); + } + + [Fact] + public void CreateUdpSocketBindings_NoDiscoveredAddresses_FallsBackToWildcard() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 2 }, + enableIpv6: false, + getLocalBindAddresses: static () => Array.Empty()); + + Assert.All(bindings, binding => Assert.Null(binding.LocalAddress)); } private static int GetAvailableUdpPort() diff --git a/ZTSharp.Tests/ZeroTierUdpLocalBindAddressSourceTests.cs b/ZTSharp.Tests/ZeroTierUdpLocalBindAddressSourceTests.cs new file mode 100644 index 0000000..5b9f9c3 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierUdpLocalBindAddressSourceTests.cs @@ -0,0 +1,58 @@ +using System.Net; +using System.Net.NetworkInformation; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierUdpLocalBindAddressSourceTests +{ + [Fact] + public void SelectAddresses_RequiresDefaultGateway_AndExcludesOverlayAdapters() + { + var addresses = ZeroTierUdpLocalBindAddressSource.SelectAddresses( + [ + new( + Name: "WLAN", + Description: "Wi-Fi", + NetworkInterfaceType.Wireless80211, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("10.0.0.112")], + GatewayAddresses: [IPAddress.Parse("10.0.0.1")]), + new( + Name: "vEthernet (Default Switch)", + Description: "Hyper-V Virtual Ethernet Adapter", + NetworkInterfaceType.Ethernet, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("172.25.128.1")], + GatewayAddresses: Array.Empty()), + new( + Name: "Tailscale", + Description: "Tailscale Tunnel", + NetworkInterfaceType.Tunnel, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("100.74.185.14")], + GatewayAddresses: [IPAddress.Parse("100.100.100.100")]) + ], + enableIpv6: false); + + Assert.Equal([IPAddress.Parse("10.0.0.112")], addresses); + } + + [Fact] + public void SelectAddresses_Excludes_LinkLocal_Addresses() + { + var addresses = ZeroTierUdpLocalBindAddressSource.SelectAddresses( + [ + new( + Name: "Ethernet", + Description: "Realtek", + NetworkInterfaceType.Ethernet, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("169.254.1.20"), IPAddress.Parse("fe80::1")], + GatewayAddresses: [IPAddress.Parse("10.0.0.1")]) + ], + enableIpv6: true); + + Assert.Empty(addresses); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 31470b5..fc47e0f 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -117,7 +117,9 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( var bindings = new ZeroTierUdpSocketBinding[multipath.UdpSocketCount]; for (var i = 0; i < bindings.Length; i++) { - var localAddress = i < bindAddresses.Length ? bindAddresses[i] : null; + var localAddress = bindAddresses.Length == 0 + ? null + : bindAddresses[i % bindAddresses.Length]; bindings[i] = new ZeroTierUdpSocketBinding(localAddress, ports[i], i); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs index f4fec17..a14cdf3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs @@ -18,27 +18,31 @@ internal static class ZeroTierUdpLocalBindAddressSource public static IPAddress[] GetSnapshot(bool enableIpv6) { + return SelectAddresses( + NetworkInterface.GetAllNetworkInterfaces() + .Select(CreateInterfaceInfo) + .Where(static info => info is not null) + .Cast() + .ToArray(), + enableIpv6); + } + + internal static IPAddress[] SelectAddresses(IReadOnlyList interfaces, bool enableIpv6) + { + ArgumentNullException.ThrowIfNull(interfaces); + var addresses = new List(); - foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + for (var i = 0; i < interfaces.Count; i++) { - if (!ShouldUseInterface(nic)) + var info = interfaces[i]; + if (!ShouldUseInterface(info)) { continue; } - IPInterfaceProperties properties; - try - { - properties = nic.GetIPProperties(); - } - catch (NetworkInformationException) + for (var a = 0; a < info.UnicastAddresses.Length; a++) { - continue; - } - - foreach (var unicast in properties.UnicastAddresses) - { - var address = Canonicalize(unicast.Address); + var address = Canonicalize(info.UnicastAddresses[a]); if (address is null || !ShouldUseAddress(address, enableIpv6)) { continue; @@ -54,19 +58,45 @@ public static IPAddress[] GetSnapshot(bool enableIpv6) .ToArray(); } - private static bool ShouldUseInterface(NetworkInterface nic) + private static InterfaceInfo? CreateInterfaceInfo(NetworkInterface nic) { - if (nic.OperationalStatus != OperationalStatus.Up) + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + return null; + } + + return new InterfaceInfo( + nic.Name, + nic.Description, + nic.NetworkInterfaceType, + nic.OperationalStatus, + properties.UnicastAddresses.Select(static address => address.Address).ToArray(), + properties.GatewayAddresses.Select(static gateway => gateway.Address).ToArray()); + } + + private static bool ShouldUseInterface(InterfaceInfo info) + { + if (info.OperationalStatus != OperationalStatus.Up) + { + return false; + } + + if (info.NetworkInterfaceType is NetworkInterfaceType.Loopback or NetworkInterfaceType.Tunnel) { return false; } - if (nic.NetworkInterfaceType is NetworkInterfaceType.Loopback or NetworkInterfaceType.Tunnel) + if (!HasDefaultGateway(info.GatewayAddresses)) { return false; } - var name = (nic.Name + " " + nic.Description).ToUpperInvariant(); + var name = (info.Name + " " + info.Description).ToUpperInvariant(); for (var i = 0; i < ExcludedAdapterNameTokens.Length; i++) { if (name.Contains(ExcludedAdapterNameTokens[i].ToUpperInvariant(), StringComparison.Ordinal)) @@ -78,6 +108,33 @@ private static bool ShouldUseInterface(NetworkInterface nic) return true; } + private static bool HasDefaultGateway(IPAddress[] gatewayAddresses) + { + for (var i = 0; i < gatewayAddresses.Length; i++) + { + var gateway = Canonicalize(gatewayAddresses[i]); + if (gateway is null || IPAddress.IsLoopback(gateway)) + { + continue; + } + + if (gateway.AddressFamily == AddressFamily.InterNetworkV6 && + (gateway.Equals(IPAddress.IPv6Any) || gateway.IsIPv6LinkLocal || gateway.IsIPv6SiteLocal)) + { + continue; + } + + if (gateway.AddressFamily == AddressFamily.InterNetwork && gateway.Equals(IPAddress.Any)) + { + continue; + } + + return true; + } + + return false; + } + private static bool ShouldUseAddress(IPAddress? address, bool enableIpv6) { if (address is null || IPAddress.IsLoopback(address)) @@ -110,4 +167,12 @@ private static bool ShouldUseAddress(IPAddress? address, bool enableIpv6) return address; } + + internal sealed record InterfaceInfo( + string Name, + string Description, + NetworkInterfaceType NetworkInterfaceType, + OperationalStatus OperationalStatus, + IPAddress[] UnicastAddresses, + IPAddress[] GatewayAddresses); } From 21cf26a9fff783607d2f9a59eb84951125b20c13 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:19:14 +0100 Subject: [PATCH 176/296] Fan out direct hint bootstrap across sockets --- ZTSharp.Tests/ScriptedUdpTransport.cs | 79 ++++ ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 424 ++++++++++++------ .../Internal/ZeroTierDataplaneRuntime.cs | 10 +- .../ZeroTierUdpLocalBindAddressSource.cs | 8 +- 4 files changed, 368 insertions(+), 153 deletions(-) create mode 100644 ZTSharp.Tests/ScriptedUdpTransport.cs diff --git a/ZTSharp.Tests/ScriptedUdpTransport.cs b/ZTSharp.Tests/ScriptedUdpTransport.cs new file mode 100644 index 0000000..59ed3de --- /dev/null +++ b/ZTSharp.Tests/ScriptedUdpTransport.cs @@ -0,0 +1,79 @@ +using System.Net; +using System.Threading.Channels; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +internal sealed class ScriptedUdpTransport : IZeroTierUdpTransport +{ + private readonly Channel _receiveQueue = Channel.CreateUnbounded(); + private readonly List _sends = new(); + private readonly object _lock = new(); + + public ScriptedUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) + { + LocalSockets = localSockets; + } + + public IReadOnlyList LocalSockets { get; } + + public RecordedUdpSend[] GetSendsSnapshot() + { + lock (_lock) + { + return _sends.ToArray(); + } + } + + public void EnqueueInbound(int localSocketId, IPEndPoint remoteEndPoint, ReadOnlyMemory payload) + => _receiveQueue.Writer.TryWrite(new ZeroTierUdpDatagram(localSocketId, remoteEndPoint, payload.ToArray())); + + public ValueTask DisposeAsync() + { + _receiveQueue.Writer.TryComplete(); + return ValueTask.CompletedTask; + } + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => _receiveQueue.Reader.ReadAsync(cancellationToken); + + public async ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout); + return await _receiveQueue.Reader.ReadAsync(timeoutCts.Token); + } + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + RecordSend(localSocketId: 0, remoteEndpoint, payload, hopLimit: null); + return Task.CompletedTask; + } + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + RecordSend(localSocketId, remoteEndpoint, payload, hopLimit: null); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + RecordSend(localSocketId, remoteEndpoint, payload, hopLimit); + return Task.CompletedTask; + } + + private void RecordSend(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int? hopLimit) + { + lock (_lock) + { + _sends.Add(new RecordedUdpSend(localSocketId, remoteEndpoint, hopLimit, payload.ToArray())); + } + } +} + +internal readonly record struct RecordedUdpSend(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit, byte[] Payload); diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index b042209..a2f11c5 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -10,34 +10,22 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDataplaneRuntimeDirectPathTests { + private static readonly IPEndPoint RootEndpoint = new(IPAddress.Parse("203.0.113.1"), 9993); + [Fact] public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() { - await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var punchReceiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); var rootNodeId = new NodeId(0x1111111111); var peerNodeId = new NodeId(0x3333333333); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); - var rootKey = new byte[48]; - RandomNumberGenerator.Fill(rootKey); - - await using var runtime = new ZeroTierDataplaneRuntime( - udp, - rootNodeId: rootNodeId, - rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), - rootKey: rootKey, - rootProtocolVersion: 12, - localIdentity: localIdentity, - networkId: 0x9ad07d01093a69e3UL, - localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, - localManagedIpsV6: Array.Empty(), - inlineCom: Array.Empty()); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime(udp, localIdentity, rootNodeId, rootKey); - var runtimeEndpoint = TestUdpEndpoints.ToLoopback(runtime.LocalUdp); - var rendezvousPayload = BuildRendezvousPayload(peerNodeId, TestUdpEndpoints.ToLoopback(punchReceiver.LocalEndpoint)); var packet = ZeroTierPacketCodec.Encode( new ZeroTierPacketHeader( PacketId: 1, @@ -46,20 +34,121 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() Flags: 0, Mac: 0, VerbRaw: (byte)ZeroTierVerb.Rendezvous), - rendezvousPayload); + BuildRendezvousPayload(peerNodeId, rendezvousEndpoint)); ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 0, RootEndpoint, packet); - await rootUdp.SendAsync(runtimeEndpoint, packet); + var holePunch = await WaitForSendAsync( + udp, + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4, + TimeSpan.FromSeconds(2)); - var holePunch = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - Assert.Equal(4, holePunch.Payload.Length); + Assert.Equal(0, holePunch.LocalSocketId); + Assert.Equal(2, holePunch.HopLimit); } [Fact] public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() { - await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("203.0.113.40"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(peerIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); + + udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, udp.LocalSockets[0].LocalEndpoint)); + udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); + + var probe = await WaitForSendAsync( + udp, + send => send.RemoteEndPoint.Equals(hintedEndpoint) && + send.Payload.Length > 4 && + ZeroTierPacketCodec.TryDecode(send.Payload, out _), + TimeSpan.FromSeconds(2)); + + Assert.Equal(0, probe.LocalSocketId); + } + + [Fact] + public async Task DataplaneRuntime_MultipathPushDirectPaths_Probes_FromAllLocalSockets() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("203.0.113.40"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + ]); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(peerIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, udp.LocalSockets[1].LocalEndpoint)); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); + + await WaitForConditionAsync( + () => + { + var seenSocketIds = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint) && + send.Payload.Length > 4 && + ZeroTierPacketCodec.TryDecode(send.Payload, out _)) + .Select(send => send.LocalSocketId) + .Distinct() + .ToArray(); + return seenSocketIds.Contains(0) && seenSocketIds.Contains(1); + }, + TimeSpan.FromSeconds(2)); + } + + [Fact] + public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + [Fact] + public async Task DataplaneRuntime_MultipathPushDirectPaths_Probes_FromAllLocalSockets() + { + await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 0); + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + await using var udp = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var punchReceiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); @@ -85,8 +174,8 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() inlineCom: Array.Empty(), multipath: new ZeroTierMultipathOptions { Enabled = true }); - var runtimeEndpoint = TestUdpEndpoints.ToLoopback(runtime.LocalUdp); - + var runtimeEndpoint0 = TestUdpEndpoints.ToLoopback(socket0.LocalEndpoint); + var runtimeEndpoint1 = TestUdpEndpoints.ToLoopback(socket1.LocalEndpoint); var sharedKey = new byte[48]; ZeroTierC25519.Agree(peerIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); @@ -98,10 +187,10 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() signature: new byte[ZeroTierWorld.C25519SignatureLength], roots: Array.Empty()); - var helloPacket = ZeroTierHelloPacketBuilder.BuildPacket( + var helloPacket0 = ZeroTierHelloPacketBuilder.BuildPacket( peerIdentity, destination: localIdentity.NodeId, - physicalDestination: runtimeEndpoint, + physicalDestination: runtimeEndpoint0, planet, timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), sharedKey, @@ -111,7 +200,23 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, out _); - await rootUdp.SendAsync(runtimeEndpoint, helloPacket); + await rootUdp.SendAsync(runtimeEndpoint0, helloPacket0); + _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + + var helloPacket1 = ZeroTierHelloPacketBuilder.BuildPacket( + peerIdentity, + destination: localIdentity.NodeId, + physicalDestination: runtimeEndpoint1, + planet, + timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + sharedKey, + advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, + advertisedMajorVersion: ZeroTierHelloClient.AdvertisedMajorVersion, + advertisedMinorVersion: ZeroTierHelloClient.AdvertisedMinorVersion, + advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, + out _); + + await rootUdp.SendAsync(runtimeEndpoint1, helloPacket1); _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); var pushPayload = BuildPushDirectPathsPayload(TestUdpEndpoints.ToLoopback(punchReceiver.LocalEndpoint)); @@ -126,50 +231,37 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() pushPayload); ZeroTierPacketCrypto.Armor(pushPacket, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); + await rootUdp.SendAsync(runtimeEndpoint0, pushPacket); - await rootUdp.SendAsync(runtimeEndpoint, pushPacket); - - var sawPacketProbe = false; - for (var attempt = 0; attempt < 4; attempt++) + var seenPorts = new HashSet(); + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(3); + while (seenPorts.Count < 2 && DateTimeOffset.UtcNow < deadline) { var probe = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - if (probe.Payload.Length > 4 && ZeroTierPacketCodec.TryDecode(probe.Payload, out _)) + if (probe.Payload.Length <= 4 || !ZeroTierPacketCodec.TryDecode(probe.Payload, out _)) { - sawPacketProbe = true; - break; + continue; } - } - Assert.True(sawPacketProbe); - } + seenPorts.Add(probe.RemoteEndPoint.Port); + } - [Fact] - public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() - { - await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var punchReceiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + Assert.Contains(socket0.LocalEndpoint.Port, seenPorts); + Assert.Contains(socket1.LocalEndpoint.Port, seenPorts); + - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); Assert.True(peerIdentity.HasPrivateKey); Assert.True(peerIdentity.LocallyValidate()); var rootNodeId = new NodeId(0x1111111111); - var rootKey = new byte[48]; - RandomNumberGenerator.Fill(rootKey); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("203.0.113.41"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); - await using var runtime = new ZeroTierDataplaneRuntime( + await using var runtime = CreateRuntime( udp, - rootNodeId: rootNodeId, - rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), - rootKey: rootKey, - rootProtocolVersion: 12, localIdentity, - networkId: 0x9ad07d01093a69e3UL, - localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, - localManagedIpsV6: Array.Empty(), - inlineCom: Array.Empty(), + rootNodeId, + rootKey, multipath: new ZeroTierMultipathOptions { Enabled = true }, planetId: 1, planetTimestamp: 1); @@ -177,46 +269,12 @@ public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBefore var sharedKey = new byte[48]; ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); - var planet = new ZeroTierWorld( - id: 1, - type: ZeroTierWorldType.Planet, - timestamp: 1, - updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], - signature: new byte[ZeroTierWorld.C25519SignatureLength], - roots: Array.Empty()); - - var runtimeEndpoint = TestUdpEndpoints.ToLoopback(udp.LocalEndpoint); - var helloPacket = ZeroTierHelloPacketBuilder.BuildPacket( - peerIdentity, - destination: localIdentity.NodeId, - physicalDestination: runtimeEndpoint, - planet, - timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), - sharedKey, - advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, - advertisedMajorVersion: ZeroTierHelloClient.AdvertisedMajorVersion, - advertisedMinorVersion: ZeroTierHelloClient.AdvertisedMinorVersion, - advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, - out _); - - await rootUdp.SendAsync(runtimeEndpoint, helloPacket); - _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, udp.LocalSockets[0].LocalEndpoint)); + udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); - var hintedEndpoint = TestUdpEndpoints.ToLoopback(punchReceiver.LocalEndpoint); - var pushPayload = BuildPushDirectPathsPayload(hintedEndpoint); - var pushPacket = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: 2, - Destination: localIdentity.NodeId, - Source: peerIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.PushDirectPaths), - pushPayload); - - ZeroTierPacketCrypto.Armor(pushPacket, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); - await rootUdp.SendAsync(runtimeEndpoint, pushPacket); - _ = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); + await WaitForConditionAsync( + () => runtime.GetHintedDirectCandidatesForMaintenance(peerIdentity.NodeId).Length != 0, + TimeSpan.FromSeconds(2)); var candidates = runtime.GetHintedDirectCandidatesForMaintenance(peerIdentity.NodeId); var candidate = Assert.Single(candidates); @@ -236,46 +294,75 @@ public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPe Assert.True(peerIdentity.LocallyValidate()); var rootNodeId = new NodeId(0x1111111111); - var rootKey = new byte[48]; - RandomNumberGenerator.Fill(rootKey); + var rootKey = RandomNumberGenerator.GetBytes(48); var publicSurface = new IPEndPoint(IPAddress.Parse("203.0.113.10"), 54321); - await using var runtime = new ZeroTierDataplaneRuntime( + await using var runtime = CreateRuntime( udp, - rootNodeId: rootNodeId, - rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), - rootKey: rootKey, - rootProtocolVersion: 12, localIdentity, - networkId: 0x9ad07d01093a69e3UL, - localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, - localManagedIpsV6: Array.Empty(), - inlineCom: Array.Empty(), + rootNodeId, + rootKey, multipath: new ZeroTierMultipathOptions { Enabled = true }, planetId: 1, planetTimestamp: 1, - initialExternalSurfaceObservations: new[] - { + rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), + initialExternalSurfaceObservations: + [ new ZeroTierExternalSurfaceObservation(LocalSocketId: 0, SurfaceAddress: publicSurface) - }); + ]); var sharedKey = new byte[48]; ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); - var planet = new ZeroTierWorld( - id: 1, - type: ZeroTierWorldType.Planet, - timestamp: 1, - updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], - signature: new byte[ZeroTierWorld.C25519SignatureLength], - roots: Array.Empty()); - var runtimeEndpoint = TestUdpEndpoints.ToLoopback(udp.LocalEndpoint); - var helloPacket = ZeroTierHelloPacketBuilder.BuildPacket( + var helloPacket = BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, runtimeEndpoint); + + await rootUdp.SendAsync(runtimeEndpoint, helloPacket); + _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + + var maintenancePeers = runtime.GetPeersForMultipathMaintenanceForTests(); + Assert.Contains(peerIdentity.NodeId, maintenancePeers); + + var advertisements = runtime.GetLocalDirectPathAdvertisementsForTests(); + Assert.Contains(publicSurface, advertisements); + } + + private static ZeroTierDataplaneRuntime CreateRuntime( + IZeroTierUdpTransport udp, + ZeroTierIdentity localIdentity, + NodeId rootNodeId, + byte[] rootKey, + ZeroTierMultipathOptions? multipath = null, + ulong planetId = 0, + ulong planetTimestamp = 0, + IPEndPoint? rootEndpoint = null, + IReadOnlyList? initialExternalSurfaceObservations = null) + => new( + udp, + rootNodeId: rootNodeId, + rootEndpoint: rootEndpoint ?? RootEndpoint, + rootKey: rootKey, + rootProtocolVersion: 12, + localIdentity, + networkId: 0x9ad07d01093a69e3UL, + localManagedIpsV4: [IPAddress.Parse("10.0.0.1")], + localManagedIpsV6: Array.Empty(), + inlineCom: Array.Empty(), + multipath: multipath ?? new ZeroTierMultipathOptions(), + planetId: planetId, + planetTimestamp: planetTimestamp, + initialExternalSurfaceObservations: initialExternalSurfaceObservations); + + private static byte[] BuildRootRelayedHelloPacket( + ZeroTierIdentity peerIdentity, + ZeroTierIdentity localIdentity, + byte[] sharedKey, + IPEndPoint physicalDestination) + => ZeroTierHelloPacketBuilder.BuildPacket( peerIdentity, destination: localIdentity.NodeId, - physicalDestination: runtimeEndpoint, - planet, + physicalDestination, + CreatePlanet(), timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), sharedKey, advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, @@ -284,14 +371,24 @@ public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPe advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, out _); - await rootUdp.SendAsync(runtimeEndpoint, helloPacket); - _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); - - var maintenancePeers = runtime.GetPeersForMultipathMaintenanceForTests(); - Assert.Contains(peerIdentity.NodeId, maintenancePeers); + private static byte[] BuildPushDirectPathsPacket( + NodeId source, + NodeId destination, + byte[] sharedKey, + IPEndPoint endpoint) + { + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 2, + Destination: destination, + Source: source, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.PushDirectPaths), + BuildPushDirectPathsPayload(endpoint)); - var advertisements = runtime.GetLocalDirectPathAdvertisementsForTests(); - Assert.Contains(publicSurface, advertisements); + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); + return packet; } private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) @@ -301,9 +398,9 @@ private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) payload[0] = 0; ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, 5), with.Value); - BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(1 + 5, 2), (ushort)endpoint.Port); - payload[1 + 5 + 2] = (byte)addressBytes.Length; - addressBytes.CopyTo(payload.AsSpan(1 + 5 + 2 + 1)); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(6, 2), (ushort)endpoint.Port); + payload[8] = (byte)addressBytes.Length; + addressBytes.CopyTo(payload.AsSpan(9)); return payload; } @@ -321,24 +418,63 @@ private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint) throw new ArgumentOutOfRangeException(nameof(endpoint), "Invalid IPv4 address bytes."); } - var payload = new byte[2 + 1 + 2 + 1 + 1 + 6]; + var payload = new byte[13]; var span = payload.AsSpan(); BinaryPrimitives.WriteUInt16BigEndian(span.Slice(0, 2), 1); - var ptr = 2; - span[ptr++] = 0; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), 0); - ptr += 2; + span[2] = 0; + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(3, 2), 0); + span[5] = 4; + span[6] = 6; + addressBytes.CopyTo(span.Slice(7, 4)); + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(11, 2), (ushort)endpoint.Port); - span[ptr++] = 4; - span[ptr++] = 6; + return payload; + } - addressBytes.CopyTo(span.Slice(ptr, 4)); - ptr += 4; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), (ushort)endpoint.Port); + private static ZeroTierWorld CreatePlanet() + => new( + ZeroTierWorldType.Planet, + id: 1, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: Array.Empty()); - return payload; + private static async Task WaitForSendAsync( + ScriptedUdpTransport udp, + Func predicate, + TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + var send = udp.GetSendsSnapshot().FirstOrDefault(predicate); + if (send != default) + { + return send; + } + + await Task.Delay(20); + } + + throw new TimeoutException("Timed out waiting for matching UDP send."); } -} + private static async Task WaitForConditionAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(20); + } + + throw new TimeoutException("Timed out waiting for condition."); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 4a64da7..da4d54f 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -772,10 +772,7 @@ private async Task ProbeHintedDirectEndpointsAsync( var endpointsToProbe = _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { - var localSocketIds = _directHintPlanner.GetRotatingSocketIds( - peerNodeId, - endpointsToProbe[i], - includeFallbackLocalSockets: true); + var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpointsToProbe[i]); for (var s = 0; s < localSocketIds.Length; s++) { await SendDirectBootstrapProbeAsync( @@ -1035,10 +1032,7 @@ private async ValueTask HandleDirectEndpointHintAsync( ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId} fullHello={forceFullHello}."); } - var localSocketIds = _directHintPlanner.GetRotatingSocketIds( - peerNodeId, - endpoint, - includeFallbackLocalSockets: false); + var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); for (var i = 0; i < localSocketIds.Length; i++) { await SendDirectBootstrapProbeAsync( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs index a14cdf3..cdd06fd 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs @@ -144,7 +144,7 @@ private static bool ShouldUseAddress(IPAddress? address, bool enableIpv6) return address.AddressFamily switch { - AddressFamily.InterNetwork => !address.Equals(IPAddress.Any), + AddressFamily.InterNetwork => !address.Equals(IPAddress.Any) && !IsIpv4LinkLocal(address), AddressFamily.InterNetworkV6 => enableIpv6 && !address.Equals(IPAddress.IPv6Any) && !address.IsIPv6LinkLocal && @@ -153,6 +153,12 @@ private static bool ShouldUseAddress(IPAddress? address, bool enableIpv6) }; } + private static bool IsIpv4LinkLocal(IPAddress address) + { + var bytes = address.GetAddressBytes(); + return bytes is [169, 254, _, _]; + } + private static IPAddress? Canonicalize(IPAddress? address) { if (address is null) From 49f94f0b4153b410511ac00a1f4cda4230e0f54b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:22:42 +0100 Subject: [PATCH 177/296] Stabilize direct hint probing tests --- .../ZeroTierDirectEndpointSelectionTests.cs | 15 +++++++++++++++ .../Internal/ZeroTierDirectEndpointSelection.cs | 6 ++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs index 55c164b..d5b7bfd 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs @@ -72,6 +72,21 @@ public void Normalize_KeepsUpToPerScopeAndFamilyBudget() Assert.Equal(8, normalized.Count(endpoint => endpoint.AddressFamily == AddressFamily.InterNetworkV6)); } + [Fact] + public void Normalize_KeepsLoopback_WhenCallerExplicitlyAcceptsIt() + { + var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); + var loopback = new IPEndPoint(IPAddress.Loopback, 4242); + + var normalized = ZeroTierDirectEndpointSelection.Normalize( + new[] { loopback }, + relay, + maxEndpoints: 4, + shouldInclude: endpoint => endpoint.Equals(loopback)); + + Assert.Equal([loopback], normalized); + } + [Theory] [InlineData("10.0.0.1", true)] [InlineData("100.85.196.109", true)] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs index 36959f6..1d9fbff 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs @@ -52,7 +52,8 @@ public static IPEndPoint[] Normalize( } var scope = GetScope(canonical.Address); - if (scope is not (EndpointScope.Global or EndpointScope.Shared or EndpointScope.Pseudoprivate or EndpointScope.Private)) + var acceptsLoopback = scope == EndpointScope.Loopback && shouldInclude is not null; + if (!acceptsLoopback && scope is not (EndpointScope.Global or EndpointScope.Shared or EndpointScope.Pseudoprivate or EndpointScope.Private)) { continue; } @@ -259,7 +260,8 @@ private enum EndpointScope EndpointScope.Global, EndpointScope.Shared, EndpointScope.Pseudoprivate, - EndpointScope.Private + EndpointScope.Private, + EndpointScope.Loopback ]; private static readonly AddressFamily[] FamilyPriority = From 276e0f83a018dccb3d20ff07ad589dc528e95e60 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:30:01 +0100 Subject: [PATCH 178/296] Trim unusable direct path probes --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 115 +------- .../ZeroTierDirectEndpointPolicyTests.cs | 92 +++++- .../ZeroTierDirectHintPathPlannerTests.cs | 54 +++- .../Internal/ZeroTierDirectEndpointPolicy.cs | 277 +++++++++++++++++- .../Internal/ZeroTierDirectHintPathPlanner.cs | 56 +++- 5 files changed, 460 insertions(+), 134 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index a2f11c5..85183a7 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -21,7 +21,7 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); var rootNodeId = new NodeId(0x1111111111); var peerNodeId = new NodeId(0x3333333333); - var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime(udp, localIdentity, rootNodeId, rootKey); @@ -60,7 +60,7 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() Assert.True(peerIdentity.LocallyValidate()); var rootNodeId = new NodeId(0x1111111111); - var hintedEndpoint = new IPEndPoint(IPAddress.Parse("203.0.113.40"), 4242); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime( @@ -99,7 +99,7 @@ public async Task DataplaneRuntime_MultipathPushDirectPaths_Probes_FromAllLocalS Assert.True(peerIdentity.LocallyValidate()); var rootNodeId = new NodeId(0x1111111111); - var hintedEndpoint = new IPEndPoint(IPAddress.Parse("203.0.113.40"), 4242); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime( @@ -143,118 +143,11 @@ public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBefore var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - [Fact] - public async Task DataplaneRuntime_MultipathPushDirectPaths_Probes_FromAllLocalSockets() - { - await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 0); - await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); - await using var udp = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); - await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var punchReceiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - Assert.True(peerIdentity.HasPrivateKey); - Assert.True(peerIdentity.LocallyValidate()); - - var rootNodeId = new NodeId(0x1111111111); - var rootKey = new byte[48]; - RandomNumberGenerator.Fill(rootKey); - - await using var runtime = new ZeroTierDataplaneRuntime( - udp, - rootNodeId: rootNodeId, - rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), - rootKey: rootKey, - rootProtocolVersion: 12, - localIdentity: localIdentity, - networkId: 0x9ad07d01093a69e3UL, - localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, - localManagedIpsV6: Array.Empty(), - inlineCom: Array.Empty(), - multipath: new ZeroTierMultipathOptions { Enabled = true }); - - var runtimeEndpoint0 = TestUdpEndpoints.ToLoopback(socket0.LocalEndpoint); - var runtimeEndpoint1 = TestUdpEndpoints.ToLoopback(socket1.LocalEndpoint); - var sharedKey = new byte[48]; - ZeroTierC25519.Agree(peerIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); - - var planet = new ZeroTierWorld( - ZeroTierWorldType.Planet, - id: 1, - timestamp: 1, - updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], - signature: new byte[ZeroTierWorld.C25519SignatureLength], - roots: Array.Empty()); - - var helloPacket0 = ZeroTierHelloPacketBuilder.BuildPacket( - peerIdentity, - destination: localIdentity.NodeId, - physicalDestination: runtimeEndpoint0, - planet, - timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), - sharedKey, - advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, - advertisedMajorVersion: ZeroTierHelloClient.AdvertisedMajorVersion, - advertisedMinorVersion: ZeroTierHelloClient.AdvertisedMinorVersion, - advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, - out _); - - await rootUdp.SendAsync(runtimeEndpoint0, helloPacket0); - _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); - - var helloPacket1 = ZeroTierHelloPacketBuilder.BuildPacket( - peerIdentity, - destination: localIdentity.NodeId, - physicalDestination: runtimeEndpoint1, - planet, - timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), - sharedKey, - advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, - advertisedMajorVersion: ZeroTierHelloClient.AdvertisedMajorVersion, - advertisedMinorVersion: ZeroTierHelloClient.AdvertisedMinorVersion, - advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, - out _); - - await rootUdp.SendAsync(runtimeEndpoint1, helloPacket1); - _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); - - var pushPayload = BuildPushDirectPathsPayload(TestUdpEndpoints.ToLoopback(punchReceiver.LocalEndpoint)); - var pushPacket = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: 2, - Destination: localIdentity.NodeId, - Source: peerIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.PushDirectPaths), - pushPayload); - - ZeroTierPacketCrypto.Armor(pushPacket, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); - await rootUdp.SendAsync(runtimeEndpoint0, pushPacket); - - var seenPorts = new HashSet(); - var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(3); - while (seenPorts.Count < 2 && DateTimeOffset.UtcNow < deadline) - { - var probe = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - if (probe.Payload.Length <= 4 || !ZeroTierPacketCodec.TryDecode(probe.Payload, out _)) - { - continue; - } - - seenPorts.Add(probe.RemoteEndPoint.Port); - } - - Assert.Contains(socket0.LocalEndpoint.Port, seenPorts); - Assert.Contains(socket1.LocalEndpoint.Port, seenPorts); - - Assert.True(peerIdentity.HasPrivateKey); Assert.True(peerIdentity.LocallyValidate()); var rootNodeId = new NodeId(0x1111111111); - var hintedEndpoint = new IPEndPoint(IPAddress.Parse("203.0.113.41"), 4242); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4243); var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime( diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs index f431592..88ffb13 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs @@ -6,9 +6,9 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectEndpointPolicyTests { [Fact] - public void ShouldAccept_AcceptsPublicEndpoint() + public void ShouldAccept_AcceptsPublicEndpoint_WhenLocalNetworksUnknown() { - var policy = new ZeroTierDirectEndpointPolicy(); + var policy = new ZeroTierDirectEndpointPolicy(Array.Empty()); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); @@ -16,9 +16,38 @@ public void ShouldAccept_AcceptsPublicEndpoint() } [Fact] - public void ShouldAccept_AcceptsPrivateEndpoint() + public void ShouldAccept_RejectsPublicIpv6Endpoint_WithoutLocalIpv6Network() { - var policy = new ZeroTierDirectEndpointPolicy(); + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("2606:4700:4700::1111"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsPublicIpv6Endpoint_WithLocalIpv6Network() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("2001:db8::1"), 64) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("2606:4700:4700::1111"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsPrivateEndpoint_OnMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("172.17.0.10"), 16) + ]); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); @@ -26,9 +55,25 @@ public void ShouldAccept_AcceptsPrivateEndpoint() } [Fact] - public void ShouldAccept_AcceptsSharedEndpoint() + public void ShouldAccept_RejectsPrivateEndpoint_OutsideLocalNetworks() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsSharedEndpoint_OnMatchingLocalNetwork() { - var policy = new ZeroTierDirectEndpointPolicy(); + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("100.85.196.14"), 10) + ]); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); @@ -36,19 +81,48 @@ public void ShouldAccept_AcceptsSharedEndpoint() } [Fact] - public void ShouldAccept_AcceptsUlaEndpoint() + public void ShouldAccept_RejectsSharedEndpoint_OutsideLocalNetworks() { - var policy = new ZeroTierDirectEndpointPolicy(); + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsUlaEndpoint_OnMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("fd7a:115c:a1e0::1"), 48) + ]); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993)); Assert.True(accepted); } + [Fact] + public void ShouldAccept_RejectsUlaEndpoint_OutsideLocalNetworks() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("fd00::1"), 64) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993)); + + Assert.False(accepted); + } + [Fact] public void ShouldAccept_RejectsLinkLocalEndpoint() { - var policy = new ZeroTierDirectEndpointPolicy(); + var policy = new ZeroTierDirectEndpointPolicy(Array.Empty()); var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fe80::1"), 9993)); diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs index 35a77ed..8209b08 100644 --- a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs @@ -14,7 +14,9 @@ public void GetNextHintedCandidatesForMaintenance_RotatesSockets_AndSkipsObserve var endpoint2 = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); var endpoint3 = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); - var udp = new StubUdpTransport([0, 1]); + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); manager.SeedEndpoints([endpoint1, endpoint2, endpoint3]); @@ -37,13 +39,54 @@ [new ZeroTierPeerPhysicalPath(0, endpoint1, LastSeenUnixMs: 1)], second); } + [Fact] + public void GetPreferredAndFallbackSocketIds_FiltersByAddressFamily() + { + var peerNodeId = new NodeId(0x1111111111); + var ipv4Endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + var ipv6Endpoint = new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("fd00::1"), 10001))); + var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); + surfaces.Observe(new NodeId(1), 0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); + surfaces.Observe(new NodeId(1), 1, new IPEndPoint(IPAddress.Parse("2001:db8::1"), 60001)); + + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner(udp, surfaces, _ => manager); + + var ipv4Sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv4Endpoint); + var ipv6Sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv6Endpoint); + + Assert.Equal([0], ipv4Sockets); + Assert.Equal([1], ipv6Sockets); + } + + [Fact] + public void GetPreferredAndFallbackSocketIds_ReturnsEmpty_WhenNoSocketCanReachEndpoint() + { + var peerNodeId = new NodeId(0x1111111111); + var ipv6Endpoint = new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); + surfaces.Observe(new NodeId(1), 0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); + + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner(udp, surfaces, _ => manager); + + var sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv6Endpoint); + + Assert.Empty(sockets); + } + private sealed class StubUdpTransport : IZeroTierUdpTransport { - public StubUdpTransport(int[] socketIds) + public StubUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) { - LocalSockets = socketIds - .Select(id => new ZeroTierUdpLocalSocket(id, new IPEndPoint(IPAddress.Loopback, 10000 + id))) - .ToArray(); + LocalSockets = localSockets; } public IReadOnlyList LocalSockets { get; } @@ -66,4 +109,3 @@ public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, public ValueTask DisposeAsync() => ValueTask.CompletedTask; } } - diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs index 5ea4a67..003eaf5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs @@ -1,10 +1,46 @@ using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDirectEndpointPolicy { - private readonly bool _acceptUsableEndpoints = true; + private static readonly string[] ExcludedAdapterNameTokens = + [ + "zerotier", + "tailscale", + "nord", + "wintun", + "wireguard", + "openvpn" + ]; + + private readonly LocalNetwork[] _localNetworks; + private readonly bool _hasLocalIpv4; + private readonly bool _hasLocalIpv6; + + public ZeroTierDirectEndpointPolicy() + : this(GetLocalNetworks()) + { + } + + internal ZeroTierDirectEndpointPolicy(IReadOnlyList localNetworks) + { + ArgumentNullException.ThrowIfNull(localNetworks); + _localNetworks = localNetworks.ToArray(); + for (var i = 0; i < _localNetworks.Length; i++) + { + if (_localNetworks[i].Address.AddressFamily == AddressFamily.InterNetwork) + { + _hasLocalIpv4 = true; + } + else if (_localNetworks[i].Address.AddressFamily == AddressFamily.InterNetworkV6) + { + _hasLocalIpv6 = true; + } + } + } public bool ShouldAccept(IPEndPoint endpoint) { @@ -16,11 +52,242 @@ public bool ShouldAccept(IPEndPoint endpoint) return true; } - return _acceptUsableEndpoints && ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint); + if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)) + { + return false; + } + + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + return ShouldAcceptPublicFamily(endpoint.AddressFamily); + } + + for (var i = 0; i < _localNetworks.Length; i++) + { + if (_localNetworks[i].Contains(endpoint.Address)) + { + return true; + } + } + + return false; + } + + private bool ShouldAcceptPublicFamily(AddressFamily family) + { + if (_localNetworks.Length == 0) + { + return true; + } + + return family switch + { + AddressFamily.InterNetwork => _hasLocalIpv4, + AddressFamily.InterNetworkV6 => _hasLocalIpv6, + _ => false + }; + } + + private static LocalNetwork[] GetLocalNetworks() + { + var networks = new List(); + var interfaces = NetworkInterface.GetAllNetworkInterfaces(); + for (var i = 0; i < interfaces.Length; i++) + { + var nic = interfaces[i]; + if (!ShouldUseInterface(nic)) + { + continue; + } + + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + continue; + } + + if (!HasDefaultGateway(properties)) + { + continue; + } + + foreach (var unicast in properties.UnicastAddresses) + { + var address = Canonicalize(unicast.Address); + if (address is null || !ShouldUseAddress(address)) + { + continue; + } + + var prefixLength = GetPrefixLength(unicast); + if (prefixLength < 0) + { + continue; + } + + networks.Add(new LocalNetwork(address, prefixLength)); + } + } + + return networks + .Distinct() + .ToArray(); + } + + private static bool ShouldUseInterface(NetworkInterface nic) + { + if (nic.OperationalStatus != OperationalStatus.Up) + { + return false; + } + + if (nic.NetworkInterfaceType is NetworkInterfaceType.Loopback or NetworkInterfaceType.Tunnel) + { + return false; + } + + var name = (nic.Name + " " + nic.Description).ToUpperInvariant(); + for (var i = 0; i < ExcludedAdapterNameTokens.Length; i++) + { + if (name.Contains(ExcludedAdapterNameTokens[i].ToUpperInvariant(), StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool HasDefaultGateway(IPInterfaceProperties properties) + { + foreach (var gateway in properties.GatewayAddresses) + { + var address = Canonicalize(gateway.Address); + if (address is null || IPAddress.IsLoopback(address)) + { + continue; + } + + if (address.AddressFamily == AddressFamily.InterNetwork && address.Equals(IPAddress.Any)) + { + continue; + } + + if (address.AddressFamily == AddressFamily.InterNetworkV6 && + (address.Equals(IPAddress.IPv6Any) || address.IsIPv6LinkLocal || address.IsIPv6SiteLocal)) + { + continue; + } + + return true; + } + + return false; + } + + private static bool ShouldUseAddress(IPAddress address) + { + if (IPAddress.IsLoopback(address)) + { + return false; + } + + return address.AddressFamily switch + { + AddressFamily.InterNetwork => !address.Equals(IPAddress.Any) && !IsIpv4LinkLocal(address), + AddressFamily.InterNetworkV6 => !address.Equals(IPAddress.IPv6Any) && + !address.IsIPv6LinkLocal && + !address.IsIPv6SiteLocal, + _ => false + }; + } + + private static int GetPrefixLength(UnicastIPAddressInformation unicast) + { + if (unicast.Address.AddressFamily == AddressFamily.InterNetwork) + { + if (unicast.IPv4Mask is null) + { + return -1; + } + + return CountPrefixBits(unicast.IPv4Mask.GetAddressBytes()); + } + + return unicast.PrefixLength; + } + + private static int CountPrefixBits(byte[] bytes) + { + var bits = 0; + for (var i = 0; i < bytes.Length; i++) + { + var current = bytes[i]; + for (var bit = 7; bit >= 0; bit--) + { + if ((current & (1 << bit)) == 0) + { + return bits; + } + + bits++; + } + } + + return bits; + } + + private static bool IsIpv4LinkLocal(IPAddress address) + { + var bytes = address.GetAddressBytes(); + return bytes is [169, 254, _, _]; } private static IPEndPoint Canonicalize(IPEndPoint endpoint) - => endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && endpoint.Address.IsIPv4MappedToIPv6 - ? new IPEndPoint(endpoint.Address.MapToIPv4(), endpoint.Port) - : endpoint; + => new(Canonicalize(endpoint.Address)!, endpoint.Port); + + private static IPAddress? Canonicalize(IPAddress? address) + => address is not null && + address.AddressFamily == AddressFamily.InterNetworkV6 && + address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; + + internal readonly record struct LocalNetwork(IPAddress Address, int PrefixLength) + { + public bool Contains(IPAddress address) + { + address = Canonicalize(address)!; + var networkAddress = Canonicalize(Address)!; + if (address.AddressFamily != networkAddress.AddressFamily) + { + return false; + } + + var addressBytes = address.GetAddressBytes(); + var networkBytes = networkAddress.GetAddressBytes(); + var fullBytes = PrefixLength / 8; + var remainingBits = PrefixLength % 8; + + for (var i = 0; i < fullBytes; i++) + { + if (addressBytes[i] != networkBytes[i]) + { + return false; + } + } + + if (remainingBits == 0) + { + return true; + } + + var mask = unchecked((byte)(0xFF << (8 - remainingBits))); + return (addressBytes[fullBytes] & mask) == (networkBytes[fullBytes] & mask); + } + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs index fea7720..13f1a56 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -145,6 +145,7 @@ public void RemovePeer(NodeId peerNodeId) private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeFallbackLocalSockets) { + endpoint = Canonicalize(endpoint); var localSockets = _udp.LocalSockets; if (localSockets.Count == 0) { @@ -155,7 +156,8 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); for (var i = 0; i < preferredFromHints.Length; i++) { - if (!preferred.Contains(preferredFromHints[i])) + if (SocketCanReachEndpoint(localSockets, preferredFromHints[i], endpoint) && + !preferred.Contains(preferredFromHints[i])) { preferred.Add(preferredFromHints[i]); } @@ -166,7 +168,9 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF for (var i = 0; i < localSockets.Count; i++) { var socketId = localSockets[i].Id; - if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && !preferred.Contains(socketId)) + if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && + SocketCanReachEndpoint(localSockets, socketId, endpoint) && + !preferred.Contains(socketId)) { preferred.Add(socketId); } @@ -178,11 +182,57 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF return preferred.ToArray(); } - return localSockets.Select(static socket => socket.Id).ToArray(); + return localSockets + .Where(socket => SocketCanReachEndpoint(localSockets, socket.Id, endpoint)) + .Select(static socket => socket.Id) + .ToArray(); } private DirectBootstrapProbeCursor GetOrCreateCursor(NodeId peerNodeId) => _probeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); + + private static bool SocketCanReachEndpoint( + IReadOnlyList localSockets, + int socketId, + IPEndPoint endpoint) + { + for (var i = 0; i < localSockets.Count; i++) + { + if (localSockets[i].Id != socketId) + { + continue; + } + + return AddressFamiliesAreCompatible(localSockets[i].LocalEndpoint.Address, endpoint.Address); + } + + return false; + } + + private static bool AddressFamiliesAreCompatible(IPAddress localAddress, IPAddress remoteAddress) + { + remoteAddress = Canonicalize(remoteAddress); + localAddress = Canonicalize(localAddress); + + if (localAddress.AddressFamily == remoteAddress.AddressFamily) + { + return true; + } + + return localAddress.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && + localAddress.Equals(IPAddress.IPv6Any) && + remoteAddress.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork; + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && endpoint.Address.IsIPv4MappedToIPv6 + ? new IPEndPoint(endpoint.Address.MapToIPv4(), endpoint.Port) + : endpoint; + + private static IPAddress Canonicalize(IPAddress address) + => address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; } internal sealed class DirectBootstrapProbeCursor From ce4d16a78f02a43bf690c3077a8bdbf8fef8bc22 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:40:27 +0100 Subject: [PATCH 179/296] Probe direct hints with HELLO first --- ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs | 2 ++ ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 85183a7..f544317 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -83,7 +83,9 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() ZeroTierPacketCodec.TryDecode(send.Payload, out _), TimeSpan.FromSeconds(2)); + Assert.True(ZeroTierPacketCodec.TryDecode(probe.Payload, out var decoded)); Assert.Equal(0, probe.LocalSocketId); + Assert.Equal(ZeroTierVerb.Hello, decoded.Header.Verb); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index da4d54f..3d974cd 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -780,7 +780,7 @@ await SendDirectBootstrapProbeAsync( localSocketIds[s], endpointsToProbe[i], sharedKey, - forceFullHello: false, + forceFullHello: s == 0, DirectHelloMinIntervalMs, cancellationToken) .ConfigureAwait(false); @@ -1040,7 +1040,7 @@ await SendDirectBootstrapProbeAsync( localSocketIds[i], endpoint, sharedKey, - forceFullHello, + forceFullHello || i == 0, DirectHelloMinIntervalMs, cancellationToken) .ConfigureAwait(false); From 1896717bca384deefdb367a796ca79162961dc43 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:49:03 +0100 Subject: [PATCH 180/296] Filter pushed direct paths by peer reachability --- ...ocalDirectPathAdvertisementPlannerTests.cs | 53 ++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 12 +++- .../Internal/ZeroTierDirectEndpointPolicy.cs | 57 +++++++++++++++ ...TierLocalDirectPathAdvertisementPlanner.cs | 70 +++++++++++++++++++ 4 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs new file mode 100644 index 0000000..ab204a8 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs @@ -0,0 +1,53 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierLocalDirectPathAdvertisementPlannerTests +{ + private static readonly IPEndPoint PublicLocal = new(IPAddress.Parse("212.241.85.84"), 50060); + private static readonly IPEndPoint PrivateLocal = new(IPAddress.Parse("10.0.0.112"), 50060); + + [Fact] + public void SelectForPeer_DropsPrivateLocalAdvertisements_ForPublicPeer() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [new IPEndPoint(IPAddress.Parse("176.66.90.119"), 14849)], + []); + + Assert.Equal([PublicLocal], selected); + } + + [Fact] + public void SelectForPeer_KeepsMatchingPrivateLocalAdvertisements_ForSameSubnetPeer() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [new IPEndPoint(IPAddress.Parse("10.0.0.7"), 14849)], + []); + + Assert.Equal([PublicLocal, PrivateLocal], selected); + } + + [Fact] + public void SelectForPeer_DropsPrivateLocalAdvertisements_WhenPeerEndpointsUnknown() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [], + []); + + Assert.Equal([PublicLocal], selected); + } + + private static ZeroTierLocalDirectPathAdvertisementPlanner CreatePlanner() + => new(new ZeroTierDirectEndpointPolicy( + [new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24)])); +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 3d974cd..124d1fb 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -51,6 +51,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; private readonly ZeroTierDirectEndpointPolicy _directEndpointPolicy; + private readonly ZeroTierLocalDirectPathAdvertisementPlanner _localDirectAdvertisementPlanner; private readonly ZeroTierLocalDirectPathAdvertisementSource _localDirectPathAdvertisementsSource; private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _authenticatedPeers = new(); @@ -201,6 +202,7 @@ public ZeroTierDataplaneRuntime( _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); + _localDirectAdvertisementPlanner = new ZeroTierLocalDirectPathAdvertisementPlanner(_directEndpointPolicy); _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); @@ -863,7 +865,7 @@ await SendHelloPacketAsync( private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { - var advertisements = GetLocalDirectPathAdvertisements(); + var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId); if (advertisements.Length == 0) { return; @@ -910,6 +912,14 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } + private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) + { + var localAdvertisements = GetLocalDirectPathAdvertisements(); + var hintedPeerEndpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + var observedPeerPaths = _peerPaths.GetSnapshot(peerNodeId); + return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); + } + private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { if (_inlineCom.Length == 0) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs index 003eaf5..364e611 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs @@ -73,6 +73,39 @@ public bool ShouldAccept(IPEndPoint endpoint) return false; } + public bool ShouldAdvertiseLocalEndpoint(IPEndPoint localEndpoint, IReadOnlyList peerEndpoints) + { + ArgumentNullException.ThrowIfNull(localEndpoint); + ArgumentNullException.ThrowIfNull(peerEndpoints); + + localEndpoint = Canonicalize(localEndpoint); + if (!ShouldAccept(localEndpoint)) + { + return false; + } + + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(localEndpoint)) + { + return true; + } + + for (var i = 0; i < peerEndpoints.Count; i++) + { + var peerEndpoint = Canonicalize(peerEndpoints[i]); + if (!ShouldAccept(peerEndpoint)) + { + continue; + } + + if (SharesLocalNetwork(localEndpoint.Address, peerEndpoint.Address)) + { + return true; + } + } + + return false; + } + private bool ShouldAcceptPublicFamily(AddressFamily family) { if (_localNetworks.Length == 0) @@ -88,6 +121,30 @@ private bool ShouldAcceptPublicFamily(AddressFamily family) }; } + private bool SharesLocalNetwork(IPAddress localAddress, IPAddress peerAddress) + { + if (localAddress.AddressFamily != peerAddress.AddressFamily) + { + return false; + } + + for (var i = 0; i < _localNetworks.Length; i++) + { + var network = _localNetworks[i]; + if (!network.Contains(localAddress)) + { + continue; + } + + if (network.Contains(peerAddress)) + { + return true; + } + } + + return false; + } + private static LocalNetwork[] GetLocalNetworks() { var networks = new List(); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs new file mode 100644 index 0000000..e99d369 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs @@ -0,0 +1,70 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierLocalDirectPathAdvertisementPlanner +{ + private readonly ZeroTierDirectEndpointPolicy _policy; + + public ZeroTierLocalDirectPathAdvertisementPlanner(ZeroTierDirectEndpointPolicy policy) + { + _policy = policy ?? throw new ArgumentNullException(nameof(policy)); + } + + public IPEndPoint[] SelectForPeer( + IReadOnlyList localAdvertisements, + IReadOnlyList hintedPeerEndpoints, + IReadOnlyList observedPeerPaths) + { + ArgumentNullException.ThrowIfNull(localAdvertisements); + ArgumentNullException.ThrowIfNull(hintedPeerEndpoints); + ArgumentNullException.ThrowIfNull(observedPeerPaths); + + var peerEndpoints = BuildPeerEndpoints(hintedPeerEndpoints, observedPeerPaths); + var selected = new List(localAdvertisements.Count); + for (var i = 0; i < localAdvertisements.Count; i++) + { + var localEndpoint = localAdvertisements[i]; + if (_policy.ShouldAdvertiseLocalEndpoint(localEndpoint, peerEndpoints)) + { + selected.Add(localEndpoint); + } + } + + return selected.Count == 0 + ? localAdvertisements.ToArray() + : selected.ToArray(); + } + + private static IPEndPoint[] BuildPeerEndpoints( + IReadOnlyList hintedPeerEndpoints, + IReadOnlyList observedPeerPaths) + { + var endpoints = new List(hintedPeerEndpoints.Count + observedPeerPaths.Count); + var seen = new HashSet(StringComparer.Ordinal); + + for (var i = 0; i < hintedPeerEndpoints.Count; i++) + { + AddUnique(endpoints, seen, hintedPeerEndpoints[i]); + } + + for (var i = 0; i < observedPeerPaths.Count; i++) + { + AddUnique(endpoints, seen, observedPeerPaths[i].RemoteEndPoint); + } + + return endpoints.ToArray(); + } + + private static void AddUnique(List endpoints, HashSet seen, IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var canonical = UdpEndpointNormalization.Normalize(endpoint); + if (seen.Add(canonical.ToString())) + { + endpoints.Add(canonical); + } + } +} From 631313078556a3ed8b8fff6eed3acc7cb76426ac Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:53:46 +0100 Subject: [PATCH 181/296] Track self-awareness per reporter path --- .../ZeroTierDirectHintPathPlannerTests.cs | 18 ++++- ...oTierExternalSurfaceAddressTrackerTests.cs | 45 ++++++++++-- .../ZeroTierDataplanePeerDatagramProcessor.cs | 15 ++-- .../Internal/ZeroTierDataplaneRuntime.cs | 10 ++- .../ZeroTierExternalSurfaceAddressTracker.cs | 71 ++++++++++++++++++- 5 files changed, 142 insertions(+), 17 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs index 8209b08..500b1b7 100644 --- a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs @@ -50,8 +50,16 @@ public void GetPreferredAndFallbackSocketIds_FiltersByAddressFamily() new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("fd00::1"), 10001))); var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); - surfaces.Observe(new NodeId(1), 0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); - surfaces.Observe(new NodeId(1), 1, new IPEndPoint(IPAddress.Parse("2001:db8::1"), 60001)); + surfaces.Observe( + new NodeId(1), + 0, + new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); + surfaces.Observe( + new NodeId(1), + 1, + new IPEndPoint(IPAddress.Parse("2001:db8::2"), 9993), + new IPEndPoint(IPAddress.Parse("2001:db8::1"), 60001)); var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); var planner = new ZeroTierDirectHintPathPlanner(udp, surfaces, _ => manager); @@ -72,7 +80,11 @@ public void GetPreferredAndFallbackSocketIds_ReturnsEmpty_WhenNoSocketCanReachEn var udp = new StubUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); - surfaces.Observe(new NodeId(1), 0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); + surfaces.Observe( + new NodeId(1), + 0, + new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); var planner = new ZeroTierDirectHintPathPlanner(udp, surfaces, _ => manager); diff --git a/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs b/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs index a363d49..a2d1df3 100644 --- a/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs +++ b/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs @@ -13,9 +13,21 @@ public void Observe_TracksDistinctSurfaceAddressesPerLocalSocket() var tracker = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromSeconds(10), nowUnixMs: () => now); var localSocketId = 2; - tracker.Observe(new NodeId(0x1111111111), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); - tracker.Observe(new NodeId(0x2222222222), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); - tracker.Observe(new NodeId(0x3333333333), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.2"), 10001)); + tracker.Observe( + new NodeId(0x1111111111), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.10"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + new NodeId(0x2222222222), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.11"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + new NodeId(0x3333333333), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.12"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.2"), 10001)); var snapshot = tracker.GetSnapshot(localSocketId); Assert.Equal(2, snapshot.Length); @@ -28,11 +40,36 @@ public void Observe_ExpiresEntries() var tracker = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromMilliseconds(100), nowUnixMs: () => now); var localSocketId = 2; - tracker.Observe(new NodeId(0x1111111111), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + new NodeId(0x1111111111), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.10"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); Assert.Single(tracker.GetSnapshot(localSocketId)); now += 2_000; Assert.Empty(tracker.GetSnapshot(localSocketId)); } + + [Fact] + public void Observe_KeepsDistinctReporterPathsForSamePeerAndSocket() + { + var tracker = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromSeconds(10)); + var peerNodeId = new NodeId(0x1111111111); + + tracker.Observe( + peerNodeId, + localSocketId: 2, + new IPEndPoint(IPAddress.Parse("203.0.113.10"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + peerNodeId, + localSocketId: 2, + new IPEndPoint(IPAddress.Parse("203.0.113.11"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.2"), 10001)); + + var snapshot = tracker.GetSnapshot(localSocketId: 2); + Assert.Equal(2, snapshot.Length); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index 27edb44..80fe36d 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -91,13 +91,14 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c cancellationToken) .ConfigureAwait(false); + var observedNewHop0Path = false; if (inboundHello is { } hello && decoded.Header.HopCount == 0) { - _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + observedNewHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); if (hello.ReportedLocalSurfaceAddress is { } reportedSurface) { - _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, reportedSurface); + _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, reportedSurface); } if (ZeroTierTrace.Enabled) @@ -113,6 +114,12 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c _handleAuthenticatedPeer?.Invoke(peerNodeId); } + if (observedNewHop0Path && _handleUnknownDirectPathAsync is not null) + { + await _handleUnknownDirectPathAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) + .ConfigureAwait(false); + } + return; } @@ -249,9 +256,9 @@ await _peerEcho { _peerEcho.ObserveHelloOkRtt(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, ok.TimestampEcho); - if (ok.ExternalSurfaceAddress is { } surface) + if (decoded.Header.HopCount == 0 && ok.ExternalSurfaceAddress is { } surface) { - _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, surface); + _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, surface); } } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 124d1fb..c54a4b3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -328,7 +328,11 @@ private void SeedInitialExternalSurfaceObservations(IReadOnlyList? nowUnixMs _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } - public void Observe(NodeId reportingPeerNodeId, int localSocketId, IPEndPoint surfaceAddress) + public void Observe( + NodeId reportingPeerNodeId, + int localSocketId, + IPEndPoint reporterPhysicalAddress, + IPEndPoint surfaceAddress) { + ArgumentNullException.ThrowIfNull(reporterPhysicalAddress); ArgumentNullException.ThrowIfNull(surfaceAddress); var now = _nowUnixMs(); - var key = new ZeroTierExternalSurfaceKey(localSocketId, reportingPeerNodeId); + var key = new ZeroTierExternalSurfaceKey( + localSocketId, + reportingPeerNodeId, + Normalize(reporterPhysicalAddress), + GetScope(surfaceAddress.Address)); var stored = new IPEndPoint(surfaceAddress.Address, surfaceAddress.Port); _entries[key] = new Entry(stored, now); CleanupIfNeeded(now); @@ -75,7 +84,63 @@ private void CleanupIfNeeded(long nowUnixMs) } private readonly record struct Entry(IPEndPoint SurfaceAddress, long LastSeenUnixMs); + + private static IPEndPoint Normalize(IPEndPoint endpoint) + => endpoint.Address.IsIPv4MappedToIPv6 + ? new IPEndPoint(endpoint.Address.MapToIPv4(), endpoint.Port) + : endpoint; + + private static ZeroTierExternalSurfaceScope GetScope(IPAddress address) + { + if (address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + if (IPAddress.IsLoopback(address)) + { + return ZeroTierExternalSurfaceScope.Loopback; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + var bytes = address.GetAddressBytes(); + if (bytes is [10, _, _, _] || + bytes is [172, >= 16 and <= 31, _, _] || + bytes is [192, 168, _, _]) + { + return ZeroTierExternalSurfaceScope.Private; + } + + return ZeroTierExternalSurfaceScope.Global; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + var bytes = address.GetAddressBytes(); + if ((bytes[0] & 0xFE) == 0xFC) + { + return ZeroTierExternalSurfaceScope.Private; + } + + return ZeroTierExternalSurfaceScope.Global; + } + + return ZeroTierExternalSurfaceScope.None; + } } -internal readonly record struct ZeroTierExternalSurfaceKey(int LocalSocketId, NodeId ReportingPeerNodeId); +internal readonly record struct ZeroTierExternalSurfaceKey( + int LocalSocketId, + NodeId ReportingPeerNodeId, + IPEndPoint ReporterPhysicalAddress, + ZeroTierExternalSurfaceScope Scope); + +internal enum ZeroTierExternalSurfaceScope +{ + None, + Loopback, + Private, + Global +} From 5ee9db473d0f8f100656bdc747a321dcab0bcd67 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 02:54:14 +0100 Subject: [PATCH 182/296] Fix inbound direct HELLO confirmation build --- .../Internal/ZeroTierDataplanePeerDatagramProcessor.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index 80fe36d..68c023f 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -91,10 +91,10 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c cancellationToken) .ConfigureAwait(false); - var observedNewHop0Path = false; + var observedNewInboundHop0Path = false; if (inboundHello is { } hello && decoded.Header.HopCount == 0) { - observedNewHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + observedNewInboundHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); if (hello.ReportedLocalSurfaceAddress is { } reportedSurface) { @@ -114,7 +114,7 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c _handleAuthenticatedPeer?.Invoke(peerNodeId); } - if (observedNewHop0Path && _handleUnknownDirectPathAsync is not null) + if (observedNewInboundHop0Path && _handleUnknownDirectPathAsync is not null) { await _handleUnknownDirectPathAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) .ConfigureAwait(false); From e3b91d782cdcaf09eb7168d2d0fc604903ed1af5 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:00:50 +0100 Subject: [PATCH 183/296] Retry repeated push-direct candidates --- ...roTierDirectEndpointManagerPushFlagsTests.cs | 11 ++++++----- .../Internal/ZeroTierDirectEndpointManager.cs | 17 ++++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index e2e9562..67440c0 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -55,7 +55,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_KnownEndpoint_IsNotRebootstrapped() + public async Task PushDirectPaths_KnownEndpoint_IsRebootstrappedAndUpdatesSocketAffinity() { var udp = new RecordingUdpTransport(); @@ -76,14 +76,15 @@ public async Task PushDirectPaths_KnownEndpoint_IsNotRebootstrapped() var payload = BuildPushDirectPathsPayload(endpoint, flags: 0); await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); - await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 1, CancellationToken.None); - Assert.Single(hintedEndpoints); + Assert.Equal(new[] { endpoint, endpoint }, hintedEndpoints); + Assert.Equal(new[] { 0, 1 }, manager.GetPreferredLocalSocketIds(endpoint).Order().ToArray()); Assert.Empty(udp.Sends); } [Fact] - public async Task PushDirectPaths_NewEndpoint_DoesNotHolePunchOrRememberReceivingSocketAffinity() + public async Task PushDirectPaths_NewEndpoint_RemembersReceivingSocketAffinityWithoutHolePunch() { var udp = new RecordingUdpTransport(); @@ -97,7 +98,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 8005535..fcb5ff8 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -157,7 +157,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( var forget = new HashSet(StringComparer.Ordinal); var redirect = new List(); var add = new List(); - var redirectKeys = new HashSet(StringComparer.Ordinal); + var forceFullHelloByEndpoint = new Dictionary(StringComparer.Ordinal); for (var i = 0; i < paths.Length; i++) { @@ -168,17 +168,19 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( if ((flags & PushDirectPathsFlagForgetPath) != 0) { forget.Add(key); + forceFullHelloByEndpoint.Remove(key); continue; } if ((flags & PushDirectPathsFlagClusterRedirect) != 0) { redirect.Add(endpoint); - redirectKeys.Add(key); + forceFullHelloByEndpoint[key] = true; } else { add.Add(endpoint); + forceFullHelloByEndpoint.TryAdd(key, false); } } @@ -186,9 +188,6 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( IPEndPoint[] endpointsToProbe; lock (_lock) { - var previousKeys = _directEndpoints - .Select(FormatEndpointKey) - .ToHashSet(StringComparer.Ordinal); var merged = _directEndpoints .Where(ep => !forget.Contains(FormatEndpointKey(ep))) .Concat(redirect) @@ -201,11 +200,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( _shouldAcceptEndpoint, maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); endpointsToProbe = endpoints - .Where(endpoint => - { - var key = FormatEndpointKey(endpoint); - return redirectKeys.Contains(key) || !previousKeys.Contains(key); - }) + .Where(endpoint => forceFullHelloByEndpoint.ContainsKey(FormatEndpointKey(endpoint))) .ToArray(); _directEndpoints = endpoints; PrunePreferredLocalSockets_NoLock(endpoints); @@ -226,7 +221,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { if (_handleDirectEndpointHintAsync is not null) { - var forceFullHello = redirectKeys.Contains(FormatEndpointKey(endpoint)); + var forceFullHello = forceFullHelloByEndpoint[FormatEndpointKey(endpoint)]; await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, forceFullHello, cancellationToken).ConfigureAwait(false); } } From 6b60c6cecf81c228c1dc4be8f5673e00bcca5bcc Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:12:40 +0100 Subject: [PATCH 184/296] Fan out root path advertisements across sockets --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 39 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 20 +++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index f544317..1119864 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -1,4 +1,5 @@ using System.Buffers.Binary; +using System.Linq; using System.Net; using System.Security.Cryptography; using ZTSharp.ZeroTier; @@ -222,6 +223,44 @@ public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPe Assert.Contains(publicSurface, advertisements); } + [Fact] + public async Task DataplaneRuntime_SendViaRootSockets_FansOutAcrossLocalSockets() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + ]); + + var payload = new byte[] { 1, 2, 3 }; + await runtime.SendViaRootSocketsForTestsAsync(payload); + + var socketIds = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => send.Payload.SequenceEqual(payload)) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + Assert.Equal(new[] { 0, 1 }, socketIds); + } + private static ZeroTierDataplaneRuntime CreateRuntime( IZeroTierUdpTransport udp, ZeroTierIdentity localIdentity, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c54a4b3..f935991 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -905,7 +905,7 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha $"[zerotier] TX PUSH_DIRECT_PATHS via root for {peerNodeId}: {ZeroTierDirectEndpointSelection.Format(advertisements)}."); } - await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + await SendViaRootSocketsAsync(packet, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) { @@ -924,6 +924,21 @@ private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); } + private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken) + { + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + return; + } + + for (var i = 0; i < localSockets.Count; i++) + { + await _udp.SendAsync(localSockets[i].Id, _rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + } + private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { if (_inlineCom.Length == 0) @@ -1806,6 +1821,9 @@ internal NodeId[] GetPeersForMultipathMaintenanceForTests() internal IPEndPoint[] GetLocalDirectPathAdvertisementsForTests() => GetLocalDirectPathAdvertisements(); + internal Task SendViaRootSocketsForTestsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken = default) + => SendViaRootSocketsAsync(packet, cancellationToken); + private NodeId[] GetPeersForMultipathMaintenance() { var peers = _peerPaths.GetPeersSnapshot(); From 89495880ac2bb29eb19260ed8cf2edea839aac9d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:15:46 +0100 Subject: [PATCH 185/296] Fan out relayed credentials across root sockets --- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index f935991..a228fc7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -977,7 +977,7 @@ private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] ZeroTierTrace.WriteLine($"[zerotier] TX NETWORK_CREDENTIALS via root for {peerNodeId}."); } - await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + await SendViaRootSocketsAsync(packet, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) { From 8da299c475306c27f1a3bdccfb06b13c188f3f51 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:22:10 +0100 Subject: [PATCH 186/296] Use peer root socket affinity for relayed control --- .../ZeroTierPeerRootSocketAffinityTests.cs | 45 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 20 ++++++++- .../ZeroTierPeerRootSocketAffinity.cs | 36 +++++++++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierPeerRootSocketAffinityTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierPeerRootSocketAffinity.cs diff --git a/ZTSharp.Tests/ZeroTierPeerRootSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierPeerRootSocketAffinityTests.cs new file mode 100644 index 0000000..1715352 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPeerRootSocketAffinityTests.cs @@ -0,0 +1,45 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPeerRootSocketAffinityTests +{ + [Fact] + public void Observe_IgnoresNonRootEndpoints() + { + var peerNodeId = new NodeId(0x1111111111); + var affinity = new ZeroTierPeerRootSocketAffinity(new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993)); + + affinity.Observe(peerNodeId, localSocketId: 4, new IPEndPoint(IPAddress.Parse("176.66.90.119"), 35998)); + + Assert.False(affinity.TryGet(peerNodeId, out _)); + } + + [Fact] + public void Observe_RootEndpoint_RemembersLatestSocket() + { + var peerNodeId = new NodeId(0x1111111111); + var rootEndpoint = new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993); + var affinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); + + affinity.Observe(peerNodeId, localSocketId: 3, rootEndpoint); + affinity.Observe(peerNodeId, localSocketId: 6, rootEndpoint); + + Assert.True(affinity.TryGet(peerNodeId, out var localSocketId)); + Assert.Equal(6, localSocketId); + } + + [Fact] + public void Remove_ClearsRememberedSocket() + { + var peerNodeId = new NodeId(0x1111111111); + var rootEndpoint = new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993); + var affinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); + + affinity.Observe(peerNodeId, localSocketId: 6, rootEndpoint); + affinity.Remove(peerNodeId); + + Assert.False(affinity.TryGet(peerNodeId, out _)); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index a228fc7..47fd9c1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -49,6 +49,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerQosManager _peerQos; private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly ZeroTierPeerBondPolicyEngine _bondEngine; + private readonly ZeroTierPeerRootSocketAffinity _peerRootSocketAffinity; private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; private readonly ZeroTierDirectEndpointPolicy _directEndpointPolicy; private readonly ZeroTierLocalDirectPathAdvertisementPlanner _localDirectAdvertisementPlanner; @@ -201,6 +202,7 @@ public ZeroTierDataplaneRuntime( _peerQos = new ZeroTierPeerQosManager(); _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); + _peerRootSocketAffinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); _localDirectAdvertisementPlanner = new ZeroTierLocalDirectPathAdvertisementPlanner(_directEndpointPolicy); _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); @@ -905,7 +907,7 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha $"[zerotier] TX PUSH_DIRECT_PATHS via root for {peerNodeId}: {ZeroTierDirectEndpointSelection.Format(advertisements)}."); } - await SendViaRootSocketsAsync(packet, cancellationToken).ConfigureAwait(false); + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) { @@ -939,6 +941,16 @@ private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, Cancella } } + private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet, CancellationToken cancellationToken) + { + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) + { + return _udp.SendAsync(localSocketId, _rootEndpoint, packet, cancellationToken); + } + + return SendViaRootSocketsAsync(packet, cancellationToken); + } + private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { if (_inlineCom.Length == 0) @@ -977,7 +989,7 @@ private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] ZeroTierTrace.WriteLine($"[zerotier] TX NETWORK_CREDENTIALS via root for {peerNodeId}."); } - await SendViaRootSocketsAsync(packet, cancellationToken).ConfigureAwait(false); + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) { @@ -1324,6 +1336,7 @@ private void HandlePeerHelloOk( ZeroTierHelloOkPayload ok) { ArgumentNullException.ThrowIfNull(receivedVia); + _peerRootSocketAffinity.Observe(peerNodeId, receivedLocalSocketId, receivedVia); var matchedPending = TryTakePendingHello(peerNodeId, ok.InRePacketId, out var pending); var observedLocalSocketId = matchedPending ? pending.LocalSocketId : receivedLocalSocketId; @@ -1748,6 +1761,8 @@ private ValueTask HandlePeerControlPacketAsync( ReadOnlyMemory payload, CancellationToken cancellationToken) { + _peerRootSocketAffinity.Observe(peerNodeId, receivedLocalSocketId, receivedVia); + if (verb != ZeroTierVerb.PushDirectPaths) { return ValueTask.CompletedTask; @@ -1805,6 +1820,7 @@ private void CleanupDirectEndpointManagers(long nowMs) _directEndpointLastUsedMs.TryRemove(pair.Key, out _); _directEndpoints.TryRemove(pair.Key, out _); _directHintPlanner.RemovePeer(pair.Key); + _peerRootSocketAffinity.Remove(pair.Key); } } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerRootSocketAffinity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerRootSocketAffinity.cs new file mode 100644 index 0000000..12590aa --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerRootSocketAffinity.cs @@ -0,0 +1,36 @@ +using System.Collections.Concurrent; +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierPeerRootSocketAffinity +{ + private readonly IPEndPoint _rootEndpoint; + private readonly ConcurrentDictionary _socketIdByPeer = new(); + + public ZeroTierPeerRootSocketAffinity(IPEndPoint rootEndpoint) + { + ArgumentNullException.ThrowIfNull(rootEndpoint); + _rootEndpoint = rootEndpoint; + } + + public void Observe(NodeId peerNodeId, int localSocketId, IPEndPoint receivedVia) + { + ArgumentNullException.ThrowIfNull(receivedVia); + + if (!receivedVia.Equals(_rootEndpoint)) + { + return; + } + + _socketIdByPeer[peerNodeId] = localSocketId; + } + + public bool TryGet(NodeId peerNodeId, out int localSocketId) + => _socketIdByPeer.TryGetValue(peerNodeId, out localSocketId); + + public void Remove(NodeId peerNodeId) + { + _socketIdByPeer.TryRemove(peerNodeId, out _); + } +} From 0ae43ddf7a09f584007e92ed9169ca306794b945 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:26:37 +0100 Subject: [PATCH 187/296] Prefer peer root socket for root hello --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 39 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 19 +++++++++ 2 files changed, 58 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 1119864..c8d54e6 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -261,6 +261,45 @@ public async Task DataplaneRuntime_SendViaRootSockets_FansOutAcrossLocalSockets( Assert.Equal(new[] { 0, 1 }, socketIds); } + [Fact] + public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + await runtime.SendHelloViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var socketIds = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && decoded.Header.Verb == ZeroTierVerb.Hello) + .Select(send => send.LocalSocketId) + .Distinct() + .ToArray(); + + Assert.Equal(new[] { 1 }, socketIds); + } + private static ZeroTierDataplaneRuntime CreateRuntime( IZeroTierUdpTransport udp, ZeroTierIdentity localIdentity, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 47fd9c1..d8e6200 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -848,6 +848,19 @@ private bool HasConfirmedDirectPath(NodeId peerNodeId) private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) + { + await SendHelloPacketAsync( + rootSocketId, + peerNodeId, + physicalDestination: null, + _rootEndpoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); + return; + } + var localSockets = _udp.LocalSockets; if (localSockets.Count == 0) { @@ -1840,6 +1853,12 @@ internal IPEndPoint[] GetLocalDirectPathAdvertisementsForTests() internal Task SendViaRootSocketsForTestsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken = default) => SendViaRootSocketsAsync(packet, cancellationToken); + internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId) + => _peerRootSocketAffinity.Observe(peerNodeId, localSocketId, _rootEndpoint); + + internal Task SendHelloViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) + => SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken); + private NodeId[] GetPeersForMultipathMaintenance() { var peers = _peerPaths.GetPeersSnapshot(); From 4a5a1369abc2351485103ac1f6144b0f8655dd18 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:33:58 +0100 Subject: [PATCH 188/296] Align ok-hello wire format with upstream --- ZTSharp.Tests/ZeroTierHelloClientTests.cs | 74 +++++-------------- .../ZeroTierHelloOkPacketBuilderTests.cs | 3 +- .../Internal/ZeroTierHelloOkPacketBuilder.cs | 5 +- 3 files changed, 26 insertions(+), 56 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierHelloClientTests.cs b/ZTSharp.Tests/ZeroTierHelloClientTests.cs index 8f323b4..408d3d3 100644 --- a/ZTSharp.Tests/ZeroTierHelloClientTests.cs +++ b/ZTSharp.Tests/ZeroTierHelloClientTests.cs @@ -48,10 +48,10 @@ public async Task HelloRootsAsync_SendsHello_And_ParsesOk() Assert.Equal(rootIdentity.NodeId, ok.RootNodeId); Assert.Equal(rootEndpoint, ok.RootEndpoint); - Assert.Equal(11, ok.RemoteProtocolVersion); - Assert.Equal(1, ok.RemoteMajorVersion); - Assert.Equal(12, ok.RemoteMinorVersion); - Assert.Equal(0, ok.RemoteRevision); + Assert.Equal(ZeroTierHelloClient.AdvertisedProtocolVersion, ok.RemoteProtocolVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedMajorVersion, ok.RemoteMajorVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedMinorVersion, ok.RemoteMinorVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedRevision, ok.RemoteRevision); Assert.Equal(new IPEndPoint(IPAddress.Loopback, 9993), ok.ExternalSurfaceAddress); var observation = Assert.Single(ok.ExternalSurfaceObservations); Assert.Equal(0, observation.LocalSocketId); @@ -143,7 +143,7 @@ public async Task HelloAsync_SendsHello_AndCompletesOnOk() timeout: TimeSpan.FromSeconds(2), cancellationToken: CancellationToken.None); - Assert.Equal(11, remoteProtocolVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedProtocolVersion, remoteProtocolVersion); await serverTask; } @@ -272,30 +272,13 @@ private static async Task RunHelloServerOnceAsync( var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian( helloPacketBytes.AsSpan(ZeroTierPacketHeader.Length + 5, 8)); - var surface = new IPEndPoint(IPAddress.Loopback, 9993); - var surfaceLength = ZeroTierInetAddressCodec.GetSerializedLength(surface); - - var okPayloadLength = 1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength; - var okPayload = new byte[okPayloadLength]; - okPayload[0] = (byte)ZeroTierVerb.Hello; - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1, 8), hello.Header.PacketId); - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(9, 8), helloTimestamp); - okPayload[17] = 11; - okPayload[18] = 1; - okPayload[19] = 12; - BinaryPrimitives.WriteUInt16BigEndian(okPayload.AsSpan(20, 2), 0); - _ = ZeroTierInetAddressCodec.Serialize(surface, okPayload.AsSpan(22)); - - var okHeader = new ZeroTierPacketHeader( - PacketId: 2, - Destination: localIdentity.NodeId, - Source: rootIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Ok); - - var okPacket = ZeroTierPacketCodec.Encode(okHeader, okPayload); - ZeroTierPacketCrypto.Armor(okPacket, sharedKey, encryptPayload: true); + var okPacket = BuildHelloOkPacket( + rootNodeId: rootIdentity.NodeId, + localNodeId: localIdentity.NodeId, + sharedKey: sharedKey, + inRePacketId: hello.Header.PacketId, + helloTimestampEcho: helloTimestamp, + surface: new IPEndPoint(IPAddress.Loopback, 9993)); await transport.SendAsync(helloDatagram.RemoteEndPoint, okPacket).ConfigureAwait(false); } @@ -459,30 +442,13 @@ private static byte[] BuildHelloOkPacket( ulong helloTimestampEcho, IPEndPoint? surface = null) { - surface ??= new IPEndPoint(IPAddress.Loopback, 9993); - var surfaceLength = ZeroTierInetAddressCodec.GetSerializedLength(surface); - - var okPayloadLength = 1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength; - var okPayload = new byte[okPayloadLength]; - okPayload[0] = (byte)ZeroTierVerb.Hello; - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1, 8), inRePacketId); - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(9, 8), helloTimestampEcho); - okPayload[17] = 11; - okPayload[18] = 1; - okPayload[19] = 12; - BinaryPrimitives.WriteUInt16BigEndian(okPayload.AsSpan(20, 2), 0); - _ = ZeroTierInetAddressCodec.Serialize(surface, okPayload.AsSpan(22)); - - var okHeader = new ZeroTierPacketHeader( - PacketId: 2, - Destination: localNodeId, - Source: rootNodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Ok); - - var okPacket = ZeroTierPacketCodec.Encode(okHeader, okPayload); - ZeroTierPacketCrypto.Armor(okPacket, sharedKey, encryptPayload: true); - return okPacket; + return ZeroTierHelloOkPacketBuilder.BuildPacket( + packetId: 2, + destination: localNodeId, + source: rootNodeId, + inRePacketId: inRePacketId, + helloTimestampEcho: helloTimestampEcho, + externalSurfaceAddress: surface ?? new IPEndPoint(IPAddress.Loopback, 9993), + sharedKey: sharedKey); } } diff --git a/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs b/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs index f6fa031..d1c2550 100644 --- a/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs +++ b/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs @@ -54,7 +54,8 @@ public void BuildPacket_EncodesOkHello() Assert.True(ZeroTierInetAddressCodec.TryDeserialize(payload.Slice(22), out var parsedSurface, out var read)); Assert.Equal(surface, parsedSurface); - Assert.Equal(payload.Length - 22, read); + Assert.Equal(payload.Length - 24, read); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(22 + read, 2))); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs index 51c2c13..342ae50 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs @@ -24,7 +24,7 @@ public static byte[] BuildPacket( ? 0 : ZeroTierInetAddressCodec.GetSerializedLength(externalSurfaceAddress); - var payload = new byte[1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength]; + var payload = new byte[1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength + 2]; payload[0] = (byte)ZeroTierVerb.Hello; BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(1, 8), inRePacketId); BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(9, 8), helloTimestampEcho); @@ -38,6 +38,9 @@ public static byte[] BuildPacket( _ = ZeroTierInetAddressCodec.Serialize(externalSurfaceAddress, payload.AsSpan(22)); } + // Upstream always appends a u16 world-update payload length, even when zero. + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(payload.Length - 2, 2), 0); + var header = new ZeroTierPacketHeader( PacketId: packetId, Destination: destination, From 3978c615102441980bcd07225979182f0081ff49 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:38:30 +0100 Subject: [PATCH 189/296] Relax direct reply correlation for NAT remaps --- ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs | 27 ++++++++++++++ .../ZeroTierReplyCorrelationTests.cs | 17 +++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 14 ++++++++ .../Internal/ZeroTierPeerEchoManager.cs | 36 +++++++++++++------ .../Internal/ZeroTierReplyCorrelation.cs | 10 ++++++ 5 files changed, 93 insertions(+), 11 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierReplyCorrelationTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierReplyCorrelation.cs diff --git a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs index b6b6940..30036e9 100644 --- a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs @@ -116,6 +116,32 @@ await mgr.HandleEchoRequestAsync( Assert.Equal(1 + 8, payload.Length); } + [Fact] + public async Task EchoOk_AcceptsReplyKeyMatch_FromRemappedEndpoint() + { + var udp = new RecordingUdpTransport(); + + var now = 1_000L; + var localNodeId = new NodeId(0x2222222222); + var peerNodeId = new NodeId(0x1111111111); + var mgr = new ZeroTierPeerEchoManager(udp, localNodeId, getPeerProtocolVersion: _ => 12, nowUnixMs: () => now); + + var sendEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var replyEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4243); + var sharedKey = Enumerable.Repeat((byte)7, 48).ToArray(); + + await mgr.TrySendEchoProbeAsync(peerNodeId, localSocketId: 1, sendEndpoint, sharedKey, CancellationToken.None); + + var onWireEchoPacketId = BinaryPrimitives.ReadUInt64BigEndian(udp.Sends[0].Payload.Span.Slice(0, 8)); + var remappedReplyPacketId = (onWireEchoPacketId & 0xffffffff00000000UL) | 0x00000000000000a5UL; + + now += 25; + mgr.HandleEchoOk(peerNodeId, localSocketId: 1, replyEndpoint, remappedReplyPacketId, ReadOnlySpan.Empty); + + Assert.True(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, replyEndpoint, out var rttMs)); + Assert.Equal(25, rttMs); + Assert.False(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, sendEndpoint, out _)); + } private sealed class RecordingUdpTransport : IZeroTierUdpTransport { public List Sends { get; } = new(); @@ -154,3 +180,4 @@ public Task SendWithHopLimitAsync( private readonly record struct SendCall(int LocalSocketId, IPEndPoint RemoteEndPoint, ReadOnlyMemory Payload, int? HopLimit); } + diff --git a/ZTSharp.Tests/ZeroTierReplyCorrelationTests.cs b/ZTSharp.Tests/ZeroTierReplyCorrelationTests.cs new file mode 100644 index 0000000..eb3ed35 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierReplyCorrelationTests.cs @@ -0,0 +1,17 @@ +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierReplyCorrelationTests +{ + [Fact] + public void Matches_UsesUpper32Bits() + { + const ulong expectedPacketId = 0x0123456789abcdefUL; + const ulong matchingReplyPacketId = 0x01234567000000aaUL; + const ulong differentReplyPacketId = 0x89abcdef000000aaUL; + + Assert.True(ZeroTierReplyCorrelation.Matches(expectedPacketId, matchingReplyPacketId)); + Assert.False(ZeroTierReplyCorrelation.Matches(expectedPacketId, differentReplyPacketId)); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d8e6200..2dca61f 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1455,6 +1455,19 @@ private bool TryTakePendingHello(NodeId peerNodeId, ulong packetId, out PendingH return true; } + foreach (var candidate in _pendingHelloProbes) + { + if (!ZeroTierReplyCorrelation.Matches(candidate.Key, packetId)) + { + continue; + } + + if (_pendingHelloProbes.TryRemove(candidate.Key, out pending) && pending.PeerNodeId == peerNodeId) + { + return true; + } + } + pending = default; return false; } @@ -1901,3 +1914,4 @@ internal readonly record struct PendingHelloProbe( IPEndPoint? PhysicalDestination, long SentAtMs); + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs index d300431..adad13c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs @@ -160,14 +160,7 @@ public void HandleEchoOk( { ArgumentNullException.ThrowIfNull(remoteEndPoint); - if (!_pendingByPacketId.TryGetValue(inRePacketId, out var pending)) - { - return; - } - - if (pending.PathKey.PeerNodeId != peerNodeId || - pending.PathKey.Path.LocalSocketId != localSocketId || - !pending.PathKey.Path.RemoteEndPoint.Equals(remoteEndPoint)) + if (!TryTakePendingEcho(peerNodeId, inRePacketId, out var pending)) { return; } @@ -179,12 +172,32 @@ public void HandleEchoOk( return; } - if (!_pendingByPacketId.TryRemove(inRePacketId, out _)) + var observedPath = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); + _lastRttMsByPath[observedPath] = new RttEntry((int)rtt, LastUpdatedMs: now); + } + + private bool TryTakePendingEcho(NodeId peerNodeId, ulong inRePacketId, out PendingEcho pending) + { + if (_pendingByPacketId.TryRemove(inRePacketId, out pending) && pending.PathKey.PeerNodeId == peerNodeId) { - return; + return true; } - _lastRttMsByPath[pending.PathKey] = new RttEntry((int)rtt, LastUpdatedMs: now); + foreach (var candidate in _pendingByPacketId) + { + if (!ZeroTierReplyCorrelation.Matches(candidate.Key, inRePacketId)) + { + continue; + } + + if (_pendingByPacketId.TryRemove(candidate.Key, out pending) && pending.PathKey.PeerNodeId == peerNodeId) + { + return true; + } + } + + pending = default; + return false; } private void CleanupPendingIfNeeded(long nowUnixMs) @@ -248,3 +261,4 @@ private void CleanupCachesIfNeeded(long nowMs) } internal readonly record struct ZeroTierPeerEchoPathKey(NodeId PeerNodeId, ZeroTierPeerPhysicalPathKey Path); + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierReplyCorrelation.cs b/ZTSharp/ZeroTier/Internal/ZeroTierReplyCorrelation.cs new file mode 100644 index 0000000..2941e0e --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierReplyCorrelation.cs @@ -0,0 +1,10 @@ +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierReplyCorrelation +{ + public static uint GetReplyKey(ulong packetId) + => (uint)(packetId >> 32); + + public static bool Matches(ulong expectedPacketId, ulong actualPacketId) + => GetReplyKey(expectedPacketId) == GetReplyKey(actualPacketId); +} From 0d230c1e67aa7aadad978769e9fe9546206deb21 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:46:34 +0100 Subject: [PATCH 190/296] Relay public rendezvous hints to peers --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 57 +++++++++++++++---- .../Internal/ZeroTierDataplaneRuntime.cs | 52 +++++++++++++++++ .../Protocol/ZeroTierRendezvousCodec.cs | 22 +++++++ 3 files changed, 119 insertions(+), 12 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index c8d54e6..dea2366 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -300,6 +300,50 @@ public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() Assert.Equal(new[] { 1 }, socketIds); } + [Fact] + public async Task DataplaneRuntime_SendPublicRendezvousViaRoot_UsesPeerRootSocketAffinity_AndPublicAdvertisements() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 60001)) + ]); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + await runtime.SendPublicRendezvousViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var packet = Assert.Single(udp.GetSendsSnapshot(), send => send.RemoteEndPoint.Equals(RootEndpoint)); + Assert.Equal(1, packet.LocalSocketId); + Assert.True(ZeroTierPacketCrypto.Dearmor(packet.Payload, sharedKey)); + Assert.True(ZeroTierPacketCodec.TryDecode(packet.Payload, out var decoded)); + Assert.Equal(ZeroTierVerb.Rendezvous, decoded.Header.Verb); + Assert.True(ZeroTierRendezvousCodec.TryParse(packet.Payload.AsSpan(ZeroTierPacketHeader.IndexPayload), out var rendezvous)); + Assert.Equal(localIdentity.NodeId, rendezvous.With); + Assert.Equal(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), rendezvous.Endpoint); + } + private static ZeroTierDataplaneRuntime CreateRuntime( IZeroTierUdpTransport udp, ZeroTierIdentity localIdentity, @@ -365,18 +409,7 @@ private static byte[] BuildPushDirectPathsPacket( } private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) - { - var addressBytes = endpoint.Address.GetAddressBytes(); - var payload = new byte[1 + 5 + 2 + 1 + addressBytes.Length]; - - payload[0] = 0; - ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, 5), with.Value); - BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(6, 2), (ushort)endpoint.Port); - payload[8] = (byte)addressBytes.Length; - addressBytes.CopyTo(payload.AsSpan(9)); - - return payload; - } + => ZeroTierRendezvousCodec.BuildPayload(with, endpoint); private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint) { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 2dca61f..c517e81 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -717,6 +717,7 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } @@ -764,6 +765,7 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -931,6 +933,53 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } + private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) + .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) + .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) + .ToArray(); + if (advertisements.Length == 0) + { + return; + } + + try + { + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + for (var i = 0; i < advertisements.Length; i++) + { + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, advertisements[i])); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisements[i]}."); + } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { var localAdvertisements = GetLocalDirectPathAdvertisements(); @@ -1872,6 +1921,9 @@ internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId internal Task SendHelloViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) => SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken); + internal Task SendPublicRendezvousViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) + => SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken); + private NodeId[] GetPeersForMultipathMaintenance() { var peers = _peerPaths.GetPeersSnapshot(); diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs index ada7f5b..d7087e2 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs @@ -13,6 +13,28 @@ internal static class ZeroTierRendezvousCodec private const int AddressLength = 5; private const int MinPayloadLength = 1 + AddressLength + 2 + 1; + public static byte[] BuildPayload(NodeId with, IPEndPoint endpoint, byte flags = 0) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var address = endpoint.Address.IsIPv4MappedToIPv6 + ? endpoint.Address.MapToIPv4() + : endpoint.Address; + var addressBytes = address.GetAddressBytes(); + if (endpoint.Port == 0 || (addressBytes.Length != 4 && addressBytes.Length != 16)) + { + throw new ArgumentOutOfRangeException(nameof(endpoint), endpoint, "Rendezvous endpoint must be IPv4 or IPv6 with a non-zero port."); + } + + var payload = new byte[MinPayloadLength + addressBytes.Length]; + payload[0] = flags; + ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, AddressLength), with.Value); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(1 + AddressLength, 2), (ushort)endpoint.Port); + payload[1 + AddressLength + 2] = (byte)addressBytes.Length; + addressBytes.CopyTo(payload.AsSpan(MinPayloadLength)); + return payload; + } + public static bool TryParse(ReadOnlySpan payload, out ZeroTierRendezvous rendezvous) { if (payload.Length < MinPayloadLength) From 7589a689035eecf5da0f6e5e6ac356b8d59e7ff8 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:53:12 +0100 Subject: [PATCH 191/296] Prefer affine public rendezvous hint --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 41 ++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 64 +++++++++++-------- 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index dea2366..c5188d1 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -344,6 +344,47 @@ public async Task DataplaneRuntime_SendPublicRendezvousViaRoot_UsesPeerRootSocke Assert.Equal(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), rendezvous.Endpoint); } + [Fact] + public async Task DataplaneRuntime_SendPublicRendezvousViaRoot_PrefersAffineSocketPublicSurface() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + ]); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + await runtime.SendPublicRendezvousViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var packet = Assert.Single(udp.GetSendsSnapshot(), send => send.RemoteEndPoint.Equals(RootEndpoint)); + Assert.Equal(1, packet.LocalSocketId); + Assert.True(ZeroTierPacketCrypto.Dearmor(packet.Payload, sharedKey)); + Assert.True(ZeroTierRendezvousCodec.TryParse(packet.Payload.AsSpan(ZeroTierPacketHeader.IndexPayload), out var rendezvous)); + Assert.Equal(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001), rendezvous.Endpoint); + } + private static ZeroTierDataplaneRuntime CreateRuntime( IZeroTierUdpTransport udp, ZeroTierIdentity localIdentity, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c517e81..336c907 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -935,11 +935,8 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { - var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) - .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) - .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) - .ToArray(); - if (advertisements.Length == 0) + var advertisement = GetPreferredPublicRendezvousAdvertisement(peerNodeId); + if (advertisement is null) { return; } @@ -947,29 +944,26 @@ private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sh try { var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - for (var i = 0; i < advertisements.Length; i++) - { - var packet = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Rendezvous), - ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, advertisements[i])); - ZeroTierPacketCrypto.Armor( - packet, - ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), - encryptPayload: true); - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisements[i]}."); - } + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, advertisement)); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); - await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisement}."); } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) { @@ -980,6 +974,24 @@ private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sh } } + private IPEndPoint? GetPreferredPublicRendezvousAdvertisement(NodeId peerNodeId) + { + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) + { + var socketPublicSurface = _surfaceAddresses + .GetSnapshot(localSocketId) + .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) + .FirstOrDefault(); + if (socketPublicSurface is not null) + { + return socketPublicSurface; + } + } + + return GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) + .FirstOrDefault(ZeroTierDirectEndpointSelection.IsPublicEndpoint); + } + private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { var localAdvertisements = GetLocalDirectPathAdvertisements(); From 7bf882a8c0a7ae4037c8ec024f30f1942f20e86c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:59:50 +0100 Subject: [PATCH 192/296] Prefer single socket for direct hints --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 23 ++-- .../Internal/ZeroTierDataplaneRuntime.cs | 35 +++--- ...ierRelayRendezvousAdvertisementSelector.cs | 108 ++++++++++++++++++ 3 files changed, 135 insertions(+), 31 deletions(-) create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index c5188d1..fb502a0 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -90,7 +90,7 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() } [Fact] - public async Task DataplaneRuntime_MultipathPushDirectPaths_Probes_FromAllLocalSockets() + public async Task DataplaneRuntime_MultipathPushDirectPaths_PrefersReceivingSocketForInitialProbe() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), @@ -123,19 +123,16 @@ public async Task DataplaneRuntime_MultipathPushDirectPaths_Probes_FromAllLocalS udp.EnqueueInbound(localSocketId: 1, RootEndpoint, BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, udp.LocalSockets[1].LocalEndpoint)); udp.EnqueueInbound(localSocketId: 1, RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); - await WaitForConditionAsync( - () => - { - var seenSocketIds = udp.GetSendsSnapshot() - .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint) && - send.Payload.Length > 4 && - ZeroTierPacketCodec.TryDecode(send.Payload, out _)) - .Select(send => send.LocalSocketId) - .Distinct() - .ToArray(); - return seenSocketIds.Contains(0) && seenSocketIds.Contains(1); - }, + var probe = await WaitForSendAsync( + udp, + send => send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(hintedEndpoint) && + send.Payload.Length > 4 && + ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && + decoded.Header.Verb == ZeroTierVerb.Hello, TimeSpan.FromSeconds(2)); + + Assert.Equal(1, probe.LocalSocketId); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 336c907..30fde9c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -782,7 +782,10 @@ private async Task ProbeHintedDirectEndpointsAsync( var endpointsToProbe = _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { - var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpointsToProbe[i]); + var localSocketIds = _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpointsToProbe[i], + includeFallbackLocalSockets: true); for (var s = 0; s < localSocketIds.Length; s++) { await SendDirectBootstrapProbeAsync( @@ -975,22 +978,13 @@ private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sh } private IPEndPoint? GetPreferredPublicRendezvousAdvertisement(NodeId peerNodeId) - { - if (_peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) - { - var socketPublicSurface = _surfaceAddresses - .GetSnapshot(localSocketId) - .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) - .FirstOrDefault(); - if (socketPublicSurface is not null) - { - return socketPublicSurface; - } - } - - return GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) - .FirstOrDefault(ZeroTierDirectEndpointSelection.IsPublicEndpoint); - } + => ZeroTierRelayRendezvousAdvertisementSelector.Select( + GetPeerAwareLocalDirectPathAdvertisements(peerNodeId), + _peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId) + ? _surfaceAddresses.GetSnapshot(localSocketId) + : Array.Empty(), + GetOrCreateDirectEndpointManager(peerNodeId).Endpoints, + _peerPaths.GetSnapshot(peerNodeId)); private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { @@ -1147,7 +1141,10 @@ private async ValueTask HandleDirectEndpointHintAsync( ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId} fullHello={forceFullHello}."); } - var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + var localSocketIds = _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true); for (var i = 0; i < localSocketIds.Length; i++) { await SendDirectBootstrapProbeAsync( @@ -1979,3 +1976,5 @@ internal readonly record struct PendingHelloProbe( long SentAtMs); + + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs new file mode 100644 index 0000000..d20f328 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs @@ -0,0 +1,108 @@ +using System.Net; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierRelayRendezvousAdvertisementSelector +{ + public static IPEndPoint? Select( + IReadOnlyList localAdvertisements, + IReadOnlyList preferredSocketAdvertisements, + IReadOnlyList hintedPeerEndpoints, + IReadOnlyList observedPeerPaths) + { + ArgumentNullException.ThrowIfNull(localAdvertisements); + ArgumentNullException.ThrowIfNull(preferredSocketAdvertisements); + ArgumentNullException.ThrowIfNull(hintedPeerEndpoints); + ArgumentNullException.ThrowIfNull(observedPeerPaths); + + var preferredFamilies = GetPreferredFamilies(hintedPeerEndpoints, observedPeerPaths); + return SelectPublic(preferredSocketAdvertisements, preferredFamilies) ?? + SelectPublic(localAdvertisements, preferredFamilies); + } + + private static AddressFamily[] GetPreferredFamilies( + IReadOnlyList hintedPeerEndpoints, + IReadOnlyList observedPeerPaths) + { + var hasPublicV4 = false; + var hasPublicV6 = false; + + for (var i = 0; i < hintedPeerEndpoints.Count; i++) + { + ObserveFamily(hintedPeerEndpoints[i], ref hasPublicV4, ref hasPublicV6); + } + + for (var i = 0; i < observedPeerPaths.Count; i++) + { + ObserveFamily(observedPeerPaths[i].RemoteEndPoint, ref hasPublicV4, ref hasPublicV6); + } + + if (hasPublicV6 && hasPublicV4) + { + return [AddressFamily.InterNetworkV6, AddressFamily.InterNetwork]; + } + + if (hasPublicV6) + { + return [AddressFamily.InterNetworkV6]; + } + + if (hasPublicV4) + { + return [AddressFamily.InterNetwork]; + } + + return [AddressFamily.InterNetworkV6, AddressFamily.InterNetwork]; + } + + private static void ObserveFamily(IPEndPoint endpoint, ref bool hasPublicV4, ref bool hasPublicV6) + { + ArgumentNullException.ThrowIfNull(endpoint); + + if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + return; + } + + switch (endpoint.AddressFamily) + { + case AddressFamily.InterNetwork: + hasPublicV4 = true; + break; + case AddressFamily.InterNetworkV6: + hasPublicV6 = true; + break; + } + } + + private static IPEndPoint? SelectPublic( + IReadOnlyList candidates, + AddressFamily[] preferredFamilies) + { + for (var i = 0; i < preferredFamilies.Length; i++) + { + for (var c = 0; c < candidates.Count; c++) + { + var candidate = candidates[c]; + if (candidate.AddressFamily == preferredFamilies[i] && + ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidate)) + { + return candidate; + } + } + } + + for (var i = 0; i < candidates.Count; i++) + { + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidates[i])) + { + return candidates[i]; + } + } + + return null; + } +} + + From db4f9ece879af482e37321984f8c484899b7e415 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 04:19:13 +0100 Subject: [PATCH 193/296] Align direct bootstrap with upstream hints --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 171 ------------------ .../Internal/ZeroTierDataplanePeerSecurity.cs | 14 ++ .../Internal/ZeroTierDataplaneRuntime.cs | 84 +++------ ...ierRelayRendezvousAdvertisementSelector.cs | 108 ----------- 4 files changed, 41 insertions(+), 336 deletions(-) delete mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index fb502a0..87ba9b3 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -49,92 +49,6 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() Assert.Equal(2, holePunch.HopLimit); } - [Fact] - public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsDirectProbe() - { - await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - Assert.True(peerIdentity.HasPrivateKey); - Assert.True(peerIdentity.LocallyValidate()); - - var rootNodeId = new NodeId(0x1111111111); - var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); - var rootKey = RandomNumberGenerator.GetBytes(48); - - await using var runtime = CreateRuntime( - udp, - localIdentity, - rootNodeId, - rootKey, - multipath: new ZeroTierMultipathOptions { Enabled = true }); - - var sharedKey = new byte[48]; - ZeroTierC25519.Agree(peerIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); - - udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, udp.LocalSockets[0].LocalEndpoint)); - udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); - - var probe = await WaitForSendAsync( - udp, - send => send.RemoteEndPoint.Equals(hintedEndpoint) && - send.Payload.Length > 4 && - ZeroTierPacketCodec.TryDecode(send.Payload, out _), - TimeSpan.FromSeconds(2)); - - Assert.True(ZeroTierPacketCodec.TryDecode(probe.Payload, out var decoded)); - Assert.Equal(0, probe.LocalSocketId); - Assert.Equal(ZeroTierVerb.Hello, decoded.Header.Verb); - } - - [Fact] - public async Task DataplaneRuntime_MultipathPushDirectPaths_PrefersReceivingSocketForInitialProbe() - { - await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - Assert.True(peerIdentity.HasPrivateKey); - Assert.True(peerIdentity.LocallyValidate()); - - var rootNodeId = new NodeId(0x1111111111); - var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); - var rootKey = RandomNumberGenerator.GetBytes(48); - - await using var runtime = CreateRuntime( - udp, - localIdentity, - rootNodeId, - rootKey, - multipath: new ZeroTierMultipathOptions { Enabled = true }, - initialExternalSurfaceObservations: - [ - new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), - new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) - ]); - - var sharedKey = new byte[48]; - ZeroTierC25519.Agree(peerIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); - - udp.EnqueueInbound(localSocketId: 1, RootEndpoint, BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, udp.LocalSockets[1].LocalEndpoint)); - udp.EnqueueInbound(localSocketId: 1, RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); - - var probe = await WaitForSendAsync( - udp, - send => send.LocalSocketId == 1 && - send.RemoteEndPoint.Equals(hintedEndpoint) && - send.Payload.Length > 4 && - ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && - decoded.Header.Verb == ZeroTierVerb.Hello, - TimeSpan.FromSeconds(2)); - - Assert.Equal(1, probe.LocalSocketId); - } - [Fact] public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() { @@ -297,91 +211,6 @@ public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() Assert.Equal(new[] { 1 }, socketIds); } - [Fact] - public async Task DataplaneRuntime_SendPublicRendezvousViaRoot_UsesPeerRootSocketAffinity_AndPublicAdvertisements() - { - await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - Assert.True(peerIdentity.HasPrivateKey); - Assert.True(peerIdentity.LocallyValidate()); - - var rootNodeId = new NodeId(0x1111111111); - var rootKey = RandomNumberGenerator.GetBytes(48); - await using var runtime = CreateRuntime( - udp, - localIdentity, - rootNodeId, - rootKey, - multipath: new ZeroTierMultipathOptions { Enabled = true }, - planetId: 1, - planetTimestamp: 1, - initialExternalSurfaceObservations: - [ - new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), - new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 60001)) - ]); - - var sharedKey = new byte[48]; - ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); - - runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); - await runtime.SendPublicRendezvousViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); - - var packet = Assert.Single(udp.GetSendsSnapshot(), send => send.RemoteEndPoint.Equals(RootEndpoint)); - Assert.Equal(1, packet.LocalSocketId); - Assert.True(ZeroTierPacketCrypto.Dearmor(packet.Payload, sharedKey)); - Assert.True(ZeroTierPacketCodec.TryDecode(packet.Payload, out var decoded)); - Assert.Equal(ZeroTierVerb.Rendezvous, decoded.Header.Verb); - Assert.True(ZeroTierRendezvousCodec.TryParse(packet.Payload.AsSpan(ZeroTierPacketHeader.IndexPayload), out var rendezvous)); - Assert.Equal(localIdentity.NodeId, rendezvous.With); - Assert.Equal(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), rendezvous.Endpoint); - } - - [Fact] - public async Task DataplaneRuntime_SendPublicRendezvousViaRoot_PrefersAffineSocketPublicSurface() - { - await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - Assert.True(peerIdentity.HasPrivateKey); - Assert.True(peerIdentity.LocallyValidate()); - - var rootNodeId = new NodeId(0x1111111111); - var rootKey = RandomNumberGenerator.GetBytes(48); - await using var runtime = CreateRuntime( - udp, - localIdentity, - rootNodeId, - rootKey, - multipath: new ZeroTierMultipathOptions { Enabled = true }, - planetId: 1, - planetTimestamp: 1, - initialExternalSurfaceObservations: - [ - new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), - new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) - ]); - - var sharedKey = new byte[48]; - ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); - - runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); - await runtime.SendPublicRendezvousViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); - - var packet = Assert.Single(udp.GetSendsSnapshot(), send => send.RemoteEndPoint.Equals(RootEndpoint)); - Assert.Equal(1, packet.LocalSocketId); - Assert.True(ZeroTierPacketCrypto.Dearmor(packet.Payload, sharedKey)); - Assert.True(ZeroTierRendezvousCodec.TryParse(packet.Payload.AsSpan(ZeroTierPacketHeader.IndexPayload), out var rendezvous)); - Assert.Equal(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001), rendezvous.Endpoint); - } - private static ZeroTierDataplaneRuntime CreateRuntime( IZeroTierUdpTransport udp, ZeroTierIdentity localIdentity, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs index d515d77..9a73593 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs @@ -79,6 +79,20 @@ internal void ObservePeerVersion(NodeId peerNodeId, byte peerProtocolVersion, by peerRevision); } + internal void PrimePeerForTests( + NodeId peerNodeId, + byte[] sharedKey, + byte peerProtocolVersion, + byte peerMajorVersion, + byte peerMinorVersion, + ushort peerRevision) + { + ArgumentNullException.ThrowIfNull(sharedKey); + + CachePeerKey(peerNodeId, sharedKey.ToArray(), Environment.TickCount64); + ObservePeerVersion(peerNodeId, peerProtocolVersion, peerMajorVersion, peerMinorVersion, peerRevision); + } + public bool TryGetPeerKey(NodeId peerNodeId, out byte[] key) { key = Array.Empty(); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 30fde9c..de922be 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -717,7 +717,6 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } @@ -765,7 +764,6 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -793,7 +791,7 @@ await SendDirectBootstrapProbeAsync( localSocketIds[s], endpointsToProbe[i], sharedKey, - forceFullHello: s == 0, + forceFullHello: false, DirectHelloMinIntervalMs, cancellationToken) .ConfigureAwait(false); @@ -936,56 +934,6 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } - private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) - { - var advertisement = GetPreferredPublicRendezvousAdvertisement(peerNodeId); - if (advertisement is null) - { - return; - } - - try - { - var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - var packet = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Rendezvous), - ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, advertisement)); - ZeroTierPacketCrypto.Armor( - packet, - ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), - encryptPayload: true); - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisement}."); - } - - await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); - } - } - } - - private IPEndPoint? GetPreferredPublicRendezvousAdvertisement(NodeId peerNodeId) - => ZeroTierRelayRendezvousAdvertisementSelector.Select( - GetPeerAwareLocalDirectPathAdvertisements(peerNodeId), - _peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId) - ? _surfaceAddresses.GetSnapshot(localSocketId) - : Array.Empty(), - GetOrCreateDirectEndpointManager(peerNodeId).Endpoints, - _peerPaths.GetSnapshot(peerNodeId)); - private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { var localAdvertisements = GetLocalDirectPathAdvertisements(); @@ -1147,12 +1095,12 @@ private async ValueTask HandleDirectEndpointHintAsync( includeFallbackLocalSockets: true); for (var i = 0; i < localSocketIds.Length; i++) { - await SendDirectBootstrapProbeAsync( + await SendDirectBootstrapProbeAsync( peerNodeId, localSocketIds[i], endpoint, sharedKey, - forceFullHello || i == 0, + forceFullHello, DirectHelloMinIntervalMs, cancellationToken) .ConfigureAwait(false); @@ -1930,8 +1878,30 @@ internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId internal Task SendHelloViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) => SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken); - internal Task SendPublicRendezvousViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) - => SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken); + internal void PrimePeerForTests( + ZeroTierIdentity peerIdentity, + byte peerProtocolVersion = ZeroTierHelloClient.AdvertisedProtocolVersion, + byte peerMajorVersion = ZeroTierHelloClient.AdvertisedMajorVersion, + byte peerMinorVersion = ZeroTierHelloClient.AdvertisedMinorVersion, + ushort peerRevision = ZeroTierHelloClient.AdvertisedRevision) + { + ArgumentNullException.ThrowIfNull(peerIdentity); + + if (_localIdentity.PrivateKey is null) + { + throw new InvalidOperationException("Local identity private key is required for test priming."); + } + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(_localIdentity.PrivateKey, peerIdentity.PublicKey, sharedKey); + _peerSecurity.PrimePeerForTests( + peerIdentity.NodeId, + sharedKey, + peerProtocolVersion, + peerMajorVersion, + peerMinorVersion, + peerRevision); + } private NodeId[] GetPeersForMultipathMaintenance() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs deleted file mode 100644 index d20f328..0000000 --- a/ZTSharp/ZeroTier/Internal/ZeroTierRelayRendezvousAdvertisementSelector.cs +++ /dev/null @@ -1,108 +0,0 @@ -using System.Net; -using System.Net.Sockets; - -namespace ZTSharp.ZeroTier.Internal; - -internal static class ZeroTierRelayRendezvousAdvertisementSelector -{ - public static IPEndPoint? Select( - IReadOnlyList localAdvertisements, - IReadOnlyList preferredSocketAdvertisements, - IReadOnlyList hintedPeerEndpoints, - IReadOnlyList observedPeerPaths) - { - ArgumentNullException.ThrowIfNull(localAdvertisements); - ArgumentNullException.ThrowIfNull(preferredSocketAdvertisements); - ArgumentNullException.ThrowIfNull(hintedPeerEndpoints); - ArgumentNullException.ThrowIfNull(observedPeerPaths); - - var preferredFamilies = GetPreferredFamilies(hintedPeerEndpoints, observedPeerPaths); - return SelectPublic(preferredSocketAdvertisements, preferredFamilies) ?? - SelectPublic(localAdvertisements, preferredFamilies); - } - - private static AddressFamily[] GetPreferredFamilies( - IReadOnlyList hintedPeerEndpoints, - IReadOnlyList observedPeerPaths) - { - var hasPublicV4 = false; - var hasPublicV6 = false; - - for (var i = 0; i < hintedPeerEndpoints.Count; i++) - { - ObserveFamily(hintedPeerEndpoints[i], ref hasPublicV4, ref hasPublicV6); - } - - for (var i = 0; i < observedPeerPaths.Count; i++) - { - ObserveFamily(observedPeerPaths[i].RemoteEndPoint, ref hasPublicV4, ref hasPublicV6); - } - - if (hasPublicV6 && hasPublicV4) - { - return [AddressFamily.InterNetworkV6, AddressFamily.InterNetwork]; - } - - if (hasPublicV6) - { - return [AddressFamily.InterNetworkV6]; - } - - if (hasPublicV4) - { - return [AddressFamily.InterNetwork]; - } - - return [AddressFamily.InterNetworkV6, AddressFamily.InterNetwork]; - } - - private static void ObserveFamily(IPEndPoint endpoint, ref bool hasPublicV4, ref bool hasPublicV6) - { - ArgumentNullException.ThrowIfNull(endpoint); - - if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) - { - return; - } - - switch (endpoint.AddressFamily) - { - case AddressFamily.InterNetwork: - hasPublicV4 = true; - break; - case AddressFamily.InterNetworkV6: - hasPublicV6 = true; - break; - } - } - - private static IPEndPoint? SelectPublic( - IReadOnlyList candidates, - AddressFamily[] preferredFamilies) - { - for (var i = 0; i < preferredFamilies.Length; i++) - { - for (var c = 0; c < candidates.Count; c++) - { - var candidate = candidates[c]; - if (candidate.AddressFamily == preferredFamilies[i] && - ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidate)) - { - return candidate; - } - } - } - - for (var i = 0; i < candidates.Count; i++) - { - if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidates[i])) - { - return candidates[i]; - } - } - - return null; - } -} - - From 61c563dd3e798f26b1a5039c35936cc4bba6a185 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 04:29:44 +0100 Subject: [PATCH 194/296] Fan out push-direct probes across sockets --- ...TierDirectEndpointManagerPushFlagsTests.cs | 38 ++++++++++++++++--- .../Internal/ZeroTierDataplaneRuntime.cs | 16 +++++--- .../Internal/ZeroTierDirectEndpointManager.cs | 27 +++++++++++-- 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 67440c0..45b90b9 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -66,7 +66,7 @@ public async Task PushDirectPaths_KnownEndpoint_IsRebootstrappedAndUpdatesSocket udp, relay, peerNodeId, - handleDirectEndpointHintAsync: (_, _, endpoint, _, _) => + handleDirectEndpointHintAsync: (_, _, endpoint, _, _, _) => { hintedEndpoints.Add(endpoint); return ValueTask.CompletedTask; @@ -102,6 +102,34 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Empty(udp.Sends); } + [Fact] + public async Task PushDirectPaths_NormalHint_RequestsAllEligibleSockets() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, false, true, 1), hints[0]); + } + [Fact] public async Task PushDirectPaths_ClusterRedirect_UsesReceivingSocketAndForcesFullHello() { @@ -109,14 +137,14 @@ public async Task PushDirectPaths_ClusterRedirect_UsesReceivingSocketAndForcesFu var relay = new IPEndPoint(IPAddress.Loopback, 9999); var peerNodeId = new NodeId(0x1111111111); - var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, int LocalSocketId)>(); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); var manager = new ZeroTierDirectEndpointManager( udp, relay, peerNodeId, - handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, _) => + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => { - hints.Add((endpoint, forceFullHello, localSocketId)); + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); return ValueTask.CompletedTask; }); @@ -127,7 +155,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, true, 1), hints[0]); + Assert.Equal((endpoint, true, false, 1), hints[0]); Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index de922be..ef3eb78 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1061,6 +1061,7 @@ private async ValueTask HandleDirectEndpointHintAsync( int receivedLocalSocketId, IPEndPoint endpoint, bool forceFullHello, + bool useAllEligibleLocalSockets, CancellationToken cancellationToken) { if (!_multipath.Enabled) @@ -1086,16 +1087,19 @@ private async ValueTask HandleDirectEndpointHintAsync( if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId} fullHello={forceFullHello}."); + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId} fullHello={forceFullHello} allSockets={useAllEligibleLocalSockets}."); } - var localSocketIds = _directHintPlanner.GetRotatingSocketIds( - peerNodeId, - endpoint, - includeFallbackLocalSockets: true); + var localSocketIds = useAllEligibleLocalSockets + ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint) + : _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true); for (var i = 0; i < localSocketIds.Length; i++) { - await SendDirectBootstrapProbeAsync( + await SendDirectBootstrapProbeAsync( peerNodeId, localSocketIds[i], endpoint, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index fcb5ff8..cfdddd9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -26,7 +26,7 @@ internal sealed class ZeroTierDirectEndpointManager private readonly IZeroTierUdpTransport _udp; private readonly IPEndPoint _relayEndpoint; private readonly NodeId _remoteNodeId; - private readonly Func? _handleDirectEndpointHintAsync; + private readonly Func? _handleDirectEndpointHintAsync; private readonly Func? _shouldAcceptEndpoint; private readonly object _lock = new(); @@ -41,7 +41,7 @@ public ZeroTierDirectEndpointManager( IZeroTierUdpTransport udp, IPEndPoint relayEndpoint, NodeId remoteNodeId, - Func? handleDirectEndpointHintAsync = null, + Func? handleDirectEndpointHintAsync = null, Func? shouldAcceptEndpoint = null) { ArgumentNullException.ThrowIfNull(udp); @@ -119,7 +119,14 @@ public async ValueTask HandleRendezvousFromRootAsync( hopLimit: RendezvousHolePunchHopLimit); if (_handleDirectEndpointHintAsync is not null) { - await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, false, cancellationToken).ConfigureAwait(false); + await _handleDirectEndpointHintAsync( + _remoteNodeId, + receivedLocalSocketId, + endpoint, + false, + false, + cancellationToken) + .ConfigureAwait(false); } } @@ -158,6 +165,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( var redirect = new List(); var add = new List(); var forceFullHelloByEndpoint = new Dictionary(StringComparer.Ordinal); + var useAllEligibleLocalSocketsByEndpoint = new Dictionary(StringComparer.Ordinal); for (var i = 0; i < paths.Length; i++) { @@ -169,6 +177,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { forget.Add(key); forceFullHelloByEndpoint.Remove(key); + useAllEligibleLocalSocketsByEndpoint.Remove(key); continue; } @@ -176,11 +185,13 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { redirect.Add(endpoint); forceFullHelloByEndpoint[key] = true; + useAllEligibleLocalSocketsByEndpoint[key] = false; } else { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); } } @@ -222,7 +233,15 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( if (_handleDirectEndpointHintAsync is not null) { var forceFullHello = forceFullHelloByEndpoint[FormatEndpointKey(endpoint)]; - await _handleDirectEndpointHintAsync(_remoteNodeId, receivedLocalSocketId, endpoint, forceFullHello, cancellationToken).ConfigureAwait(false); + var useAllEligibleLocalSockets = useAllEligibleLocalSocketsByEndpoint[FormatEndpointKey(endpoint)]; + await _handleDirectEndpointHintAsync( + _remoteNodeId, + receivedLocalSocketId, + endpoint, + forceFullHello, + useAllEligibleLocalSockets, + cancellationToken) + .ConfigureAwait(false); } } From 70dc25b0530f9d1d5a8681c80c4da069e994c460 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 04:51:35 +0100 Subject: [PATCH 195/296] Tighten direct hint socket selection --- ...ZeroTierDirectHintPathPlannerRouteTests.cs | 69 ++++++++++++++++++ ...oTierDirectPathSocketAdmissibilityTests.cs | 48 +++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 20 +++++- .../Internal/ZeroTierDirectHintPathPlanner.cs | 19 +++-- .../ZeroTierDirectPathSocketAdmissibility.cs | 66 +++++++++++++++++ .../Internal/ZeroTierDirectRouteResolver.cs | 71 +++++++++++++++++++ 6 files changed, 285 insertions(+), 8 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs create mode 100644 ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectRouteResolver.cs diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs new file mode 100644 index 0000000..ea691d6 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs @@ -0,0 +1,69 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectHintPathPlannerRouteTests +{ + [Fact] + public void GetPreferredAndFallbackSocketIds_UsesRouteSelectedSocket_ForPublicEndpoint() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("192.168.224.1"), 10001))); + var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); + surfaces.Observe( + new NodeId(1), + 0, + new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); + surfaces.Observe( + new NodeId(1), + 1, + new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)); + + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + surfaces, + _ => manager, + new ZeroTierDirectPathSocketAdmissibility( + new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24), + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("192.168.224.1"), 24) + ]), + new ZeroTierDirectRouteResolver(_ => IPAddress.Parse("10.0.0.112")))); + + var sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + + Assert.Equal([0], sockets); + } + + private sealed class StubUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) : IZeroTierUdpTransport + { + public IReadOnlyList LocalSockets { get; } = localSockets; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs b/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs new file mode 100644 index 0000000..72a932e --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs @@ -0,0 +1,48 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectPathSocketAdmissibilityTests +{ + [Fact] + public void ShouldUsePath_AllowsOnlySocketSelectedByRoute_ForPublicEndpoint() + { + var admissibility = CreateAdmissibility(endpoint => + endpoint.Address.Equals(IPAddress.Parse("176.66.90.119")) + ? IPAddress.Parse("10.0.0.112") + : null); + + var accepted = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + var rejected = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("192.168.224.1"), 10001)), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + + Assert.True(accepted); + Assert.False(rejected); + } + + [Fact] + public void ShouldUsePath_FallsBackToAddressFamily_WhenRouteUnknown() + { + var admissibility = CreateAdmissibility(_ => null); + + var accepted = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + + Assert.True(accepted); + } + + private static ZeroTierDirectPathSocketAdmissibility CreateAdmissibility(Func resolveLocalAddress) + => new( + new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24), + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("192.168.224.1"), 24) + ]), + new ZeroTierDirectRouteResolver(resolveLocalAddress)); +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index ef3eb78..d656c8b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -24,6 +24,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const int DirectBootstrapHintProbeBudget = 2; private const int DirectHintMaintenanceProbeBudget = 2; private const long DirectHintFullHelloIntervalMs = 60_000; + private const long DirectOnlyHintHelloIntervalMs = 5_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -205,7 +206,13 @@ public ZeroTierDataplaneRuntime( _peerRootSocketAffinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); _localDirectAdvertisementPlanner = new ZeroTierLocalDirectPathAdvertisementPlanner(_directEndpointPolicy); - _directHintPlanner = new ZeroTierDirectHintPathPlanner(udp, _surfaceAddresses, GetOrCreateDirectEndpointManager); + _directHintPlanner = new ZeroTierDirectHintPathPlanner( + udp, + _surfaceAddresses, + GetOrCreateDirectEndpointManager, + new ZeroTierDirectPathSocketAdmissibility( + _directEndpointPolicy, + new ZeroTierDirectRouteResolver())); var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); @@ -1508,6 +1515,9 @@ private void CleanupPendingHellosIfNeeded(long nowMs) private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) => _peerNegotiation.TryGetRemoteUtility(peerNodeId, localSocketId, remoteEndPoint, out var util) ? util : (short)0; + private bool ShouldForceFullHelloForHintedCandidate(NodeId peerNodeId) + => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) @@ -1566,14 +1576,18 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati for (var c = 0; c < hintedCandidates.Length; c++) { var candidate = hintedCandidates[c]; + var forceFullHello = ShouldForceFullHelloForHintedCandidate(peerNodeId); + var minIntervalMs = forceFullHello + ? DirectOnlyHintHelloIntervalMs + : DirectHintFullHelloIntervalMs; await SendDirectBootstrapProbeAsync( peerNodeId, candidate.LocalSocketId, candidate.RemoteEndPoint, key, - forceFullHello: false, - DirectHintFullHelloIntervalMs, + forceFullHello, + minIntervalMs, cancellationToken) .ConfigureAwait(false); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs index 13f1a56..cdc01c4 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -9,12 +9,14 @@ internal sealed class ZeroTierDirectHintPathPlanner private readonly IZeroTierUdpTransport _udp; private readonly ZeroTierExternalSurfaceAddressTracker _surfaceAddresses; private readonly Func _getDirectEndpointManager; + private readonly ZeroTierDirectPathSocketAdmissibility? _socketAdmissibility; private readonly ConcurrentDictionary _probeCursors = new(); public ZeroTierDirectHintPathPlanner( IZeroTierUdpTransport udp, ZeroTierExternalSurfaceAddressTracker surfaceAddresses, - Func getDirectEndpointManager) + Func getDirectEndpointManager, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility = null) { ArgumentNullException.ThrowIfNull(udp); ArgumentNullException.ThrowIfNull(surfaceAddresses); @@ -23,6 +25,7 @@ public ZeroTierDirectHintPathPlanner( _udp = udp; _surfaceAddresses = surfaceAddresses; _getDirectEndpointManager = getDirectEndpointManager; + _socketAdmissibility = socketAdmissibility; } public IPEndPoint[] TakeNextHintedEndpoints(NodeId peerNodeId, IPEndPoint[] hinted, int budget) @@ -156,7 +159,7 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); for (var i = 0; i < preferredFromHints.Length; i++) { - if (SocketCanReachEndpoint(localSockets, preferredFromHints[i], endpoint) && + if (SocketCanReachEndpoint(localSockets, preferredFromHints[i], endpoint, _socketAdmissibility) && !preferred.Contains(preferredFromHints[i])) { preferred.Add(preferredFromHints[i]); @@ -169,7 +172,7 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF { var socketId = localSockets[i].Id; if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && - SocketCanReachEndpoint(localSockets, socketId, endpoint) && + SocketCanReachEndpoint(localSockets, socketId, endpoint, _socketAdmissibility) && !preferred.Contains(socketId)) { preferred.Add(socketId); @@ -183,7 +186,7 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF } return localSockets - .Where(socket => SocketCanReachEndpoint(localSockets, socket.Id, endpoint)) + .Where(socket => SocketCanReachEndpoint(localSockets, socket.Id, endpoint, _socketAdmissibility)) .Select(static socket => socket.Id) .ToArray(); } @@ -194,7 +197,8 @@ private DirectBootstrapProbeCursor GetOrCreateCursor(NodeId peerNodeId) private static bool SocketCanReachEndpoint( IReadOnlyList localSockets, int socketId, - IPEndPoint endpoint) + IPEndPoint endpoint, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility) { for (var i = 0; i < localSockets.Count; i++) { @@ -203,6 +207,11 @@ private static bool SocketCanReachEndpoint( continue; } + if (socketAdmissibility is not null) + { + return socketAdmissibility.ShouldUsePath(localSockets[i], endpoint); + } + return AddressFamiliesAreCompatible(localSockets[i].LocalEndpoint.Address, endpoint.Address); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs new file mode 100644 index 0000000..8eda624 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs @@ -0,0 +1,66 @@ +using System.Net; +using System.Net.Sockets; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectPathSocketAdmissibility +{ + private readonly ZeroTierDirectEndpointPolicy _endpointPolicy; + private readonly ZeroTierDirectRouteResolver _routeResolver; + + public ZeroTierDirectPathSocketAdmissibility( + ZeroTierDirectEndpointPolicy endpointPolicy, + ZeroTierDirectRouteResolver routeResolver) + { + _endpointPolicy = endpointPolicy ?? throw new ArgumentNullException(nameof(endpointPolicy)); + _routeResolver = routeResolver ?? throw new ArgumentNullException(nameof(routeResolver)); + } + + public bool ShouldUsePath(ZeroTierUdpLocalSocket localSocket, IPEndPoint remoteEndpoint) + { + ArgumentNullException.ThrowIfNull(remoteEndpoint); + + remoteEndpoint = Canonicalize(remoteEndpoint); + if (!_endpointPolicy.ShouldAccept(remoteEndpoint)) + { + return false; + } + + var localAddress = Canonicalize(localSocket.LocalEndpoint.Address); + if (!AddressFamiliesAreCompatible(localAddress, remoteEndpoint.Address)) + { + return false; + } + + if (IsWildcard(localAddress) || !_routeResolver.TryResolve(remoteEndpoint, out var routedLocalAddress)) + { + return true; + } + + return localAddress.Equals(Canonicalize(routedLocalAddress)); + } + + private static bool IsWildcard(IPAddress address) + => address.Equals(IPAddress.Any) || address.Equals(IPAddress.IPv6Any); + + private static bool AddressFamiliesAreCompatible(IPAddress localAddress, IPAddress remoteAddress) + { + if (localAddress.AddressFamily == remoteAddress.AddressFamily) + { + return true; + } + + return localAddress.AddressFamily == AddressFamily.InterNetworkV6 && + localAddress.Equals(IPAddress.IPv6Any) && + remoteAddress.AddressFamily == AddressFamily.InterNetwork; + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => new(Canonicalize(endpoint.Address), endpoint.Port); + + private static IPAddress Canonicalize(IPAddress address) + => address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectRouteResolver.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectRouteResolver.cs new file mode 100644 index 0000000..c8e0cd9 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectRouteResolver.cs @@ -0,0 +1,71 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectRouteResolver +{ + private static readonly TimeSpan CacheTtl = TimeSpan.FromSeconds(30); + + private readonly Func _resolveLocalAddress; + private readonly Func _nowMs; + private readonly ConcurrentDictionary _cache = new(StringComparer.Ordinal); + + public ZeroTierDirectRouteResolver( + Func? resolveLocalAddress = null, + Func? nowMs = null) + { + _resolveLocalAddress = resolveLocalAddress ?? ResolveLocalAddressCore; + _nowMs = nowMs ?? (() => Environment.TickCount64); + } + + public bool TryResolve(IPEndPoint remoteEndpoint, out IPAddress localAddress) + { + ArgumentNullException.ThrowIfNull(remoteEndpoint); + + remoteEndpoint = Canonicalize(remoteEndpoint); + var now = _nowMs(); + var key = remoteEndpoint.ToString(); + if (_cache.TryGetValue(key, out var cached) && unchecked(now - cached.ExpiresAtMs) < 0) + { + localAddress = cached.LocalAddress; + return true; + } + + var resolved = _resolveLocalAddress(remoteEndpoint); + if (resolved is null) + { + localAddress = IPAddress.None; + return false; + } + + localAddress = Canonicalize(resolved); + _cache[key] = new CacheEntry(localAddress, now + (long)CacheTtl.TotalMilliseconds); + return true; + } + + private static IPAddress? ResolveLocalAddressCore(IPEndPoint remoteEndpoint) + { + try + { + using var socket = new Socket(remoteEndpoint.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + socket.Connect(remoteEndpoint); + return ((IPEndPoint?)socket.LocalEndPoint)?.Address; + } + catch (Exception ex) when (ex is SocketException or NotSupportedException or InvalidOperationException) + { + return null; + } + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => new(Canonicalize(endpoint.Address), endpoint.Port); + + private static IPAddress Canonicalize(IPAddress address) + => address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; + + private readonly record struct CacheEntry(IPAddress LocalAddress, long ExpiresAtMs); +} From a3847ca4d5cc154aaf36d17a1d9e043b5d32ac23 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 04:54:38 +0100 Subject: [PATCH 196/296] Fan out direct-only HELLO probes --- .../Internal/ZeroTierDataplaneRuntime.cs | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d656c8b..653ae4a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1580,16 +1580,22 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati var minIntervalMs = forceFullHello ? DirectOnlyHintHelloIntervalMs : DirectHintFullHelloIntervalMs; + var localSocketIds = forceFullHello + ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, candidate.RemoteEndPoint) + : [candidate.LocalSocketId]; - await SendDirectBootstrapProbeAsync( - peerNodeId, - candidate.LocalSocketId, - candidate.RemoteEndPoint, - key, - forceFullHello, - minIntervalMs, - cancellationToken) - .ConfigureAwait(false); + for (var s = 0; s < localSocketIds.Length; s++) + { + await SendDirectBootstrapProbeAsync( + peerNodeId, + localSocketIds[s], + candidate.RemoteEndPoint, + key, + forceFullHello, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + } } for (var p = 0; p < paths.Length; p++) From 29965f4155b2476fe814f4278fd9214aa6c34676 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:21:15 +0100 Subject: [PATCH 197/296] Reuse direct HELLO packets across sockets --- .../Internal/ZeroTierDataplaneRuntime.cs | 184 ++++++++++++++++-- 1 file changed, 166 insertions(+), 18 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 653ae4a..44f3548 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -61,7 +61,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); private readonly ConcurrentDictionary _lastDirectPathPushSentMs = new(); private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); - private readonly ConcurrentDictionary _pendingHelloProbes = new(); + private readonly ConcurrentDictionary _pendingHelloProbes = new(); private long _lastPendingHelloCleanupMs; private readonly Channel _peerQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) @@ -1206,7 +1206,14 @@ private async Task SendHelloPacketAsync( ZeroTierHelloClient.AdvertisedRevision, out var packetId); - TrackPendingHello(peerNodeId, localSocketId, sendTo, physicalDestination, packetId); + TrackPendingHello( + packetId, + new PendingHelloProbe( + PeerNodeId: peerNodeId, + LocalSocketId: localSocketId, + SendTo: sendTo, + PhysicalDestination: physicalDestination, + SentAtMs: Environment.TickCount64)); await _udp.SendAsync(localSocketId, sendTo, packet, cancellationToken).ConfigureAwait(false); } @@ -1235,6 +1242,81 @@ private Task SendHelloPacketAsync( DirectHelloMinIntervalMs, cancellationToken); + private async Task SendHelloPacketAcrossSocketsAsync( + int[] localSocketIds, + NodeId peerNodeId, + IPEndPoint? physicalDestination, + IPEndPoint sendTo, + byte[] sharedKey, + long minIntervalMs, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(localSocketIds); + + var eligibleSocketIds = new List(localSocketIds.Length); + for (var i = 0; i < localSocketIds.Length; i++) + { + if (physicalDestination is not null && + !ShouldSendDirectHello(peerNodeId, localSocketIds[i], sendTo, minIntervalMs)) + { + continue; + } + + eligibleSocketIds.Add(localSocketIds[i]); + } + + if (eligibleSocketIds.Count == 0) + { + return; + } + + try + { + var sentAtMs = Environment.TickCount64; + var packet = ZeroTierHelloPacketBuilder.BuildPacket( + _localIdentity, + peerNodeId, + physicalDestination, + timestamp: (ulong)sentAtMs, + _planetId, + _planetTimestamp, + sharedKey, + ZeroTierHelloClient.AdvertisedProtocolVersion, + ZeroTierHelloClient.AdvertisedMajorVersion, + ZeroTierHelloClient.AdvertisedMinorVersion, + ZeroTierHelloClient.AdvertisedRevision, + out var packetId); + + TrackPendingHello( + packetId, + eligibleSocketIds + .Select(socketId => new PendingHelloProbe( + PeerNodeId: peerNodeId, + LocalSocketId: socketId, + SendTo: sendTo, + PhysicalDestination: physicalDestination, + SentAtMs: sentAtMs)) + .ToArray()); + + for (var i = 0; i < eligibleSocketIds.Count; i++) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={eligibleSocketIds[i]}, advertised={physicalDestination})."); + } + + await _udp.SendAsync(eligibleSocketIds[i], sendTo, packet, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] HELLO bootstrap send failed to {sendTo}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + private bool ShouldSendDirectHello(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, long minIntervalMs) { var endpoint = remoteEndPoint.Address.IsIPv4MappedToIPv6 @@ -1368,7 +1450,7 @@ private void HandlePeerHelloOk( ArgumentNullException.ThrowIfNull(receivedVia); _peerRootSocketAffinity.Observe(peerNodeId, receivedLocalSocketId, receivedVia); - var matchedPending = TryTakePendingHello(peerNodeId, ok.InRePacketId, out var pending); + var matchedPending = TryTakePendingHello(peerNodeId, ok.InRePacketId, receivedLocalSocketId, receivedVia, out var pending); var observedLocalSocketId = matchedPending ? pending.LocalSocketId : receivedLocalSocketId; var trustedSurface = matchedPending && hopCount == 0 && @@ -1448,26 +1530,28 @@ private void HandleAuthenticatedPeer(NodeId peerNodeId) _authenticatedPeers[peerNodeId] = 0; } - private void TrackPendingHello( - NodeId peerNodeId, - int localSocketId, - IPEndPoint sendTo, - IPEndPoint? physicalDestination, - ulong packetId) + private void TrackPendingHello(ulong packetId, PendingHelloProbe probe) { CleanupPendingHellosIfNeeded(Environment.TickCount64); - _pendingHelloProbes[packetId] = new PendingHelloProbe( - PeerNodeId: peerNodeId, - LocalSocketId: localSocketId, - SendTo: sendTo, - PhysicalDestination: physicalDestination, - SentAtMs: Environment.TickCount64); + _pendingHelloProbes[packetId] = new PendingHelloProbeSet([probe]); } - private bool TryTakePendingHello(NodeId peerNodeId, ulong packetId, out PendingHelloProbe pending) + private void TrackPendingHello(ulong packetId, PendingHelloProbe[] probes) + { + CleanupPendingHellosIfNeeded(Environment.TickCount64); + _pendingHelloProbes[packetId] = new PendingHelloProbeSet(probes); + } + + private bool TryTakePendingHello( + NodeId peerNodeId, + ulong packetId, + int receivedLocalSocketId, + IPEndPoint receivedVia, + out PendingHelloProbe pending) { CleanupPendingHellosIfNeeded(Environment.TickCount64); - if (_pendingHelloProbes.TryRemove(packetId, out pending) && pending.PeerNodeId == peerNodeId) + if (_pendingHelloProbes.TryRemove(packetId, out var exact) && + exact.TryTakeBest(peerNodeId, receivedLocalSocketId, receivedVia, out pending)) { return true; } @@ -1479,7 +1563,8 @@ private bool TryTakePendingHello(NodeId peerNodeId, ulong packetId, out PendingH continue; } - if (_pendingHelloProbes.TryRemove(candidate.Key, out pending) && pending.PeerNodeId == peerNodeId) + if (_pendingHelloProbes.TryRemove(candidate.Key, out var correlated) && + correlated.TryTakeBest(peerNodeId, receivedLocalSocketId, receivedVia, out pending)) { return true; } @@ -1584,6 +1669,20 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, candidate.RemoteEndPoint) : [candidate.LocalSocketId]; + if (forceFullHello && localSocketIds.Length > 1) + { + await SendHelloPacketAcrossSocketsAsync( + localSocketIds, + peerNodeId, + candidate.RemoteEndPoint, + candidate.RemoteEndPoint, + key, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + continue; + } + for (var s = 0; s < localSocketIds.Length; s++) { await SendDirectBootstrapProbeAsync( @@ -1969,6 +2068,55 @@ internal readonly record struct PendingHelloProbe( IPEndPoint? PhysicalDestination, long SentAtMs); +internal readonly record struct PendingHelloProbeSet(PendingHelloProbe[] Probes) +{ + public long SentAtMs => Probes.Length == 0 ? 0 : Probes.Min(static probe => probe.SentAtMs); + + public bool TryTakeBest(NodeId peerNodeId, int receivedLocalSocketId, IPEndPoint receivedVia, out PendingHelloProbe probe) + { + for (var i = 0; i < Probes.Length; i++) + { + if (Probes[i].PeerNodeId == peerNodeId && Probes[i].LocalSocketId == receivedLocalSocketId) + { + probe = Probes[i]; + return true; + } + } + + for (var i = 0; i < Probes.Length; i++) + { + if (Probes[i].PeerNodeId != peerNodeId) + { + continue; + } + + if (Probes[i].PhysicalDestination is { } physicalDestination && physicalDestination.Equals(receivedVia)) + { + probe = Probes[i]; + return true; + } + + if (Probes[i].SendTo.Equals(receivedVia)) + { + probe = Probes[i]; + return true; + } + } + + for (var i = 0; i < Probes.Length; i++) + { + if (Probes[i].PeerNodeId == peerNodeId) + { + probe = Probes[i]; + return true; + } + } + + probe = default; + return false; + } +} + From d78e4666d8c9b0f93b54ef23e58ec33c84d8962a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:23:44 +0100 Subject: [PATCH 198/296] Force direct-only hinted HELLO bootstrap --- .../Internal/ZeroTierDataplaneRuntime.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 44f3548..9e6f253 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1092,6 +1092,8 @@ private async ValueTask HandleDirectEndpointHintAsync( return; } + forceFullHello |= ShouldForceFullHelloForHintedCandidate(peerNodeId); + if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine( @@ -1104,6 +1106,20 @@ private async ValueTask HandleDirectEndpointHintAsync( peerNodeId, endpoint, includeFallbackLocalSockets: true); + if (forceFullHello && useAllEligibleLocalSockets && localSocketIds.Length > 1) + { + await SendHelloPacketAcrossSocketsAsync( + localSocketIds, + peerNodeId, + endpoint, + endpoint, + sharedKey, + DirectHelloMinIntervalMs, + cancellationToken) + .ConfigureAwait(false); + return; + } + for (var i = 0; i < localSocketIds.Length; i++) { await SendDirectBootstrapProbeAsync( From 26c7711c40820bab415149f7882abf7af3e8f4ee Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:29:57 +0100 Subject: [PATCH 199/296] Probe hinted direct paths on all admissible sockets --- ...ZeroTierDirectHintPathPlannerRouteTests.cs | 60 +++++++++++++++ .../Internal/ZeroTierDirectHintPathPlanner.cs | 73 ++++++++++++++----- 2 files changed, 113 insertions(+), 20 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs index ea691d6..771038e 100644 --- a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs @@ -45,6 +45,66 @@ public void GetPreferredAndFallbackSocketIds_UsesRouteSelectedSocket_ForPublicEn Assert.Equal([0], sockets); } + [Fact] + public void GetPreferredAndFallbackSocketIds_IncludesAdmissibleSockets_WithoutSurfaceObservations() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001)), + new ZeroTierUdpLocalSocket(2, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10002))); + var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); + surfaces.Observe( + new NodeId(1), + 0, + new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); + + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + surfaces, + _ => manager, + new ZeroTierDirectPathSocketAdmissibility( + new ZeroTierDirectEndpointPolicy( + [new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24)]), + new ZeroTierDirectRouteResolver(_ => IPAddress.Parse("10.0.0.112")))); + + var sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + + Assert.Equal([0, 1, 2], sockets); + } + + [Fact] + public void GetRotatingSocketIds_RotatesAcrossAdmissibleSockets_WithoutSurfaceObservations() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001)), + new ZeroTierUdpLocalSocket(2, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10002))); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)), + _ => new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId), + new ZeroTierDirectPathSocketAdmissibility( + new ZeroTierDirectEndpointPolicy( + [new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24)]), + new ZeroTierDirectRouteResolver(_ => IPAddress.Parse("10.0.0.112")))); + + var seen = new HashSet(); + for (var i = 0; i < 6; i++) + { + seen.Add(planner.GetRotatingSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true).Single()); + } + + Assert.Equal([0, 1, 2], seen.OrderBy(static id => id).ToArray()); + } + private sealed class StubUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) : IZeroTierUdpTransport { public IReadOnlyList LocalSockets { get; } = localSockets; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs index cdc01c4..041dca7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -157,27 +157,12 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF var preferred = new List(localSockets.Count); var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); - for (var i = 0; i < preferredFromHints.Length; i++) - { - if (SocketCanReachEndpoint(localSockets, preferredFromHints[i], endpoint, _socketAdmissibility) && - !preferred.Contains(preferredFromHints[i])) - { - preferred.Add(preferredFromHints[i]); - } - } + AddSocketIds(preferred, preferredFromHints, localSockets, endpoint, _socketAdmissibility); if (includeFallbackLocalSockets || preferred.Count == 0) { - for (var i = 0; i < localSockets.Count; i++) - { - var socketId = localSockets[i].Id; - if (_surfaceAddresses.GetSnapshot(socketId).Length != 0 && - SocketCanReachEndpoint(localSockets, socketId, endpoint, _socketAdmissibility) && - !preferred.Contains(socketId)) - { - preferred.Add(socketId); - } - } + AddSocketIds(preferred, GetSocketIdsWithSurfaceObservations(localSockets, endpoint), localSockets, endpoint, _socketAdmissibility); + AddSocketIds(preferred, GetAdmissibleSocketIds(localSockets, endpoint, _socketAdmissibility), localSockets, endpoint, _socketAdmissibility); } if (preferred.Count != 0) @@ -185,10 +170,58 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF return preferred.ToArray(); } - return localSockets - .Where(socket => SocketCanReachEndpoint(localSockets, socket.Id, endpoint, _socketAdmissibility)) + return GetAdmissibleSocketIds(localSockets, endpoint, _socketAdmissibility); + } + + private int[] GetSocketIdsWithSurfaceObservations(IReadOnlyList localSockets, IPEndPoint endpoint) + { + var sockets = new List(localSockets.Count); + for (var i = 0; i < localSockets.Count; i++) + { + var socketId = localSockets[i].Id; + if (_surfaceAddresses.GetSnapshot(socketId).Length == 0) + { + continue; + } + + if (!SocketCanReachEndpoint(localSockets, socketId, endpoint, _socketAdmissibility)) + { + continue; + } + + sockets.Add(socketId); + } + + return sockets.ToArray(); + } + + private static int[] GetAdmissibleSocketIds( + IReadOnlyList localSockets, + IPEndPoint endpoint, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility) + => localSockets + .Where(socket => SocketCanReachEndpoint(localSockets, socket.Id, endpoint, socketAdmissibility)) .Select(static socket => socket.Id) .ToArray(); + + private static void AddSocketIds( + List target, + int[] candidates, + IReadOnlyList localSockets, + IPEndPoint endpoint, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility) + { + for (var i = 0; i < candidates.Length; i++) + { + var socketId = candidates[i]; + if (!SocketCanReachEndpoint(localSockets, socketId, endpoint, socketAdmissibility) || + target.Contains(socketId)) + { + continue; + } + + target.Add(socketId); + } } private DirectBootstrapProbeCursor GetOrCreateCursor(NodeId peerNodeId) From 584f08341bb68f065f702cc4bb85cf315638eefd Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:32:11 +0100 Subject: [PATCH 200/296] Probe hinted direct paths on all admissible sockets --- ...ZeroTierDirectHintPathPlannerRouteTests.cs | 22 --------------- .../ZeroTierDirectHintPathPlannerTests.cs | 24 ++--------------- .../Internal/ZeroTierDataplaneRuntime.cs | 1 - .../Internal/ZeroTierDirectHintPathPlanner.cs | 27 ------------------- 4 files changed, 2 insertions(+), 72 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs index 771038e..0ecd96a 100644 --- a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs @@ -15,22 +15,9 @@ public void GetPreferredAndFallbackSocketIds_UsesRouteSelectedSocket_ForPublicEn var udp = new StubUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("192.168.224.1"), 10001))); - var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); - surfaces.Observe( - new NodeId(1), - 0, - new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); - surfaces.Observe( - new NodeId(1), - 1, - new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)); - var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); var planner = new ZeroTierDirectHintPathPlanner( udp, - surfaces, _ => manager, new ZeroTierDirectPathSocketAdmissibility( new ZeroTierDirectEndpointPolicy( @@ -55,17 +42,9 @@ public void GetPreferredAndFallbackSocketIds_IncludesAdmissibleSockets_WithoutSu new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001)), new ZeroTierUdpLocalSocket(2, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10002))); - var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); - surfaces.Observe( - new NodeId(1), - 0, - new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); - var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); var planner = new ZeroTierDirectHintPathPlanner( udp, - surfaces, _ => manager, new ZeroTierDirectPathSocketAdmissibility( new ZeroTierDirectEndpointPolicy( @@ -89,7 +68,6 @@ public void GetRotatingSocketIds_RotatesAcrossAdmissibleSockets_WithoutSurfaceOb new ZeroTierUdpLocalSocket(2, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10002))); var planner = new ZeroTierDirectHintPathPlanner( udp, - new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)), _ => new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId), new ZeroTierDirectPathSocketAdmissibility( new ZeroTierDirectEndpointPolicy( diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs index 500b1b7..a36dc23 100644 --- a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs @@ -22,7 +22,6 @@ public void GetNextHintedCandidatesForMaintenance_RotatesSockets_AndSkipsObserve var planner = new ZeroTierDirectHintPathPlanner( udp, - new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)), _ => manager); var first = planner.GetNextHintedCandidatesForMaintenance( @@ -49,20 +48,8 @@ public void GetPreferredAndFallbackSocketIds_FiltersByAddressFamily() var udp = new StubUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("fd00::1"), 10001))); - var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); - surfaces.Observe( - new NodeId(1), - 0, - new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); - surfaces.Observe( - new NodeId(1), - 1, - new IPEndPoint(IPAddress.Parse("2001:db8::2"), 9993), - new IPEndPoint(IPAddress.Parse("2001:db8::1"), 60001)); - var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); - var planner = new ZeroTierDirectHintPathPlanner(udp, surfaces, _ => manager); + var planner = new ZeroTierDirectHintPathPlanner(udp, _ => manager); var ipv4Sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv4Endpoint); var ipv6Sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv6Endpoint); @@ -79,15 +66,8 @@ public void GetPreferredAndFallbackSocketIds_ReturnsEmpty_WhenNoSocketCanReachEn var udp = new StubUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); - var surfaces = new ZeroTierExternalSurfaceAddressTracker(TimeSpan.FromMinutes(1)); - surfaces.Observe( - new NodeId(1), - 0, - new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993), - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)); - var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); - var planner = new ZeroTierDirectHintPathPlanner(udp, surfaces, _ => manager); + var planner = new ZeroTierDirectHintPathPlanner(udp, _ => manager); var sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv6Endpoint); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 9e6f253..0edb6db 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -208,7 +208,6 @@ public ZeroTierDataplaneRuntime( _localDirectAdvertisementPlanner = new ZeroTierLocalDirectPathAdvertisementPlanner(_directEndpointPolicy); _directHintPlanner = new ZeroTierDirectHintPathPlanner( udp, - _surfaceAddresses, GetOrCreateDirectEndpointManager, new ZeroTierDirectPathSocketAdmissibility( _directEndpointPolicy, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs index 041dca7..7350f90 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -7,23 +7,19 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierDirectHintPathPlanner { private readonly IZeroTierUdpTransport _udp; - private readonly ZeroTierExternalSurfaceAddressTracker _surfaceAddresses; private readonly Func _getDirectEndpointManager; private readonly ZeroTierDirectPathSocketAdmissibility? _socketAdmissibility; private readonly ConcurrentDictionary _probeCursors = new(); public ZeroTierDirectHintPathPlanner( IZeroTierUdpTransport udp, - ZeroTierExternalSurfaceAddressTracker surfaceAddresses, Func getDirectEndpointManager, ZeroTierDirectPathSocketAdmissibility? socketAdmissibility = null) { ArgumentNullException.ThrowIfNull(udp); - ArgumentNullException.ThrowIfNull(surfaceAddresses); ArgumentNullException.ThrowIfNull(getDirectEndpointManager); _udp = udp; - _surfaceAddresses = surfaceAddresses; _getDirectEndpointManager = getDirectEndpointManager; _socketAdmissibility = socketAdmissibility; } @@ -161,7 +157,6 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF if (includeFallbackLocalSockets || preferred.Count == 0) { - AddSocketIds(preferred, GetSocketIdsWithSurfaceObservations(localSockets, endpoint), localSockets, endpoint, _socketAdmissibility); AddSocketIds(preferred, GetAdmissibleSocketIds(localSockets, endpoint, _socketAdmissibility), localSockets, endpoint, _socketAdmissibility); } @@ -173,28 +168,6 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF return GetAdmissibleSocketIds(localSockets, endpoint, _socketAdmissibility); } - private int[] GetSocketIdsWithSurfaceObservations(IReadOnlyList localSockets, IPEndPoint endpoint) - { - var sockets = new List(localSockets.Count); - for (var i = 0; i < localSockets.Count; i++) - { - var socketId = localSockets[i].Id; - if (_surfaceAddresses.GetSnapshot(socketId).Length == 0) - { - continue; - } - - if (!SocketCanReachEndpoint(localSockets, socketId, endpoint, _socketAdmissibility)) - { - continue; - } - - sockets.Add(socketId); - } - - return sockets.ToArray(); - } - private static int[] GetAdmissibleSocketIds( IReadOnlyList localSockets, IPEndPoint endpoint, From 5a5a214535ec0356e9fc140b2d74c23391c75882 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:34:14 +0100 Subject: [PATCH 201/296] Prefer root family for multipath UDP binds --- ...ketRuntimeBootstrapperUdpTransportTests.cs | 28 +++++++++ .../ZeroTierSocketRuntimeBootstrapper.cs | 62 ++++++++++++++++++- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index 4256d86..3c5b274 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -111,6 +111,34 @@ public void CreateUdpSocketBindings_MultipathEnabled_UsesDiscoveredBindAddresses binding => Assert.Equal(IPAddress.Parse("2001:db8::10"), binding.LocalAddress)); } + [Fact] + public void CreateUdpSocketBindings_WithPreferredAddressFamily_ReusesMatchingAddressesOnly() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + preferredBindAddressFamily: AddressFamily.InterNetwork, + getLocalBindAddresses: () => + [ + IPAddress.Parse("192.0.2.10"), + IPAddress.Parse("2001:db8::10") + ]); + + Assert.All(bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress)); + } + + [Fact] + public void CreateUdpSocketBindings_WithUnavailablePreferredAddressFamily_KeepsOriginalRotation() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + preferredBindAddressFamily: AddressFamily.InterNetworkV6, + getLocalBindAddresses: () => [IPAddress.Parse("192.0.2.10")]); + + Assert.All(bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress)); + } + [Fact] public void CreateUdpSocketBindings_NoDiscoveredAddresses_FallsBackToWildcard() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index fc47e0f..181dbb2 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -11,7 +11,10 @@ internal static class ZeroTierSocketRuntimeBootstrapper { internal readonly record struct ZeroTierUdpSocketBinding(IPAddress? LocalAddress, int LocalPort, int LocalSocketId); - internal static async ValueTask CreateUdpTransportAsync(ZeroTierMultipathOptions multipath, bool enableIpv6) + internal static async ValueTask CreateUdpTransportAsync( + ZeroTierMultipathOptions multipath, + bool enableIpv6, + AddressFamily? preferredBindAddressFamily = null) { ArgumentNullException.ThrowIfNull(multipath); @@ -20,7 +23,7 @@ internal static async ValueTask CreateUdpTransportAsync(Z return new ZeroTierUdpTransport(localPort: 0, enableIpv6: enableIpv6, localSocketId: 0); } - var bindings = CreateUdpSocketBindings(multipath, enableIpv6); + var bindings = CreateUdpSocketBindings(multipath, enableIpv6, preferredBindAddressFamily); if (bindings.Length == 1) { var binding = bindings[0]; @@ -70,6 +73,7 @@ internal static async ValueTask CreateUdpTransportAsync(Z internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( ZeroTierMultipathOptions multipath, bool enableIpv6, + AddressFamily? preferredBindAddressFamily = null, Func? getLocalBindAddresses = null) { ArgumentNullException.ThrowIfNull(multipath); @@ -113,6 +117,7 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( var bindAddresses = multipath.UdpSocketCount > 1 ? (getLocalBindAddresses ?? (() => ZeroTierUdpLocalBindAddressSource.GetSnapshot(enableIpv6))).Invoke() : Array.Empty(); + bindAddresses = OrderBindAddresses(bindAddresses, preferredBindAddressFamily); var bindings = new ZeroTierUdpSocketBinding[multipath.UdpSocketCount]; for (var i = 0; i < bindings.Length; i++) @@ -126,6 +131,24 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( return bindings; } + private static IPAddress[] OrderBindAddresses(IPAddress[] bindAddresses, AddressFamily? preferredBindAddressFamily) + { + if (bindAddresses.Length == 0 || preferredBindAddressFamily is null) + { + return bindAddresses; + } + + var preferred = bindAddresses + .Where(address => address.AddressFamily == preferredBindAddressFamily) + .ToArray(); + if (preferred.Length == 0) + { + return bindAddresses; + } + + return preferred; + } + [SuppressMessage( "Reliability", "CA2000:Dispose objects before losing scope", @@ -146,7 +169,11 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( ArgumentNullException.ThrowIfNull(managedIps); ArgumentNullException.ThrowIfNull(inlineCom); - var udp = await CreateUdpTransportAsync(multipath, enableIpv6: true).ConfigureAwait(false); + var udp = await CreateUdpTransportAsync( + multipath, + enableIpv6: true, + preferredBindAddressFamily: SelectPreferredBindAddressFamily(planet)) + .ConfigureAwait(false); try { var localManagedIpsV6 = managedIps @@ -178,4 +205,33 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( throw; } } + + private static AddressFamily? SelectPreferredBindAddressFamily(ZeroTierWorld planet) + { + for (var i = 0; i < planet.Roots.Count; i++) + { + var root = planet.Roots[i]; + for (var e = 0; e < root.StableEndpoints.Count; e++) + { + if (root.StableEndpoints[e].AddressFamily == AddressFamily.InterNetwork) + { + return AddressFamily.InterNetwork; + } + } + } + + for (var i = 0; i < planet.Roots.Count; i++) + { + var root = planet.Roots[i]; + for (var e = 0; e < root.StableEndpoints.Count; e++) + { + if (root.StableEndpoints[e].AddressFamily == AddressFamily.InterNetworkV6) + { + return AddressFamily.InterNetworkV6; + } + } + } + + return null; + } } From 2ce7c4b1a3be5dfeb441105106ee285d623261ce Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:37:47 +0100 Subject: [PATCH 202/296] Prefer route-selected bind address for multipath --- ...ketRuntimeBootstrapperUdpTransportTests.cs | 18 +++++++ .../ZeroTierSocketRuntimeBootstrapper.cs | 53 +++++++++++++++---- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index 3c5b274..232e5cc 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -127,6 +127,24 @@ public void CreateUdpSocketBindings_WithPreferredAddressFamily_ReusesMatchingAdd Assert.All(bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress)); } + [Fact] + public void CreateUdpSocketBindings_WithPreferredLocalBindAddress_ReusesRoutedAddressOnly() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + preferredBindAddressFamily: AddressFamily.InterNetwork, + preferredLocalBindAddress: IPAddress.Parse("192.0.2.11"), + getLocalBindAddresses: () => + [ + IPAddress.Parse("192.0.2.10"), + IPAddress.Parse("192.0.2.11"), + IPAddress.Parse("2001:db8::10") + ]); + + Assert.All(bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.11"), binding.LocalAddress)); + } + [Fact] public void CreateUdpSocketBindings_WithUnavailablePreferredAddressFamily_KeepsOriginalRotation() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 181dbb2..ab0e669 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -14,7 +14,8 @@ internal static class ZeroTierSocketRuntimeBootstrapper internal static async ValueTask CreateUdpTransportAsync( ZeroTierMultipathOptions multipath, bool enableIpv6, - AddressFamily? preferredBindAddressFamily = null) + AddressFamily? preferredBindAddressFamily = null, + IPAddress? preferredLocalBindAddress = null) { ArgumentNullException.ThrowIfNull(multipath); @@ -23,7 +24,11 @@ internal static async ValueTask CreateUdpTransportAsync( return new ZeroTierUdpTransport(localPort: 0, enableIpv6: enableIpv6, localSocketId: 0); } - var bindings = CreateUdpSocketBindings(multipath, enableIpv6, preferredBindAddressFamily); + var bindings = CreateUdpSocketBindings( + multipath, + enableIpv6, + preferredBindAddressFamily, + preferredLocalBindAddress); if (bindings.Length == 1) { var binding = bindings[0]; @@ -74,6 +79,7 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( ZeroTierMultipathOptions multipath, bool enableIpv6, AddressFamily? preferredBindAddressFamily = null, + IPAddress? preferredLocalBindAddress = null, Func? getLocalBindAddresses = null) { ArgumentNullException.ThrowIfNull(multipath); @@ -117,7 +123,7 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( var bindAddresses = multipath.UdpSocketCount > 1 ? (getLocalBindAddresses ?? (() => ZeroTierUdpLocalBindAddressSource.GetSnapshot(enableIpv6))).Invoke() : Array.Empty(); - bindAddresses = OrderBindAddresses(bindAddresses, preferredBindAddressFamily); + bindAddresses = OrderBindAddresses(bindAddresses, preferredBindAddressFamily, preferredLocalBindAddress); var bindings = new ZeroTierUdpSocketBinding[multipath.UdpSocketCount]; for (var i = 0; i < bindings.Length; i++) @@ -131,9 +137,28 @@ internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( return bindings; } - private static IPAddress[] OrderBindAddresses(IPAddress[] bindAddresses, AddressFamily? preferredBindAddressFamily) + private static IPAddress[] OrderBindAddresses( + IPAddress[] bindAddresses, + AddressFamily? preferredBindAddressFamily, + IPAddress? preferredLocalBindAddress) { - if (bindAddresses.Length == 0 || preferredBindAddressFamily is null) + if (bindAddresses.Length == 0) + { + return bindAddresses; + } + + if (preferredLocalBindAddress is not null) + { + var routed = bindAddresses + .Where(address => address.Equals(preferredLocalBindAddress)) + .ToArray(); + if (routed.Length != 0) + { + return routed; + } + } + + if (preferredBindAddressFamily is null) { return bindAddresses; } @@ -169,10 +194,20 @@ private static IPAddress[] OrderBindAddresses(IPAddress[] bindAddresses, Address ArgumentNullException.ThrowIfNull(managedIps); ArgumentNullException.ThrowIfNull(inlineCom); + var preferredRootEndpoint = SelectPreferredBindEndpoint(planet); + var preferredBindAddressFamily = preferredRootEndpoint?.AddressFamily; + var routeResolver = new ZeroTierDirectRouteResolver(); + var preferredLocalBindAddress = + preferredRootEndpoint is not null && + routeResolver.TryResolve(preferredRootEndpoint, out var routedLocalAddress) + ? routedLocalAddress + : null; + var udp = await CreateUdpTransportAsync( multipath, enableIpv6: true, - preferredBindAddressFamily: SelectPreferredBindAddressFamily(planet)) + preferredBindAddressFamily: preferredBindAddressFamily, + preferredLocalBindAddress: preferredLocalBindAddress) .ConfigureAwait(false); try { @@ -206,7 +241,7 @@ private static IPAddress[] OrderBindAddresses(IPAddress[] bindAddresses, Address } } - private static AddressFamily? SelectPreferredBindAddressFamily(ZeroTierWorld planet) + private static IPEndPoint? SelectPreferredBindEndpoint(ZeroTierWorld planet) { for (var i = 0; i < planet.Roots.Count; i++) { @@ -215,7 +250,7 @@ private static IPAddress[] OrderBindAddresses(IPAddress[] bindAddresses, Address { if (root.StableEndpoints[e].AddressFamily == AddressFamily.InterNetwork) { - return AddressFamily.InterNetwork; + return root.StableEndpoints[e]; } } } @@ -227,7 +262,7 @@ private static IPAddress[] OrderBindAddresses(IPAddress[] bindAddresses, Address { if (root.StableEndpoints[e].AddressFamily == AddressFamily.InterNetworkV6) { - return AddressFamily.InterNetworkV6; + return root.StableEndpoints[e]; } } } From 268303d2ec8293e13c868dad6df6285b05703440 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:39:11 +0100 Subject: [PATCH 203/296] Fan out rendezvous bootstrap across eligible sockets --- ...irectEndpointManagerSocketAffinityTests.cs | 28 +++++++++++++++++++ .../Internal/ZeroTierDirectEndpointManager.cs | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index b9e0dd1..ac8bd67 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -30,6 +30,34 @@ await manager.HandleRendezvousFromRootAsync( Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); } + [Fact] + public async Task Rendezvous_RequestsAllEligibleSockets_ForDirectBootstrap() + { + await using var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, hintedEndpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((hintedEndpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, false, true, 1), hints[0]); + } + [Fact] public async Task PushDirectPaths_UsesReceivingLocalSocket_ForNewEndpoints() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index cfdddd9..6e6f053 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -124,7 +124,7 @@ await _handleDirectEndpointHintAsync( receivedLocalSocketId, endpoint, false, - false, + true, cancellationToken) .ConfigureAwait(false); } From 21bd4ac4760de04c5a4288ae82b16e7dd02e9044 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:48:44 +0100 Subject: [PATCH 204/296] Extend direct-only HTTP connect timeout --- ZTSharp.Tests/ZeroTierApiTests.cs | 34 ++++++++++++++++++++++++++++++ ZTSharp/ZeroTier/ZeroTierSocket.cs | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierApiTests.cs b/ZTSharp.Tests/ZeroTierApiTests.cs index 768e35d..ccea34b 100644 --- a/ZTSharp.Tests/ZeroTierApiTests.cs +++ b/ZTSharp.Tests/ZeroTierApiTests.cs @@ -73,4 +73,38 @@ await Assert.ThrowsAsync(async () => _ = await socket.ConnectTcpAsync(new IPEndPoint(IPAddress.IPv6Loopback, 80)).ConfigureAwait(false); }); } + + [Fact] + public async Task SuggestedHttpConnectTimeout_IsExtended_ForDirectOnlyMultipath() + { + await using var socket = await ZeroTierSocket.CreateAsync(new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-test-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = false + } + }); + + Assert.Equal(TimeSpan.FromSeconds(120), socket.SuggestedHttpConnectTimeout); + } + + [Fact] + public async Task SuggestedHttpConnectTimeout_RemainsShort_WhenRelayFallbackIsAllowed() + { + await using var socket = await ZeroTierSocket.CreateAsync(new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-test-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = true + } + }); + + Assert.Equal(TimeSpan.FromSeconds(10), socket.SuggestedHttpConnectTimeout); + } } diff --git a/ZTSharp/ZeroTier/ZeroTierSocket.cs b/ZTSharp/ZeroTier/ZeroTierSocket.cs index 49fe30a..ac90bad 100644 --- a/ZTSharp/ZeroTier/ZeroTierSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierSocket.cs @@ -46,7 +46,7 @@ internal ZeroTierSocket(ZeroTierSocketOptions options, string statePath, ZeroTie internal TimeSpan SuggestedHttpConnectTimeout => _options.Multipath.Enabled && !_options.Multipath.AllowRootRelayFallback - ? TimeSpan.FromSeconds(45) + ? TimeSpan.FromSeconds(120) : TimeSpan.FromSeconds(10); public static Task CreateAsync( From 788c224d22c8e5c8102248d288c362316cb71c9b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:53:08 +0100 Subject: [PATCH 205/296] Keep relayed bootstrap alive on all root sockets --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 45 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 10 ++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 87ba9b3..af5730b 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -211,6 +211,50 @@ public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() Assert.Equal(new[] { 1 }, socketIds); } + [Fact] + public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithoutConfirmedPath_FansOutAcrossRootSockets() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = false + }, + planetId: 1, + planetTimestamp: 1); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + await runtime.SendHelloViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var socketIds = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && decoded.Header.Verb == ZeroTierVerb.Hello) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + Assert.Equal(new[] { 0, 1 }, socketIds); + } + private static ZeroTierDataplaneRuntime CreateRuntime( IZeroTierUdpTransport udp, ZeroTierIdentity localIdentity, @@ -351,3 +395,4 @@ private static async Task WaitForConditionAsync(Func condition, TimeSpan t throw new TimeoutException("Timed out waiting for condition."); } } + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 0edb6db..eb07d1a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -857,7 +857,8 @@ private bool HasConfirmedDirectPath(NodeId peerNodeId) private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { - if (_peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) + if (ShouldUsePeerRootSocketAffinity(peerNodeId) && + _peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) { await SendHelloPacketAsync( rootSocketId, @@ -965,7 +966,8 @@ private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, Cancella private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet, CancellationToken cancellationToken) { - if (_peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) + if (ShouldUsePeerRootSocketAffinity(peerNodeId) && + _peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) { return _udp.SendAsync(localSocketId, _rootEndpoint, packet, cancellationToken); } @@ -973,6 +975,9 @@ private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet return SendViaRootSocketsAsync(packet, cancellationToken); } + private bool ShouldUsePeerRootSocketAffinity(NodeId peerNodeId) + => _multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId); + private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { if (_inlineCom.Length == 0) @@ -2135,3 +2140,4 @@ public bool TryTakeBest(NodeId peerNodeId, int receivedLocalSocketId, IPEndPoint + From de1e44610049c4fdf61440843b7bd0dd8bdc6957 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:56:57 +0100 Subject: [PATCH 206/296] Allow direct-only payloads on hinted paths --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 28 +++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 42 ++++++++++++------- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index af5730b..5bb9646 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -89,6 +89,34 @@ await WaitForConditionAsync( Assert.Equal(0, candidate.LocalSocketId); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_CanSelectHintedPath_WithoutConfirmedHop0() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var peerNodeId = new NodeId(0x3333333333); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoint); + + Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var selected)); + Assert.Equal(hintedEndpoint, selected.RemoteEndPoint); + Assert.Equal(0, selected.LocalSocketId); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index eb07d1a..1fee10b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -654,13 +654,13 @@ private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationTok { cancellationToken.ThrowIfCancellationRequested(); - if (!_multipath.Enabled || HasSufficientDirectPath(peerNodeId)) + if (!_multipath.Enabled || HasUsableDirectPath(peerNodeId)) { return; } var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - if (HasSufficientDirectPath(peerNodeId)) + if (HasUsableDirectPath(peerNodeId)) { return; } @@ -671,7 +671,10 @@ private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationTok try { - await bootstrapTask.WaitAsync(cancellationToken).ConfigureAwait(false); + if (!CanOptimisticallyUseHintedDirectPath(peerNodeId)) + { + await bootstrapTask.WaitAsync(cancellationToken).ConfigureAwait(false); + } } finally { @@ -681,9 +684,13 @@ private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationTok } } - if (!_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId)) + if (!HasUsableDirectPath(peerNodeId)) { - EnsureRelayAllowedForPayload(peerNodeId, reason: "direct bootstrap did not confirm a peer path"); + EnsureRelayAllowedForPayload( + peerNodeId, + reason: _multipath.AllowRootRelayFallback + ? "direct bootstrap did not discover a peer path" + : "direct bootstrap did not discover a usable peer path"); } if (ZeroTierTrace.Enabled) @@ -814,6 +821,9 @@ private bool HasSufficientDirectPath(NodeId peerNodeId) ? HasDirectPathCandidate(peerNodeId) : HasConfirmedDirectPath(peerNodeId); + private bool HasUsableDirectPath(NodeId peerNodeId) + => HasSufficientDirectPath(peerNodeId) || CanOptimisticallyUseHintedDirectPath(peerNodeId); + private bool HasConfirmedDirectPath(NodeId peerNodeId) { if (_peerPaths.GetSnapshot(peerNodeId).Length != 0) @@ -855,6 +865,11 @@ private bool HasConfirmedDirectPath(NodeId peerNodeId) return false; } + private bool CanOptimisticallyUseHintedDirectPath(NodeId peerNodeId) + => !_multipath.AllowRootRelayFallback && + _peerPaths.GetSnapshot(peerNodeId).Length == 0 && + GetOrCreateDirectEndpointManager(peerNodeId).Endpoints.Length != 0; + private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { if (ShouldUsePeerRootSocketAffinity(peerNodeId) && @@ -1375,17 +1390,6 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel return true; } - if (!_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId)) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] Reject hinted direct path selection: peer={peerNodeId} hinted={ZeroTierDirectEndpointSelection.Format(hinted)}."); - } - - selected = default; - return false; - } - if (hinted.Length > 0) { var endpointIndex = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); @@ -2015,6 +2019,12 @@ internal IPEndPoint[] GetLocalDirectPathAdvertisementsForTests() internal Task SendViaRootSocketsForTestsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken = default) => SendViaRootSocketsAsync(packet, cancellationToken); + internal bool TrySelectDirectPathForTests(NodeId peerNodeId, uint flowId, out ZeroTierSelectedPeerPath selected) + => TrySelectDirectPath(peerNodeId, flowId, out selected); + + internal void SeedDirectEndpointsForTests(NodeId peerNodeId, params IPEndPoint[] endpoints) + => GetOrCreateDirectEndpointManager(peerNodeId).SeedEndpoints(endpoints); + internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId) => _peerRootSocketAffinity.Observe(peerNodeId, localSocketId, _rootEndpoint); From 118bbe785a63c966e060ff68c2eaa6f7da5f6913 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:00:50 +0100 Subject: [PATCH 207/296] Rotate hinted direct payload paths --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 32 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 22 ++++++++----- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 5bb9646..5333419 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -117,6 +117,38 @@ public async Task DataplaneRuntime_DirectOnly_CanSelectHintedPath_WithoutConfirm Assert.Equal(0, selected.LocalSocketId); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_RotatesAcrossHintedPaths_BeforeConfirmation() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var peerNodeId = new NodeId(0x3333333333); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoints); + + Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var first)); + Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var second)); + Assert.NotEqual(first.RemoteEndPoint, second.RemoteEndPoint); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 1fee10b..321a5c1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1392,19 +1392,25 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel if (hinted.Length > 0) { - var endpointIndex = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); - var preferredLocalSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[endpointIndex]); - var socketIndex = preferredLocalSocketIds.Length <= 1 - ? 0 - : (int)((flowId / (uint)hinted.Length) % (uint)preferredLocalSocketIds.Length); - var localSocketId = preferredLocalSocketIds[socketIndex]; + var useRotatingHintedPath = !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + var endpoint = useRotatingHintedPath + ? _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, budget: 1)[0] + : hinted[hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length)]; + var preferredLocalSocketIds = useRotatingHintedPath + ? _directHintPlanner.GetRotatingSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true) + : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + var localSocketId = useRotatingHintedPath + ? preferredLocalSocketIds[0] + : preferredLocalSocketIds[preferredLocalSocketIds.Length <= 1 + ? 0 + : (int)((flowId / (uint)hinted.Length) % (uint)preferredLocalSocketIds.Length)]; if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine( - $"[zerotier] Select hinted direct path: peer={peerNodeId} endpoint={hinted[endpointIndex]} socket={localSocketId} relayAllowed={_multipath.AllowRootRelayFallback} confirmed={HasConfirmedDirectPath(peerNodeId)}."); + $"[zerotier] Select hinted direct path: peer={peerNodeId} endpoint={endpoint} socket={localSocketId} relayAllowed={_multipath.AllowRootRelayFallback} confirmed={HasConfirmedDirectPath(peerNodeId)} rotating={useRotatingHintedPath}."); } - selected = new ZeroTierSelectedPeerPath(localSocketId, hinted[endpointIndex]); + selected = new ZeroTierSelectedPeerPath(localSocketId, endpoint); return true; } From d51d6f120e1f95f725e78d3bf5d9235c213c589e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:10:50 +0100 Subject: [PATCH 208/296] Fan out direct-only SYN payloads --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 68 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 66 +++++++++++++++++- .../ZeroTierTcpHandshakeClassifier.cs | 47 +++++++++++++ 3 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierTcpHandshakeClassifier.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 5333419..e3c3592 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -4,6 +4,7 @@ using System.Security.Cryptography; using ZTSharp.ZeroTier; using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Net; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -149,6 +150,73 @@ public async Task DataplaneRuntime_DirectOnly_RotatesAcrossHintedPaths_BeforeCon Assert.NotEqual(first.RemoteEndPoint, second.RemoteEndPoint); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossHintedPaths_BeforeConfirmation() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4246) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); + + var payloadFanout = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .GroupBy(send => Convert.ToHexString(send.Payload)) + .Select(group => new + { + Endpoints = group.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray(), + Count = group.Count() + }) + .FirstOrDefault(group => group.Count == hintedEndpoints.Length); + + Assert.NotNull(payloadFanout); + Assert.Equal(hintedEndpoints, payloadFanout.Endpoints); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 321a5c1..fca34ac 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -415,7 +415,8 @@ public async ValueTask SendIpv4Async(NodeId peerNodeId, ReadOnlyMemory ipv remoteProtocolVersion: peerProtocolVersion); var flowId = _multipath.Enabled ? ZeroTierFlowId.Derive(ipv4Packet.Span) : 0; - await SendToPeerAsync(peerNodeId, packet, flowId, cancellationToken).ConfigureAwait(false); + var preferHintedDirectFanout = ZeroTierTcpHandshakeClassifier.IsInitialSyn(ipv4Packet.Span); + await SendToPeerAsync(peerNodeId, packet, flowId, preferHintedDirectFanout, cancellationToken).ConfigureAwait(false); } public async ValueTask SendEthernetFrameAsync( @@ -448,13 +449,17 @@ public async ValueTask SendEthernetFrameAsync( ? ZeroTierFlowId.Derive(frame.Span) : 0; - await SendToPeerAsync(peerNodeId, packet, flowId, cancellationToken).ConfigureAwait(false); + var preferHintedDirectFanout = + (etherType == ZeroTierFrameCodec.EtherTypeIpv4 || etherType == ZeroTierFrameCodec.EtherTypeIpv6) && + ZeroTierTcpHandshakeClassifier.IsInitialSyn(frame.Span); + await SendToPeerAsync(peerNodeId, packet, flowId, preferHintedDirectFanout, cancellationToken).ConfigureAwait(false); } private async ValueTask SendToPeerAsync( NodeId peerNodeId, ReadOnlyMemory packet, uint flowId, + bool preferHintedDirectFanout, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -484,6 +489,40 @@ private async ValueTask SendToPeerAsync( var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; + if (preferHintedDirectFanout && + TryGetDirectOnlyHintedPayloadFanout(peerNodeId, out var fanoutPaths)) + { + var directSuccess = 0; + for (var i = 0; i < fanoutPaths.Length; i++) + { + var path = fanoutPaths[i]; + if (shouldRecord) + { + _peerQos.RecordOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); + } + + try + { + await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); + directSuccess++; + } + catch (SocketException) + { + if (shouldRecord) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); + } + } + } + + if (directSuccess > 0) + { + return; + } + + EnsureRelayAllowedForPayload(peerNodeId, reason: "direct-only hinted payload fanout failed"); + } + var confirmed = _peerEcho.TryGetLastRttMs(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, out _); if (_multipath.WarmupDuplicateToRoot && _multipath.AllowRootRelayFallback && !confirmed) { @@ -634,6 +673,29 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo } } + private bool TryGetDirectOnlyHintedPayloadFanout( + NodeId peerNodeId, + out ZeroTierSelectedPeerPath[] fanoutPaths) + { + fanoutPaths = Array.Empty(); + if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) + { + return false; + } + + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length <= 1) + { + return false; + } + + fanoutPaths = _directHintPlanner.GetNextHintedCandidatesForMaintenance( + peerNodeId, + Array.Empty(), + endpointBudget: Math.Min(hinted.Length, 4)); + return fanoutPaths.Length > 1; + } + private void EnsureRelayAllowedForPayload(NodeId peerNodeId, string reason) { if (_multipath.AllowRootRelayFallback) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierTcpHandshakeClassifier.cs b/ZTSharp/ZeroTier/Internal/ZeroTierTcpHandshakeClassifier.cs new file mode 100644 index 0000000..40f2831 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierTcpHandshakeClassifier.cs @@ -0,0 +1,47 @@ +using ZTSharp.ZeroTier.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierTcpHandshakeClassifier +{ + public static bool IsInitialSyn(ReadOnlySpan ipPacket) + => TryGetTcpFlags(ipPacket, out var flags) && + (flags & TcpCodec.Flags.Syn) != 0 && + (flags & TcpCodec.Flags.Ack) == 0; + + private static bool TryGetTcpFlags(ReadOnlySpan ipPacket, out TcpCodec.Flags flags) + { + flags = 0; + + if (Ipv4Codec.TryParse(ipPacket, out _, out _, out var protocol, out var payload)) + { + return TryParseFlags(protocol, payload, out flags); + } + + if (Ipv6Codec.TryParseTransportPayload(ipPacket, out _, out _, out var nextHeader, out _, out payload)) + { + return TryParseFlags(nextHeader, payload, out flags); + } + + return false; + } + + private static bool TryParseFlags(byte protocol, ReadOnlySpan payload, out TcpCodec.Flags flags) + { + flags = 0; + if (protocol != TcpCodec.ProtocolNumber) + { + return false; + } + + return TcpCodec.TryParse( + payload, + out _, + out _, + out _, + out _, + out flags, + out _, + out _); + } +} From 4d5857c9c2d388281c27ab45510bbc1e875a0a2a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:14:42 +0100 Subject: [PATCH 209/296] Extend direct-only TCP SYN budget --- .../UserSpaceTcpClientConnectTests.cs | 36 +++++++++++++++++++ ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 22 ++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 8 +++++ .../Internal/ZeroTierSocketTcpConnector.cs | 3 +- ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs | 18 ++++++---- .../Net/UserSpaceTcpConnectOptions.cs | 19 ++++++++++ 6 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 ZTSharp/ZeroTier/Net/UserSpaceTcpConnectOptions.cs diff --git a/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs b/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs index 06b3571..e269861 100644 --- a/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs +++ b/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs @@ -116,5 +116,41 @@ public async Task ConnectAsync_RetransmitsSyn_WhenNoSynAckArrives() Assert.Equal(TcpCodec.Flags.Ack, ackFlags); Assert.Equal(1001u, ackAck); } + + [Fact] + public async Task ConnectAsync_UsesConfiguredConnectTimeout() + { + await using var link = new InspectableIpv4Link(); + var localIp = IPAddress.Parse("10.0.0.1"); + var remoteIp = IPAddress.Parse("10.0.0.2"); + const ushort remotePort = 80; + const ushort localPort = 50000; + + await using var client = new UserSpaceTcpClient( + link, + localIp, + remoteIp, + remotePort, + localPort: localPort, + mss: 1200, + connectOptions: new UserSpaceTcpConnectOptions + { + ConnectTimeout = TimeSpan.FromMilliseconds(450), + InitialRetryDelay = TimeSpan.FromMilliseconds(200), + MaxRetryDelay = TimeSpan.FromMilliseconds(200) + }); + + var connectTask = client.ConnectAsync(); + + for (var attempt = 0; attempt < 3; attempt++) + { + var syn = await link.Outgoing.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(1)); + Assert.True(Ipv4Codec.TryParse(syn.Span, out _, out _, out _, out var payload)); + Assert.True(TcpCodec.TryParse(payload, out _, out _, out _, out _, out var flags, out _, out _)); + Assert.Equal(TcpCodec.Flags.Syn, flags); + } + + await Assert.ThrowsAsync(async () => await connectTask); + } } diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index e3c3592..2a9ac40 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -217,6 +217,28 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossHintedPath Assert.Equal(hintedEndpoints, payloadFanout.Endpoints); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + Assert.Equal(TimeSpan.FromSeconds(30), runtime.GetTcpConnectOptionsForTests().ConnectTimeout); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index fca34ac..faceedb 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -2099,6 +2099,9 @@ internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId internal Task SendHelloViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) => SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken); + internal UserSpaceTcpConnectOptions GetTcpConnectOptionsForTests() + => GetSuggestedTcpConnectOptions(); + internal void PrimePeerForTests( ZeroTierIdentity peerIdentity, byte peerProtocolVersion = ZeroTierHelloClient.AdvertisedProtocolVersion, @@ -2124,6 +2127,11 @@ internal void PrimePeerForTests( peerRevision); } + internal UserSpaceTcpConnectOptions GetSuggestedTcpConnectOptions() + => _multipath.Enabled && !_multipath.AllowRootRelayFallback + ? UserSpaceTcpConnectOptions.DirectOnlyMultipath + : UserSpaceTcpConnectOptions.Default; + private NodeId[] GetPeersForMultipathMaintenance() { var peers = _peerPaths.GetPeersSnapshot(); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs index 5bb0ab5..8e28b57 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs @@ -160,7 +160,8 @@ public static async ValueTask ConnectAsync( localAddress, remote.Address, remotePort: (ushort)remote.Port, - localPort: localPort); + localPort: localPort, + connectOptions: runtime.GetSuggestedTcpConnectOptions()); try { diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs index c645d7c..3333728 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs @@ -1,4 +1,5 @@ using System.Buffers.Binary; +using System.Diagnostics; using System.IO; using System.Net; using System.Security.Cryptography; @@ -15,6 +16,7 @@ internal sealed class UserSpaceTcpClient : IAsyncDisposable private readonly ushort _remotePort; private readonly ushort _localPort; private readonly ushort _mss; + private readonly UserSpaceTcpConnectOptions _connectOptions; private readonly UserSpaceTcpConnectionSignals _signals = new(); private readonly UserSpaceTcpReceiver _receiver = new(); @@ -34,7 +36,8 @@ public UserSpaceTcpClient( IPAddress remoteAddress, ushort remotePort, ushort? localPort = null, - ushort mss = DefaultMss) + ushort mss = DefaultMss, + UserSpaceTcpConnectOptions? connectOptions = null) { ArgumentNullException.ThrowIfNull(link); ArgumentNullException.ThrowIfNull(localAddress); @@ -67,6 +70,7 @@ public UserSpaceTcpClient( _remotePort = remotePort; _localPort = localPort ?? GenerateEphemeralPort(); _mss = mss; + _connectOptions = connectOptions ?? UserSpaceTcpConnectOptions.Default; _sender = new UserSpaceTcpSender( link, @@ -113,10 +117,10 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default) var synOptions = TcpCodec.EncodeMssOption(_mss); var synSeq = _sender.AllocateNextSequence(bytes: 1); - const int synRetries = 5; - var delay = TimeSpan.FromMilliseconds(250); + var delay = _connectOptions.InitialRetryDelay; + var deadline = Stopwatch.GetTimestamp() + (long)(_connectOptions.ConnectTimeout.TotalSeconds * Stopwatch.Frequency); - for (var attempt = 0; attempt <= synRetries; attempt++) + while (true) { cancellationToken.ThrowIfCancellationRequested(); @@ -138,14 +142,14 @@ await _sender } catch (TimeoutException) { - if (attempt >= synRetries) + if (Stopwatch.GetTimestamp() >= deadline) { break; } - delay = delay < TimeSpan.FromSeconds(2) + delay = delay < _connectOptions.MaxRetryDelay ? delay + delay - : TimeSpan.FromSeconds(2); + : _connectOptions.MaxRetryDelay; } } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpConnectOptions.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpConnectOptions.cs new file mode 100644 index 0000000..11eb297 --- /dev/null +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpConnectOptions.cs @@ -0,0 +1,19 @@ +namespace ZTSharp.ZeroTier.Net; + +internal sealed class UserSpaceTcpConnectOptions +{ + public static UserSpaceTcpConnectOptions Default { get; } = new(); + + public static UserSpaceTcpConnectOptions DirectOnlyMultipath { get; } = new() + { + ConnectTimeout = TimeSpan.FromSeconds(30), + InitialRetryDelay = TimeSpan.FromMilliseconds(250), + MaxRetryDelay = TimeSpan.FromSeconds(2) + }; + + public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(8); + + public TimeSpan InitialRetryDelay { get; init; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan MaxRetryDelay { get; init; } = TimeSpan.FromSeconds(2); +} From 244d6bddbac36df98cc7db69c37c763244189768 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:17:12 +0100 Subject: [PATCH 210/296] Fan out direct-only SYNs across sockets --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 3 ++- .../Internal/ZeroTierDataplaneRuntime.cs | 24 +++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 2a9ac40..a9f6e0c 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -203,6 +203,7 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossHintedPath await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); + var expectedSendCount = hintedEndpoints.Length * udp.LocalSockets.Count; var payloadFanout = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .GroupBy(send => Convert.ToHexString(send.Payload)) @@ -211,7 +212,7 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossHintedPath Endpoints = group.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray(), Count = group.Count() }) - .FirstOrDefault(group => group.Count == hintedEndpoints.Length); + .FirstOrDefault(group => group.Count == expectedSendCount); Assert.NotNull(payloadFanout); Assert.Equal(hintedEndpoints, payloadFanout.Endpoints); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index faceedb..8f780b3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -689,10 +689,26 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return false; } - fanoutPaths = _directHintPlanner.GetNextHintedCandidatesForMaintenance( - peerNodeId, - Array.Empty(), - endpointBudget: Math.Min(hinted.Length, 4)); + var selectedEndpoints = _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, budget: Math.Min(hinted.Length, 4)); + var unique = new HashSet(); + var candidates = new List(selectedEndpoints.Length); + for (var i = 0; i < selectedEndpoints.Length; i++) + { + var endpoint = selectedEndpoints[i]; + var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + for (var s = 0; s < socketIds.Length; s++) + { + var key = new ZeroTierPeerPhysicalPathKey(socketIds[s], endpoint); + if (!unique.Add(key)) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(socketIds[s], endpoint)); + } + } + + fanoutPaths = candidates.ToArray(); return fanoutPaths.Length > 1; } From 06b1460be23030ccf69ee379e07facbf2564b560 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:30:33 +0100 Subject: [PATCH 211/296] Use echo-first direct-only hinted maintenance --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 71 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 9 ++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index a9f6e0c..56fb189 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -240,6 +240,63 @@ public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() Assert.Equal(TimeSpan.FromSeconds(30), runtime.GetTcpConnectOptionsForTests().ConnectTimeout); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesEchoAcrossAllSockets_ForModernPeers() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var hintedSends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .Select(send => new + { + send.LocalSocketId, + send.RemoteEndPoint, + Verb = TryDecodeVerb(send.Payload, sharedKey, out var verb) ? verb : (ZeroTierVerb?)null + }) + .ToArray(); + + var echoFanout = hintedSends + .Where(send => send.Verb == ZeroTierVerb.Echo) + .ToArray(); + + Assert.Equal(hintedEndpoints.Length * udp.LocalSockets.Count, echoFanout.Length); + Assert.Equal(new[] { 0, 1 }, echoFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); + Assert.Equal(hintedEndpoints, echoFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); + Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Hello); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { @@ -450,6 +507,20 @@ private static byte[] BuildRootRelayedHelloPacket( advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, out _); + private static bool TryDecodeVerb(byte[] packet, byte[] sharedKey, out ZeroTierVerb verb) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed)) + { + verb = default; + return false; + } + + verb = parsed.Header.Verb; + return true; + } + private static byte[] BuildPushDirectPathsPacket( NodeId source, NodeId destination, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 8f780b3..faf7c67 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1708,9 +1708,13 @@ private void CleanupPendingHellosIfNeeded(long nowMs) private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) => _peerNegotiation.TryGetRemoteUtility(peerNodeId, localSocketId, remoteEndPoint, out var util) ? util : (short)0; - private bool ShouldForceFullHelloForHintedCandidate(NodeId peerNodeId) + private bool ShouldFanOutHintedBootstrapAcrossSockets(NodeId peerNodeId) => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + private bool ShouldForceFullHelloForHintedCandidate(NodeId peerNodeId) + => ShouldFanOutHintedBootstrapAcrossSockets(peerNodeId) && + !CanUseEchoForDirectBootstrap(peerNodeId); + private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) @@ -1770,10 +1774,11 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati { var candidate = hintedCandidates[c]; var forceFullHello = ShouldForceFullHelloForHintedCandidate(peerNodeId); + var useAllEligibleLocalSockets = ShouldFanOutHintedBootstrapAcrossSockets(peerNodeId); var minIntervalMs = forceFullHello ? DirectOnlyHintHelloIntervalMs : DirectHintFullHelloIntervalMs; - var localSocketIds = forceFullHello + var localSocketIds = useAllEligibleLocalSockets ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, candidate.RemoteEndPoint) : [candidate.LocalSocketId]; From 1d6678663e7eb4cdc904b474b46d42e896776f4a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:38:01 +0100 Subject: [PATCH 212/296] Use same-socket rendezvous direct bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 60 +++++++++++++++++++ ...irectEndpointManagerSocketAffinityTests.cs | 4 +- .../Internal/ZeroTierDirectEndpointManager.cs | 2 +- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 56fb189..dc31da5 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -50,6 +50,66 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() Assert.Equal(2, holePunch.HopLimit); } + [Fact] + public async Task DataplaneRuntime_RendezvousBootstrap_UsesReceivingSocket_ForModernPeer() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length > 4), + TimeSpan.FromSeconds(2)); + + var bootstrapSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .ToArray(); + + var bootstrapSockets = bootstrapSends + .Select(send => send.LocalSocketId) + .Distinct() + .ToArray(); + + Assert.Equal(new[] { 1 }, bootstrapSockets); + } + [Fact] public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() { diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index ac8bd67..e96130e 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -31,7 +31,7 @@ await manager.HandleRendezvousFromRootAsync( } [Fact] - public async Task Rendezvous_RequestsAllEligibleSockets_ForDirectBootstrap() + public async Task Rendezvous_RequestsReceivingSocket_ForDirectBootstrap() { await using var udp = new RecordingUdpTransport(); var relay = new IPEndPoint(IPAddress.Loopback, 9999); @@ -55,7 +55,7 @@ await manager.HandleRendezvousFromRootAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Equal((endpoint, false, false, 1), hints[0]); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 6e6f053..cfdddd9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -124,7 +124,7 @@ await _handleDirectEndpointHintAsync( receivedLocalSocketId, endpoint, false, - true, + false, cancellationToken) .ConfigureAwait(false); } From 1933d01fca01cecc275cb22321a2e8a2fd102d45 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:41:27 +0100 Subject: [PATCH 213/296] Honor same-socket direct hint bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 12 +++---- .../Internal/ZeroTierDataplaneRuntime.cs | 31 +++++++++++++++---- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index dc31da5..4b2f65e 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -97,17 +97,13 @@ await WaitForConditionAsync( () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length > 4), TimeSpan.FromSeconds(2)); - var bootstrapSends = udp.GetSendsSnapshot() + var firstBootstrapSend = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) - .ToArray(); - - var bootstrapSockets = bootstrapSends - .Select(send => send.LocalSocketId) - .Distinct() - .ToArray(); + .OrderBy(send => send.LocalSocketId) + .First(); - Assert.Equal(new[] { 1 }, bootstrapSockets); + Assert.Equal(1, firstBootstrapSend.LocalSocketId); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index faf7c67..3a7d30f 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1197,12 +1197,11 @@ private async ValueTask HandleDirectEndpointHintAsync( $"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId} fullHello={forceFullHello} allSockets={useAllEligibleLocalSockets}."); } - var localSocketIds = useAllEligibleLocalSockets - ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint) - : _directHintPlanner.GetRotatingSocketIds( - peerNodeId, - endpoint, - includeFallbackLocalSockets: true); + var localSocketIds = GetDirectHintBootstrapSocketIds( + peerNodeId, + endpoint, + receivedLocalSocketId, + useAllEligibleLocalSockets); if (forceFullHello && useAllEligibleLocalSockets && localSocketIds.Length > 1) { await SendHelloPacketAcrossSocketsAsync( @@ -1238,6 +1237,26 @@ private bool CanUseEchoForDirectBootstrap(NodeId peerNodeId) !(version.MajorVersion == 1 && version.MinorVersion == 1 && version.Revision == 0); } + private int[] GetDirectHintBootstrapSocketIds( + NodeId peerNodeId, + IPEndPoint endpoint, + int receivedLocalSocketId, + bool useAllEligibleLocalSockets) + { + if (useAllEligibleLocalSockets) + { + return _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + } + + var preferredSocketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); + if (preferredSocketIds.Length != 0) + { + return preferredSocketIds; + } + + return [receivedLocalSocketId]; + } + private Task SendDirectBootstrapProbeAsync( NodeId peerNodeId, int localSocketId, From 66504d1598562d89a681be3ee393263791a0031c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:44:18 +0100 Subject: [PATCH 214/296] Narrow first-hop direct hint bootstrap --- ...TierDirectEndpointManagerPushFlagsTests.cs | 4 +-- ...LocalDirectPathAdvertisementSourceTests.cs | 20 +++++++++++++++ .../Internal/ZeroTierDirectEndpointManager.cs | 2 +- ...oTierLocalDirectPathAdvertisementSource.cs | 25 ++++++++++++++++--- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 45b90b9..e819fd7 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -103,7 +103,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_NormalHint_RequestsAllEligibleSockets() + public async Task PushDirectPaths_NormalHint_UsesReceivingSocketFirst() { var udp = new RecordingUdpTransport(); @@ -127,7 +127,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Equal((endpoint, false, false, 1), hints[0]); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs index 4d395e1..236e84e 100644 --- a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs @@ -33,4 +33,24 @@ public void GetSnapshot_ExcludesNonPublicLocalAddresses() Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("192.168.224.1"))); Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("fd7a:115c:a1e0::5132:b90e"))); } + + [Fact] + public void GetSnapshot_DoesNotProjectPublicHostAddressesOntoPrivateBoundSockets() + { + var source = new ZeroTierLocalDirectPathAdvertisementSource( + getLocalAddresses: () => + [ + IPAddress.Parse("212.241.85.84"), + IPAddress.Parse("198.51.100.10") + ]); + + var endpoints = source.GetSnapshot( + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 62984)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 62985)) + ]); + + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("212.241.85.84"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("198.51.100.10"))); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index cfdddd9..ca99c0b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -191,7 +191,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs index 21a3fe3..e15260e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs @@ -36,20 +36,28 @@ public IPEndPoint[] GetSnapshot(IReadOnlyList localSocke var endpoints = new List(localAddresses.Length * localSockets.Count); for (var i = 0; i < localSockets.Count; i++) { - var port = localSockets[i].LocalEndpoint.Port; + var localEndpoint = localSockets[i].LocalEndpoint; + var port = localEndpoint.Port; if (port == 0) { continue; } + var boundAddress = Canonicalize(localEndpoint.Address); + var allowAnyPublicAddress = boundAddress is null || + boundAddress.Equals(IPAddress.Any) || + boundAddress.Equals(IPAddress.IPv6Any); + for (var a = 0; a < localAddresses.Length; a++) { - if (!ShouldAdvertise(localAddresses[a])) + var candidateAddress = localAddresses[a]; + if (!ShouldAdvertise(candidateAddress) || + !ShouldAdvertiseForSocket(boundAddress, candidateAddress, allowAnyPublicAddress)) { continue; } - endpoints.Add(new IPEndPoint(localAddresses[a], port)); + endpoints.Add(new IPEndPoint(candidateAddress, port)); } } @@ -118,6 +126,16 @@ private static IPAddress[] GetLocalAddresses() private static bool ShouldAdvertise(IPAddress address) => ZeroTierDirectEndpointSelection.IsPublicEndpoint(new IPEndPoint(address, 1)); + private static bool ShouldAdvertiseForSocket(IPAddress? boundAddress, IPAddress candidateAddress, bool allowAnyPublicAddress) + { + if (allowAnyPublicAddress) + { + return true; + } + + return boundAddress is not null && boundAddress.Equals(candidateAddress); + } + private static IPAddress? Canonicalize(IPAddress? address) { if (address is null) @@ -133,4 +151,3 @@ private static bool ShouldAdvertise(IPAddress address) return address; } } - From dce8ef5d2b42d54ca128b367bb592a520f1765bf Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:50:47 +0100 Subject: [PATCH 215/296] Keep maintenance hinted probes socket-affine --- ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs | 6 +++--- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 4b2f65e..dfeaedb 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -297,7 +297,7 @@ public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesEchoAcrossAllSockets_ForModernPeers() + public async Task DataplaneRuntime_DirectOnly_MaintenanceKeepsHintedEchoOnSelectedSocket_ForModernPeers() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), @@ -347,8 +347,8 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesEchoAcrossAllSocket .Where(send => send.Verb == ZeroTierVerb.Echo) .ToArray(); - Assert.Equal(hintedEndpoints.Length * udp.LocalSockets.Count, echoFanout.Length); - Assert.Equal(new[] { 0, 1 }, echoFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); + Assert.Equal(hintedEndpoints.Length, echoFanout.Length); + Assert.Equal(new[] { 0 }, echoFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); Assert.Equal(hintedEndpoints, echoFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Hello); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 3a7d30f..a5774df 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1793,13 +1793,10 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati { var candidate = hintedCandidates[c]; var forceFullHello = ShouldForceFullHelloForHintedCandidate(peerNodeId); - var useAllEligibleLocalSockets = ShouldFanOutHintedBootstrapAcrossSockets(peerNodeId); var minIntervalMs = forceFullHello ? DirectOnlyHintHelloIntervalMs : DirectHintFullHelloIntervalMs; - var localSocketIds = useAllEligibleLocalSockets - ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, candidate.RemoteEndPoint) - : [candidate.LocalSocketId]; + int[] localSocketIds = [candidate.LocalSocketId]; if (forceFullHello && localSocketIds.Length > 1) { From deeb93d3778eecf5cdcec3f30b6c0a8a0791f76d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:04:32 +0100 Subject: [PATCH 216/296] Keep direct-only hinted payloads flow-sticky --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 90 +++++++++++-------- .../Internal/ZeroTierDataplaneRuntime.cs | 55 ++++++------ 2 files changed, 82 insertions(+), 63 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index dfeaedb..9309daa 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -18,7 +18,7 @@ public sealed class ZeroTierDataplaneRuntimeDirectPathTests public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); var rootNodeId = new NodeId(0x1111111111); @@ -107,17 +107,14 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() + public async Task DataplaneRuntime_SeededHintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - Assert.True(peerIdentity.HasPrivateKey); - Assert.True(peerIdentity.LocallyValidate()); - var rootNodeId = new NodeId(0x1111111111); + var peerNodeId = new NodeId(0x3333333333); var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4243); var rootKey = RandomNumberGenerator.GetBytes(48); @@ -130,17 +127,9 @@ public async Task DataplaneRuntime_HintedEndpoints_AreVisibleToMaintenanceBefore planetId: 1, planetTimestamp: 1); - var sharedKey = new byte[48]; - ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); - - udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, udp.LocalSockets[0].LocalEndpoint)); - udp.EnqueueInbound(localSocketId: 0, RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); - - await WaitForConditionAsync( - () => runtime.GetHintedDirectCandidatesForMaintenance(peerIdentity.NodeId).Length != 0, - TimeSpan.FromSeconds(2)); + runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoint); - var candidates = runtime.GetHintedDirectCandidatesForMaintenance(peerIdentity.NodeId); + var candidates = runtime.GetHintedDirectCandidatesForMaintenance(peerNodeId); var candidate = Assert.Single(candidates); Assert.Equal(hintedEndpoint, candidate.RemoteEndPoint); Assert.Equal(0, candidate.LocalSocketId); @@ -150,7 +139,7 @@ await WaitForConditionAsync( public async Task DataplaneRuntime_DirectOnly_CanSelectHintedPath_WithoutConfirmedHop0() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); var rootNodeId = new NodeId(0x1111111111); @@ -175,10 +164,10 @@ public async Task DataplaneRuntime_DirectOnly_CanSelectHintedPath_WithoutConfirm } [Fact] - public async Task DataplaneRuntime_DirectOnly_RotatesAcrossHintedPaths_BeforeConfirmation() + public async Task DataplaneRuntime_DirectOnly_KeepsHintedPathSticky_PerFlow_BeforeConfirmation() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); var rootNodeId = new NodeId(0x1111111111); @@ -203,15 +192,47 @@ public async Task DataplaneRuntime_DirectOnly_RotatesAcrossHintedPaths_BeforeCon Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var first)); Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var second)); + Assert.Equal(first, second); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_UsesDifferentHintedPaths_ForDifferentFlows_BeforeConfirmation() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var peerNodeId = new NodeId(0x3333333333); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoints); + + Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var first)); + Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 1, out var second)); Assert.NotEqual(first.RemoteEndPoint, second.RemoteEndPoint); } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossHintedPaths_BeforeConfirmation() + public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnStickyHintedPath_BeforeConfirmation() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); @@ -259,26 +280,21 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossHintedPath await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); - var expectedSendCount = hintedEndpoints.Length * udp.LocalSockets.Count; - var payloadFanout = udp.GetSendsSnapshot() + var payloadSends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) - .GroupBy(send => Convert.ToHexString(send.Payload)) - .Select(group => new - { - Endpoints = group.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray(), - Count = group.Count() - }) - .FirstOrDefault(group => group.Count == expectedSendCount); + .Where(send => send.Payload.Length > 4) + .ToArray(); - Assert.NotNull(payloadFanout); - Assert.Equal(hintedEndpoints, payloadFanout.Endpoints); + var payloadSend = Assert.Single(payloadSends); + Assert.Equal(0, payloadSend.LocalSocketId); + Assert.Equal(hintedEndpoints[0], payloadSend.RemoteEndPoint); } [Fact] public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); var rootNodeId = new NodeId(0x1111111111); @@ -300,8 +316,8 @@ public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() public async Task DataplaneRuntime_DirectOnly_MaintenanceKeepsHintedEchoOnSelectedSocket_ForModernPeers() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index a5774df..1578b82 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -490,7 +490,7 @@ private async ValueTask SendToPeerAsync( var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; if (preferHintedDirectFanout && - TryGetDirectOnlyHintedPayloadFanout(peerNodeId, out var fanoutPaths)) + TryGetDirectOnlyHintedPayloadFanout(peerNodeId, flowId, out var fanoutPaths)) { var directSuccess = 0; for (var i = 0; i < fanoutPaths.Length; i++) @@ -675,6 +675,7 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo private bool TryGetDirectOnlyHintedPayloadFanout( NodeId peerNodeId, + uint flowId, out ZeroTierSelectedPeerPath[] fanoutPaths) { fanoutPaths = Array.Empty(); @@ -689,27 +690,20 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return false; } - var selectedEndpoints = _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, budget: Math.Min(hinted.Length, 4)); - var unique = new HashSet(); - var candidates = new List(selectedEndpoints.Length); - for (var i = 0; i < selectedEndpoints.Length; i++) + var endpoint = hinted[(int)(flowId % (uint)hinted.Length)]; + var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); + if (socketIds.Length == 0) { - var endpoint = selectedEndpoints[i]; - var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); - for (var s = 0; s < socketIds.Length; s++) - { - var key = new ZeroTierPeerPhysicalPathKey(socketIds[s], endpoint); - if (!unique.Add(key)) - { - continue; - } + socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + } - candidates.Add(new ZeroTierSelectedPeerPath(socketIds[s], endpoint)); - } + if (socketIds.Length == 0) + { + return false; } - fanoutPaths = candidates.ToArray(); - return fanoutPaths.Length > 1; + fanoutPaths = [new ZeroTierSelectedPeerPath(socketIds[0], endpoint)]; + return true; } private void EnsureRelayAllowedForPayload(NodeId peerNodeId, string reason) @@ -1489,14 +1483,23 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel if (hinted.Length > 0) { - var useRotatingHintedPath = !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); - var endpoint = useRotatingHintedPath - ? _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, budget: 1)[0] - : hinted[hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length)]; - var preferredLocalSocketIds = useRotatingHintedPath - ? _directHintPlanner.GetRotatingSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true) + var useStickyHintedPath = !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + var endpoint = hinted[hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length)]; + var preferredLocalSocketIds = useStickyHintedPath + ? _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint) : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); - var localSocketId = useRotatingHintedPath + if (preferredLocalSocketIds.Length == 0) + { + preferredLocalSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + } + + if (preferredLocalSocketIds.Length == 0) + { + selected = default; + return false; + } + + var localSocketId = useStickyHintedPath ? preferredLocalSocketIds[0] : preferredLocalSocketIds[preferredLocalSocketIds.Length <= 1 ? 0 @@ -1504,7 +1507,7 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine( - $"[zerotier] Select hinted direct path: peer={peerNodeId} endpoint={endpoint} socket={localSocketId} relayAllowed={_multipath.AllowRootRelayFallback} confirmed={HasConfirmedDirectPath(peerNodeId)} rotating={useRotatingHintedPath}."); + $"[zerotier] Select hinted direct path: peer={peerNodeId} endpoint={endpoint} socket={localSocketId} relayAllowed={_multipath.AllowRootRelayFallback} confirmed={HasConfirmedDirectPath(peerNodeId)} sticky={useStickyHintedPath}."); } selected = new ZeroTierSelectedPeerPath(localSocketId, endpoint); From 6eab6e8cf8760a03aca64753ff76a84b464b0ca6 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:07:36 +0100 Subject: [PATCH 217/296] Prefer hinted socket affinity for direct-only maintenance --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 5 +- .../Internal/ZeroTierDataplaneRuntime.cs | 51 ++++++++++++++++--- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 9309daa..8d44e6b 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -196,7 +196,7 @@ public async Task DataplaneRuntime_DirectOnly_KeepsHintedPathSticky_PerFlow_Befo } [Fact] - public async Task DataplaneRuntime_DirectOnly_UsesDifferentHintedPaths_ForDifferentFlows_BeforeConfirmation() + public async Task DataplaneRuntime_DirectOnly_PrefersFirstHintedPath_BeforeConfirmation() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); @@ -224,7 +224,8 @@ public async Task DataplaneRuntime_DirectOnly_UsesDifferentHintedPaths_ForDiffer Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var first)); Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 1, out var second)); - Assert.NotEqual(first.RemoteEndPoint, second.RemoteEndPoint); + Assert.Equal(hintedEndpoints[0], first.RemoteEndPoint); + Assert.Equal(first, second); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 1578b82..c9f178e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -690,7 +690,7 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return false; } - var endpoint = hinted[(int)(flowId % (uint)hinted.Length)]; + var endpoint = hinted[0]; var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); if (socketIds.Length == 0) { @@ -1484,7 +1484,9 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel if (hinted.Length > 0) { var useStickyHintedPath = !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); - var endpoint = hinted[hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length)]; + var endpoint = useStickyHintedPath + ? hinted[0] + : hinted[hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length)]; var preferredLocalSocketIds = useStickyHintedPath ? _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint) : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); @@ -1783,10 +1785,7 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); var paths = _peerPaths.GetSnapshot(peerNodeId); - var hintedCandidates = _directHintPlanner.GetNextHintedCandidatesForMaintenance( - peerNodeId, - paths, - DirectHintMaintenanceProbeBudget); + var hintedCandidates = GetHintedCandidatesForMaintenance(peerNodeId, paths); if (paths.Length == 0 && hintedCandidates.Length == 0) { continue; @@ -1951,6 +1950,46 @@ private async ValueTask SendPeerControlAsync( await _udp.SendAsync(localSocketId, remoteEndPoint, packet, cancellationToken).ConfigureAwait(false); } + private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths) + { + if (_multipath.AllowRootRelayFallback || observedPaths.Length != 0 || HasConfirmedDirectPath(peerNodeId)) + { + return _directHintPlanner.GetNextHintedCandidatesForMaintenance( + peerNodeId, + observedPaths, + DirectHintMaintenanceProbeBudget); + } + + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length == 0) + { + return Array.Empty(); + } + + var selectedEndpoints = _directHintPlanner.TakeNextHintedEndpoints( + peerNodeId, + hinted, + DirectHintMaintenanceProbeBudget); + var candidates = new List(selectedEndpoints.Length); + for (var i = 0; i < selectedEndpoints.Length; i++) + { + var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, selectedEndpoints[i]); + if (socketIds.Length == 0) + { + socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, selectedEndpoints[i]); + } + + if (socketIds.Length == 0) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(socketIds[0], selectedEndpoints[i])); + } + + return candidates.ToArray(); + } + private static bool TryGetPacketIdAndVerb(ReadOnlyMemory packet, out (ulong PacketId, ZeroTierVerb Verb) parsed) { if (packet.Length < ZeroTierPacketHeader.Length) From f24b27501e3c6885d88b8f607b2fb1f3bcb05429 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:09:40 +0100 Subject: [PATCH 218/296] Keep direct bootstrap probes on hinted socket --- .../Internal/ZeroTierDataplaneRuntime.cs | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c9f178e..fd1f8e4 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -865,10 +865,12 @@ private async Task ProbeHintedDirectEndpointsAsync( var endpointsToProbe = _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { - var localSocketIds = _directHintPlanner.GetRotatingSocketIds( - peerNodeId, - endpointsToProbe[i], - includeFallbackLocalSockets: true); + var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) + ? GetStickyHintedLocalSocketIds(peerNodeId, endpointsToProbe[i]) + : _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpointsToProbe[i], + includeFallbackLocalSockets: true); for (var s = 0; s < localSocketIds.Length; s++) { await SendDirectBootstrapProbeAsync( @@ -1483,17 +1485,13 @@ private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSel if (hinted.Length > 0) { - var useStickyHintedPath = !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + var useStickyHintedPath = ShouldUseStickyHintedPathSelection(peerNodeId); var endpoint = useStickyHintedPath ? hinted[0] : hinted[hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length)]; var preferredLocalSocketIds = useStickyHintedPath - ? _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint) + ? GetStickyHintedLocalSocketIds(peerNodeId, endpoint) : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); - if (preferredLocalSocketIds.Length == 0) - { - preferredLocalSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); - } if (preferredLocalSocketIds.Length == 0) { @@ -1557,6 +1555,25 @@ private bool TrySelectConfirmedHintedDirectPath( return haveBest; } + private bool ShouldUseStickyHintedPathSelection(NodeId peerNodeId) + => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + + private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + { + var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); + if (socketIds.Length == 0) + { + socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + } + + if (socketIds.Length == 0) + { + return socketIds; + } + + return [socketIds[0]]; + } + private int? GetPathLatencyMsOrNull(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) { if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) @@ -1973,11 +1990,7 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer var candidates = new List(selectedEndpoints.Length); for (var i = 0; i < selectedEndpoints.Length; i++) { - var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, selectedEndpoints[i]); - if (socketIds.Length == 0) - { - socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, selectedEndpoints[i]); - } + var socketIds = GetStickyHintedLocalSocketIds(peerNodeId, selectedEndpoints[i]); if (socketIds.Length == 0) { From fa24e21f51b6d4152984188fbaed44608edfbcf1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:12:51 +0100 Subject: [PATCH 219/296] Prefer fresh push-direct endpoints over stale ones --- ...TierDirectEndpointManagerPushFlagsTests.cs | 26 +++++++++++++++++++ .../Internal/ZeroTierDirectEndpointManager.cs | 13 +++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index e819fd7..e44e178 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -102,6 +102,32 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Empty(udp.Sends); } + [Fact] + public async Task PushDirectPaths_NewerEndpoints_ArePreferredAheadOfStaleOnes() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + var oldEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + var newEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9123); + + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(oldEndpoint, flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(newEndpoint, flags: 0), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Equal(newEndpoint, manager.Endpoints[0]); + Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(oldEndpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(newEndpoint)); + } + [Fact] public async Task PushDirectPaths_NormalHint_UsesReceivingSocketFirst() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index ca99c0b..1cf2355 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -199,10 +199,15 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( IPEndPoint[] endpointsToProbe; lock (_lock) { - var merged = _directEndpoints - .Where(ep => !forget.Contains(FormatEndpointKey(ep))) - .Concat(redirect) - .Concat(add); + var incomingKeys = redirect + .Concat(add) + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); + var merged = redirect + .Concat(add) + .Concat(_directEndpoints.Where(ep => + !forget.Contains(FormatEndpointKey(ep)) && + !incomingKeys.Contains(FormatEndpointKey(ep)))); endpoints = ZeroTierDirectEndpointSelection.Normalize( merged, From 0465e64387b89d986c9f967f1159b2304a12898c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:14:52 +0100 Subject: [PATCH 220/296] Narrow direct-only probing to one hinted endpoint --- .../ZeroTierDataplaneRuntimeDirectPathTests.cs | 4 ++-- .../ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 8d44e6b..491edc3 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -364,9 +364,9 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceKeepsHintedEchoOnSelect .Where(send => send.Verb == ZeroTierVerb.Echo) .ToArray(); - Assert.Equal(hintedEndpoints.Length, echoFanout.Length); + Assert.Single(echoFanout); Assert.Equal(new[] { 0 }, echoFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); - Assert.Equal(hintedEndpoints, echoFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); + Assert.Equal(new[] { hintedEndpoints[0] }, echoFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Hello); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index fd1f8e4..b519da5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -862,7 +862,9 @@ private async Task ProbeHintedDirectEndpointsAsync( return; } - var endpointsToProbe = _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); + var endpointsToProbe = ShouldUseStickyHintedPathSelection(peerNodeId) + ? [hinted[0]] + : _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) @@ -1983,10 +1985,12 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer return Array.Empty(); } - var selectedEndpoints = _directHintPlanner.TakeNextHintedEndpoints( - peerNodeId, - hinted, - DirectHintMaintenanceProbeBudget); + var selectedEndpoints = ShouldUseStickyHintedPathSelection(peerNodeId) + ? [hinted[0]] + : _directHintPlanner.TakeNextHintedEndpoints( + peerNodeId, + hinted, + DirectHintMaintenanceProbeBudget); var candidates = new List(selectedEndpoints.Length); for (var i = 0; i < selectedEndpoints.Length; i++) { From 2c8d4d5129f36f9ff932e5c4bba9dc1355f2fd55 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:25:34 +0100 Subject: [PATCH 221/296] Prefer rendezvous endpoints over pushed hints --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 57 +++++++++ ...TierDirectEndpointManagerPushFlagsTests.cs | 27 ++++ .../Internal/ZeroTierDataplaneRuntime.cs | 3 + .../Internal/ZeroTierDirectEndpointKey.cs | 19 +++ .../Internal/ZeroTierDirectEndpointManager.cs | 118 ++++-------------- .../ZeroTierDirectHolePunchLimiter.cs | 95 ++++++++++++++ 6 files changed, 228 insertions(+), 91 deletions(-) create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointKey.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectHolePunchLimiter.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 491edc3..ed7edcb 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -163,6 +163,63 @@ public async Task DataplaneRuntime_DirectOnly_CanSelectHintedPath_WithoutConfirm Assert.Equal(0, selected.LocalSocketId); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_PrefersRendezvousEndpoint_AfterLaterPushDirectPaths() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(pushedEndpoint)), + TimeSpan.FromSeconds(2)); + + var endpoints = runtime.GetDirectEndpointsForTests(peerIdentity.NodeId); + Assert.Equal(rendezvousEndpoint, endpoints[0]); + Assert.Contains(endpoints, endpoint => endpoint.Equals(pushedEndpoint)); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_KeepsHintedPathSticky_PerFlow_BeforeConfirmation() { diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index e44e178..0479283 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -128,6 +128,33 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(newEndpoint)); } + [Fact] + public async Task PushDirectPaths_DoesNotDemote_RendezvousEndpoint() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + + await manager.HandleRendezvousFromRootAsync( + ZeroTierRendezvousCodec.BuildPayload(peerNodeId, rendezvousEndpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(pushedEndpoint, flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.Equal(rendezvousEndpoint, manager.Endpoints[0]); + Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(pushedEndpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(rendezvousEndpoint)); + } + [Fact] public async Task PushDirectPaths_NormalHint_UsesReceivingSocketFirst() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index b519da5..19b3223 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -2171,6 +2171,9 @@ private void CleanupDirectEndpointManagers(long nowMs) internal ZeroTierSelectedPeerPath[] GetHintedDirectCandidatesForMaintenance(NodeId peerNodeId) => _directHintPlanner.GetHintedCandidates(peerNodeId, _peerPaths.GetSnapshot(peerNodeId)); + internal IPEndPoint[] GetDirectEndpointsForTests(NodeId peerNodeId) + => GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + internal Task RunMultipathMaintenanceOnceForTestsAsync(CancellationToken cancellationToken = default) => RunMultipathMaintenanceOnceAsync(cancellationToken); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointKey.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointKey.cs new file mode 100644 index 0000000..8b6cf96 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointKey.cs @@ -0,0 +1,19 @@ +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierDirectEndpointKey +{ + public static string Format(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var address = endpoint.Address; + if (address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + return $"{address}:{endpoint.Port}"; + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 1cf2355..7aa7b54 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -14,9 +14,6 @@ internal sealed class ZeroTierDirectEndpointManager private const int MaxEndpoints = 64; private const int MaxEndpointsPerScopeAndFamily = 8; private const int RendezvousHolePunchHopLimit = 2; - private const long HolePunchMinIntervalMs = 5_000; - private const long HolePunchCacheTtlMs = 60_000; - private const int HolePunchCacheMaxEntries = 2048; private const long PushDirectPathsCutoffTimeMs = 30_000; private const int PushDirectPathsCutoffLimit = 8; @@ -29,11 +26,11 @@ internal sealed class ZeroTierDirectEndpointManager private readonly Func? _handleDirectEndpointHintAsync; private readonly Func? _shouldAcceptEndpoint; private readonly object _lock = new(); + private readonly ZeroTierDirectHolePunchLimiter _holePunchLimiter = new(); private IPEndPoint[] _directEndpoints = Array.Empty(); private readonly Dictionary> _preferredLocalSocketsByEndpoint = new(StringComparer.Ordinal); - private readonly ConcurrentDictionary _holePunchLastSentMs = new(StringComparer.Ordinal); - private long _lastHolePunchCleanupMs; + private readonly HashSet _pinnedRendezvousEndpointKeys = new(StringComparer.Ordinal); private long _lastDirectPathPushReceiveMs; private int _directPathPushCutoffCount; @@ -108,6 +105,7 @@ public async ValueTask HandleRendezvousFromRootAsync( lock (_lock) { _directEndpoints = endpoints; + ReplacePinnedRendezvousEndpoints_NoLock(endpoints); RememberPreferredLocalSockets_NoLock(endpoints, receivedLocalSocketId); } @@ -203,11 +201,16 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( .Concat(add) .Select(FormatEndpointKey) .ToHashSet(StringComparer.Ordinal); - var merged = redirect + var pinnedRendezvous = _directEndpoints.Where(endpoint => + _pinnedRendezvousEndpointKeys.Contains(FormatEndpointKey(endpoint)) && + !forget.Contains(FormatEndpointKey(endpoint))); + var merged = pinnedRendezvous + .Concat(redirect) .Concat(add) .Concat(_directEndpoints.Where(ep => !forget.Contains(FormatEndpointKey(ep)) && - !incomingKeys.Contains(FormatEndpointKey(ep)))); + !incomingKeys.Contains(FormatEndpointKey(ep)) && + !_pinnedRendezvousEndpointKeys.Contains(FormatEndpointKey(ep)))); endpoints = ZeroTierDirectEndpointSelection.Normalize( merged, @@ -219,6 +222,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( .Where(endpoint => forceFullHelloByEndpoint.ContainsKey(FormatEndpointKey(endpoint))) .ToArray(); _directEndpoints = endpoints; + PrunePinnedRendezvousEndpoints_NoLock(endpoints); PrunePreferredLocalSockets_NoLock(endpoints); RememberPreferredLocalSockets_NoLock(endpointsToProbe, receivedLocalSocketId); } @@ -334,7 +338,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId { var localSockets = _udp.LocalSockets; var now = Environment.TickCount64; - CleanupHolePunchCacheIfNeeded(now); + _holePunchLimiter.CleanupIfNeeded(now); var junk = new byte[4]; RandomNumberGenerator.Fill(junk); @@ -344,7 +348,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId for (var i = 0; i < preferredLocalSocketIds.Length; i++) { var socketId = preferredLocalSocketIds[i]; - if (!ShouldSendHolePunch(socketId, endpoint, now)) + if (!_holePunchLimiter.ShouldSend(socketId, endpoint, now)) { continue; } @@ -357,7 +361,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId if (localSockets.Count == 0) { - if (!ShouldSendHolePunch(localSocketId: 0, endpoint, now)) + if (!_holePunchLimiter.ShouldSend(localSocketId: 0, endpoint, now)) { return; } @@ -369,7 +373,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId for (var i = 0; i < localSockets.Count; i++) { var socketId = localSockets[i].Id; - if (!ShouldSendHolePunch(socketId, endpoint, now)) + if (!_holePunchLimiter.ShouldSend(socketId, endpoint, now)) { continue; } @@ -400,80 +404,12 @@ private void TrySendHolePunchCore(int localSocketId, IPEndPoint endpoint, byte[] TaskScheduler.Default); } - private bool ShouldSendHolePunch(int localSocketId, IPEndPoint endpoint, long nowMs) + private void ReplacePinnedRendezvousEndpoints_NoLock(IEnumerable endpoints) { - var keyAddress = endpoint.Address; - if (keyAddress.AddressFamily == AddressFamily.InterNetworkV6 && keyAddress.IsIPv4MappedToIPv6) - { - keyAddress = keyAddress.MapToIPv4(); - } - - var key = $"{localSocketId}|{keyAddress}:{endpoint.Port}"; - - while (true) - { - if (_holePunchLastSentMs.TryGetValue(key, out var lastSent) && - unchecked(nowMs - lastSent) < HolePunchMinIntervalMs) - { - return false; - } - - if (_holePunchLastSentMs.TryAdd(key, nowMs)) - { - return true; - } - - _holePunchLastSentMs.TryGetValue(key, out lastSent); - if (unchecked(nowMs - lastSent) < HolePunchMinIntervalMs) - { - return false; - } - - if (_holePunchLastSentMs.TryUpdate(key, nowMs, lastSent)) - { - return true; - } - } - } - - private void CleanupHolePunchCacheIfNeeded(long nowMs) - { - if (_holePunchLastSentMs.Count <= HolePunchCacheMaxEntries) - { - return; - } - - var last = Volatile.Read(ref _lastHolePunchCleanupMs); - if (last != 0 && unchecked(nowMs - last) < 10_000) - { - return; - } - - if (Interlocked.CompareExchange(ref _lastHolePunchCleanupMs, nowMs, last) != last) - { - return; - } - - var cutoff = nowMs - HolePunchCacheTtlMs; - foreach (var entry in _holePunchLastSentMs) - { - if (entry.Value <= cutoff) - { - _holePunchLastSentMs.TryRemove(entry.Key, out _); - } - } - - if (_holePunchLastSentMs.Count <= HolePunchCacheMaxEntries) - { - return; - } - - var snapshot = _holePunchLastSentMs.ToArray(); - Array.Sort(snapshot, static (left, right) => left.Value.CompareTo(right.Value)); - - for (var i = 0; i < snapshot.Length && _holePunchLastSentMs.Count > HolePunchCacheMaxEntries; i++) + _pinnedRendezvousEndpointKeys.Clear(); + foreach (var endpoint in endpoints) { - _holePunchLastSentMs.TryRemove(snapshot[i].Key, out _); + _pinnedRendezvousEndpointKeys.Add(FormatEndpointKey(endpoint)); } } @@ -507,15 +443,15 @@ private void PrunePreferredLocalSockets_NoLock(IEnumerable endpoints } } - private static string FormatEndpointKey(IPEndPoint endpoint) + private void PrunePinnedRendezvousEndpoints_NoLock(IEnumerable endpoints) { - var address = endpoint.Address; - if (address.IsIPv4MappedToIPv6) - { - address = address.MapToIPv4(); - } - - return $"{address}:{endpoint.Port}"; + var keep = endpoints + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); + _pinnedRendezvousEndpointKeys.RemoveWhere(key => !keep.Contains(key)); } + + private static string FormatEndpointKey(IPEndPoint endpoint) + => ZeroTierDirectEndpointKey.Format(endpoint); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHolePunchLimiter.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHolePunchLimiter.cs new file mode 100644 index 0000000..0d02cc7 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHolePunchLimiter.cs @@ -0,0 +1,95 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectHolePunchLimiter +{ + private const long MinIntervalMs = 5_000; + private const long CacheTtlMs = 60_000; + private const int CacheMaxEntries = 2048; + + private readonly ConcurrentDictionary _lastSentMs = new(StringComparer.Ordinal); + private long _lastCleanupMs; + + public bool ShouldSend(int localSocketId, IPEndPoint endpoint, long nowMs) + { + var key = $"{localSocketId}|{ZeroTierDirectEndpointKey.Format(Normalize(endpoint))}"; + while (true) + { + if (_lastSentMs.TryGetValue(key, out var lastSent) && + unchecked(nowMs - lastSent) < MinIntervalMs) + { + return false; + } + + if (_lastSentMs.TryAdd(key, nowMs)) + { + return true; + } + + _lastSentMs.TryGetValue(key, out lastSent); + if (unchecked(nowMs - lastSent) < MinIntervalMs) + { + return false; + } + + if (_lastSentMs.TryUpdate(key, nowMs, lastSent)) + { + return true; + } + } + } + + public void CleanupIfNeeded(long nowMs) + { + if (_lastSentMs.Count <= CacheMaxEntries) + { + return; + } + + var last = Volatile.Read(ref _lastCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 10_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastCleanupMs, nowMs, last) != last) + { + return; + } + + var cutoff = nowMs - CacheTtlMs; + foreach (var entry in _lastSentMs) + { + if (entry.Value <= cutoff) + { + _lastSentMs.TryRemove(entry.Key, out _); + } + } + + if (_lastSentMs.Count <= CacheMaxEntries) + { + return; + } + + var snapshot = _lastSentMs.ToArray(); + Array.Sort(snapshot, static (left, right) => left.Value.CompareTo(right.Value)); + for (var i = 0; i < snapshot.Length && _lastSentMs.Count > CacheMaxEntries; i++) + { + _lastSentMs.TryRemove(snapshot[i].Key, out _); + } + } + + private static IPEndPoint Normalize(IPEndPoint endpoint) + { + var address = endpoint.Address; + if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + return ReferenceEquals(address, endpoint.Address) ? endpoint : new IPEndPoint(address, endpoint.Port); + } +} From f732e56232df789693b40f199f964209c3e78937 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:33:38 +0100 Subject: [PATCH 222/296] Advertise local surfaces in direct HELLO --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 78 ++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 82 +++++-------------- ...eroTierDirectHelloAdvertisementSelector.cs | 38 +++++++++ 3 files changed, 137 insertions(+), 61 deletions(-) create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index ed7edcb..7cdb643 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -106,6 +106,65 @@ await WaitForConditionAsync( Assert.Equal(1, firstBootstrapSend.LocalSocketId); } + [Fact] + public async Task DataplaneRuntime_RendezvousBootstrap_HelloAdvertisesLocalSurface() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var advertisedLocalSurface = new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(1, advertisedLocalSurface) + ]); + + runtime.PrimePeerForTests(peerIdentity, peerProtocolVersion: 4); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + var helloSend = await WaitForSendAsync( + udp, + send => send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello, + TimeSpan.FromSeconds(2)); + + Assert.Equal(advertisedLocalSurface, ReadAdvertisedHelloSurface(helloSend.Payload, sharedKey)); + } + [Fact] public async Task DataplaneRuntime_SeededHintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() { @@ -651,6 +710,25 @@ private static bool TryDecodeVerb(byte[] packet, byte[] sharedKey, out ZeroTierV return true; } + private static IPEndPoint? ReadAdvertisedHelloSurface(byte[] packet, byte[] sharedKey) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed)) + { + throw new InvalidOperationException("Failed to decode HELLO packet."); + } + + var payload = parsed.Payload.Span; + ZeroTierIdentityCodec.Deserialize(payload.Slice(13), out var identityLength); + if (!ZeroTierInetAddressCodec.TryDeserialize(payload.Slice(13 + identityLength), out var surface, out _)) + { + throw new InvalidOperationException("Failed to decode advertised HELLO surface."); + } + + return surface; + } + private static byte[] BuildPushDirectPathsPacket( NodeId source, NodeId destination, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 19b3223..93c1075 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1274,6 +1274,12 @@ private Task SendDirectBootstrapProbeAsync( cancellationToken) : TrySendEchoDirectProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken); + private IPEndPoint? GetAdvertisedDirectHelloSurface(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + => ZeroTierDirectHelloAdvertisementSelector.Select( + remoteEndPoint, + _surfaceAddresses.GetSnapshot(localSocketId), + GetPeerAwareLocalDirectPathAdvertisements(peerNodeId)); + private async Task TrySendEchoDirectProbeAsync( NodeId peerNodeId, int localSocketId, @@ -1315,17 +1321,21 @@ private async Task SendHelloPacketAsync( return; } + var advertisedLocalSurface = physicalDestination is null + ? null + : GetAdvertisedDirectHelloSurface(peerNodeId, localSocketId, sendTo); + try { if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, advertised={physicalDestination})."); + ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, advertised={advertisedLocalSurface})."); } var packet = ZeroTierHelloPacketBuilder.BuildPacket( _localIdentity, peerNodeId, - physicalDestination, + advertisedLocalSurface, timestamp: (ulong)Environment.TickCount64, _planetId, _planetTimestamp, @@ -1383,67 +1393,17 @@ private async Task SendHelloPacketAcrossSocketsAsync( { ArgumentNullException.ThrowIfNull(localSocketIds); - var eligibleSocketIds = new List(localSocketIds.Length); for (var i = 0; i < localSocketIds.Length; i++) { - if (physicalDestination is not null && - !ShouldSendDirectHello(peerNodeId, localSocketIds[i], sendTo, minIntervalMs)) - { - continue; - } - - eligibleSocketIds.Add(localSocketIds[i]); - } - - if (eligibleSocketIds.Count == 0) - { - return; - } - - try - { - var sentAtMs = Environment.TickCount64; - var packet = ZeroTierHelloPacketBuilder.BuildPacket( - _localIdentity, - peerNodeId, - physicalDestination, - timestamp: (ulong)sentAtMs, - _planetId, - _planetTimestamp, - sharedKey, - ZeroTierHelloClient.AdvertisedProtocolVersion, - ZeroTierHelloClient.AdvertisedMajorVersion, - ZeroTierHelloClient.AdvertisedMinorVersion, - ZeroTierHelloClient.AdvertisedRevision, - out var packetId); - - TrackPendingHello( - packetId, - eligibleSocketIds - .Select(socketId => new PendingHelloProbe( - PeerNodeId: peerNodeId, - LocalSocketId: socketId, - SendTo: sendTo, - PhysicalDestination: physicalDestination, - SentAtMs: sentAtMs)) - .ToArray()); - - for (var i = 0; i < eligibleSocketIds.Count; i++) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={eligibleSocketIds[i]}, advertised={physicalDestination})."); - } - - await _udp.SendAsync(eligibleSocketIds[i], sendTo, packet, cancellationToken).ConfigureAwait(false); - } - } - catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] HELLO bootstrap send failed to {sendTo}: {ex.GetType().Name}: {ex.Message}"); - } + await SendHelloPacketAsync( + localSocketIds[i], + peerNodeId, + physicalDestination, + sendTo, + sharedKey, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs new file mode 100644 index 0000000..35adc95 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs @@ -0,0 +1,38 @@ +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierDirectHelloAdvertisementSelector +{ + public static IPEndPoint? Select( + IPEndPoint remoteEndPoint, + IReadOnlyList localSocketSurfaces, + IReadOnlyList peerAwareAdvertisements) + { + ArgumentNullException.ThrowIfNull(remoteEndPoint); + ArgumentNullException.ThrowIfNull(localSocketSurfaces); + ArgumentNullException.ThrowIfNull(peerAwareAdvertisements); + + var family = remoteEndPoint.AddressFamily; + return SelectMatchingFamily(localSocketSurfaces, family) + ?? SelectMatchingFamily(peerAwareAdvertisements, family) + ?? SelectFirst(localSocketSurfaces) + ?? SelectFirst(peerAwareAdvertisements); + } + + private static IPEndPoint? SelectMatchingFamily(IReadOnlyList endpoints, System.Net.Sockets.AddressFamily family) + { + for (var i = 0; i < endpoints.Count; i++) + { + if (endpoints[i].AddressFamily == family) + { + return endpoints[i]; + } + } + + return null; + } + + private static IPEndPoint? SelectFirst(IReadOnlyList endpoints) + => endpoints.Count == 0 ? null : endpoints[0]; +} From 23664ad83bcd27ba7ed48d8f6c40cc71e48314f4 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:37:54 +0100 Subject: [PATCH 223/296] Relay public rendezvous surfaces via root --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 67 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 52 ++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 7cdb643..31969cd 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -569,6 +569,59 @@ public async Task DataplaneRuntime_SendViaRootSockets_FansOutAcrossLocalSockets( Assert.Equal(new[] { 0, 1 }, socketIds); } + [Fact] + public async Task DataplaneRuntime_SendPublicRendezvousViaRoot_AdvertisesPublicSurfaces() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + ]); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + await runtime.SendPublicRendezvousViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var endpoints = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Select(send => ReadRendezvousEndpoint(send.Payload, sharedKey)) + .Where(static endpoint => endpoint is not null) + .Cast() + .Distinct() + .OrderBy(endpoint => endpoint.Port) + .ToArray(); + + Assert.Equal( + new[] + { + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001) + }, + endpoints); + } + [Fact] public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() { @@ -729,6 +782,20 @@ private static bool TryDecodeVerb(byte[] packet, byte[] sharedKey, out ZeroTierV return surface; } + private static IPEndPoint? ReadRendezvousEndpoint(byte[] packet, byte[] sharedKey) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed) || + parsed.Header.Verb != ZeroTierVerb.Rendezvous || + !ZeroTierRendezvousCodec.TryParse(parsed.Payload.Span, out var rendezvous)) + { + return null; + } + + return rendezvous.Endpoint; + } + private static byte[] BuildPushDirectPathsPacket( NodeId source, NodeId destination, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 93c1075..47a3ba2 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -802,6 +802,7 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } @@ -849,6 +850,7 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -1032,6 +1034,53 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } + private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) + .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) + .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) + .ToArray(); + if (advertisements.Length == 0) + { + return; + } + + try + { + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + for (var i = 0; i < advertisements.Length; i++) + { + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, advertisements[i])); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisements[i]}."); + } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { var localAdvertisements = GetLocalDirectPathAdvertisements(); @@ -2158,6 +2207,9 @@ internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId internal Task SendHelloViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) => SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken); + internal Task SendPublicRendezvousViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) + => SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken); + internal UserSpaceTcpConnectOptions GetTcpConnectOptionsForTests() => GetSuggestedTcpConnectOptions(); From 2d92dc1c6db23b6c4c92b46c3b8fa49a1d1c95fd Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:41:18 +0100 Subject: [PATCH 224/296] Force HELLO on root rendezvous bootstrap --- ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs | 2 +- .../ZeroTierDirectEndpointManagerSocketAffinityTests.cs | 2 +- ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 31969cd..3cd04b3 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -99,7 +99,7 @@ await WaitForConditionAsync( var firstBootstrapSend = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) .OrderBy(send => send.LocalSocketId) .First(); diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index e96130e..40f4d60 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -55,7 +55,7 @@ await manager.HandleRendezvousFromRootAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, false, 1), hints[0]); + Assert.Equal((endpoint, true, false, 1), hints[0]); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 7aa7b54..f8011c0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -121,7 +121,7 @@ await _handleDirectEndpointHintAsync( _remoteNodeId, receivedLocalSocketId, endpoint, - false, + true, false, cancellationToken) .ConfigureAwait(false); From d6c850c33546250a54d7cc1d6f037cc1cb76dcba Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:48:17 +0100 Subject: [PATCH 225/296] Restore upstream direct HELLO endpoint semantics --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 13 ++----- .../Internal/ZeroTierDataplaneRuntime.cs | 14 ++----- ...eroTierDirectHelloAdvertisementSelector.cs | 38 ------------------- 3 files changed, 7 insertions(+), 58 deletions(-) delete mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 3cd04b3..9bc4719 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -107,7 +107,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_RendezvousBootstrap_HelloAdvertisesLocalSurface() + public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -120,7 +120,6 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloAdvertisesLocalSurfa var rootNodeId = new NodeId(0x1111111111); var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); - var advertisedLocalSurface = new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001); var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime( @@ -130,11 +129,7 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloAdvertisesLocalSurfa rootKey, multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, planetId: 1, - planetTimestamp: 1, - initialExternalSurfaceObservations: - [ - new ZeroTierExternalSurfaceObservation(1, advertisedLocalSurface) - ]); + planetTimestamp: 1); runtime.PrimePeerForTests(peerIdentity, peerProtocolVersion: 4); @@ -162,7 +157,7 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloAdvertisesLocalSurfa verb == ZeroTierVerb.Hello, TimeSpan.FromSeconds(2)); - Assert.Equal(advertisedLocalSurface, ReadAdvertisedHelloSurface(helloSend.Payload, sharedKey)); + Assert.Equal(rendezvousEndpoint, ReadReportedHelloSurface(helloSend.Payload, sharedKey)); } [Fact] @@ -763,7 +758,7 @@ private static bool TryDecodeVerb(byte[] packet, byte[] sharedKey, out ZeroTierV return true; } - private static IPEndPoint? ReadAdvertisedHelloSurface(byte[] packet, byte[] sharedKey) + private static IPEndPoint? ReadReportedHelloSurface(byte[] packet, byte[] sharedKey) { var decoded = packet.ToArray(); if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 47a3ba2..eea8598 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1323,12 +1323,6 @@ private Task SendDirectBootstrapProbeAsync( cancellationToken) : TrySendEchoDirectProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken); - private IPEndPoint? GetAdvertisedDirectHelloSurface(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) - => ZeroTierDirectHelloAdvertisementSelector.Select( - remoteEndPoint, - _surfaceAddresses.GetSnapshot(localSocketId), - GetPeerAwareLocalDirectPathAdvertisements(peerNodeId)); - private async Task TrySendEchoDirectProbeAsync( NodeId peerNodeId, int localSocketId, @@ -1370,21 +1364,19 @@ private async Task SendHelloPacketAsync( return; } - var advertisedLocalSurface = physicalDestination is null - ? null - : GetAdvertisedDirectHelloSurface(peerNodeId, localSocketId, sendTo); + var reportedRemoteSurface = physicalDestination; try { if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, advertised={advertisedLocalSurface})."); + ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, reported={reportedRemoteSurface})."); } var packet = ZeroTierHelloPacketBuilder.BuildPacket( _localIdentity, peerNodeId, - advertisedLocalSurface, + reportedRemoteSurface, timestamp: (ulong)Environment.TickCount64, _planetId, _planetTimestamp, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs deleted file mode 100644 index 35adc95..0000000 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHelloAdvertisementSelector.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Net; - -namespace ZTSharp.ZeroTier.Internal; - -internal static class ZeroTierDirectHelloAdvertisementSelector -{ - public static IPEndPoint? Select( - IPEndPoint remoteEndPoint, - IReadOnlyList localSocketSurfaces, - IReadOnlyList peerAwareAdvertisements) - { - ArgumentNullException.ThrowIfNull(remoteEndPoint); - ArgumentNullException.ThrowIfNull(localSocketSurfaces); - ArgumentNullException.ThrowIfNull(peerAwareAdvertisements); - - var family = remoteEndPoint.AddressFamily; - return SelectMatchingFamily(localSocketSurfaces, family) - ?? SelectMatchingFamily(peerAwareAdvertisements, family) - ?? SelectFirst(localSocketSurfaces) - ?? SelectFirst(peerAwareAdvertisements); - } - - private static IPEndPoint? SelectMatchingFamily(IReadOnlyList endpoints, System.Net.Sockets.AddressFamily family) - { - for (var i = 0; i < endpoints.Count; i++) - { - if (endpoints[i].AddressFamily == family) - { - return endpoints[i]; - } - } - - return null; - } - - private static IPEndPoint? SelectFirst(IReadOnlyList endpoints) - => endpoints.Count == 0 ? null : endpoints[0]; -} From 2a75b5e5ea024f1b0d25bccd259d96f9901608d8 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:57:32 +0100 Subject: [PATCH 226/296] Gate direct path promotion on hop-0 OK --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 35 +++++++++++++++++++ .../ZeroTierPeerPhysicalPathTrackerTests.cs | 15 ++++++++ .../ZeroTierDataplanePeerDatagramProcessor.cs | 19 +++++++--- .../ZeroTierPeerPhysicalPathTracker.cs | 23 ++++++++++++ 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 9bc4719..9bf4870 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -189,6 +189,41 @@ public async Task DataplaneRuntime_SeededHintedEndpoints_AreVisibleToMaintenance Assert.Equal(0, candidate.LocalSocketId); } + [Fact] + public async Task DataplaneRuntime_DirectHello_DoesNotConfirmUnknownPath() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint)); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var runtimeEndpoint = TestUdpEndpoints.ToLoopback(udp.LocalEndpoint); + var helloPacket = BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, runtimeEndpoint); + + await rootUdp.SendAsync(runtimeEndpoint, helloPacket); + _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + + Assert.False(runtime.TrySelectDirectPathForTests(peerIdentity.NodeId, flowId: 0, out _)); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_CanSelectHintedPath_WithoutConfirmedHop0() { diff --git a/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs b/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs index 3c5d4a6..a7c7559 100644 --- a/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs @@ -37,4 +37,19 @@ public void ObserveHop0_TracksMultipleSocketsAndEndpoints() Assert.Contains(snapshot, p => p.LocalSocketId == 0 && p.RemoteEndPoint.Port == 10000); Assert.Contains(snapshot, p => p.LocalSocketId == 1 && p.RemoteEndPoint.Port == 10001); } + + [Fact] + public void ObserveKnownHop0_DoesNotPromoteUnknownPaths() + { + var tracker = new ZeroTierPeerPhysicalPathTracker(ttl: TimeSpan.FromSeconds(10)); + var peer = new NodeId(0x1111111111); + var path = new IPEndPoint(IPAddress.Loopback, 10000); + + Assert.False(tracker.ObserveKnownHop0(peer, localSocketId: 0, path)); + Assert.Empty(tracker.GetSnapshot(peer)); + + Assert.True(tracker.ObserveHop0(peer, localSocketId: 0, path)); + Assert.True(tracker.ObserveKnownHop0(peer, localSocketId: 0, path)); + Assert.Single(tracker.GetSnapshot(peer)); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index 68c023f..3dc399a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -94,7 +94,7 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c var observedNewInboundHop0Path = false; if (inboundHello is { } hello && decoded.Header.HopCount == 0) { - observedNewInboundHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + observedNewInboundHop0Path = !_peerPaths.ObserveKnownHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); if (hello.ReportedLocalSurfaceAddress is { } reportedSurface) { @@ -157,22 +157,31 @@ await _handleUnknownDirectPathAsync(peerNodeId, datagram.LocalSocketId, datagram } var observedNewHop0Path = false; + var hasKnownHop0Path = false; if (_multipathEnabled) { + var verb = (ZeroTierVerb)(packetBytes[ZeroTierPacketHeader.IndexVerb] & 0x1F); + if (decoded.Header.HopCount == 0) { - observedNewHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + if (verb == ZeroTierVerb.Ok) + { + observedNewHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + } + else + { + hasKnownHop0Path = _peerPaths.ObserveKnownHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + } + await _peerEcho .TrySendEchoProbeAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, key, cancellationToken) .ConfigureAwait(false); } - var verb = (ZeroTierVerb)(packetBytes[ZeroTierPacketHeader.IndexVerb] & 0x1F); - if (decoded.Header.HopCount == 0 && - observedNewHop0Path && verb != ZeroTierVerb.Ok && + !hasKnownHop0Path && _handleUnknownDirectPathAsync is not null) { await _handleUnknownDirectPathAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs index fec306a..3d9b39d 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs @@ -35,6 +35,29 @@ public bool ObserveHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteE return isNewPath; } + public bool ObserveKnownHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + { + ArgumentNullException.ThrowIfNull(remoteEndPoint); + + var now = _nowUnixMs(); + var peer = _peers.GetOrAdd(peerNodeId, _ => new PeerState()); + var key = new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint); + if (!peer.Paths.ContainsKey(key)) + { + CleanupIfNeeded(now); + if (peer.Paths.IsEmpty) + { + _peers.TryRemove(peerNodeId, out _); + } + + return false; + } + + peer.Paths[key] = now; + CleanupIfNeeded(now); + return true; + } + public ZeroTierPeerPhysicalPath[] GetSnapshot(NodeId peerNodeId) { var now = _nowUnixMs(); From 27eed340cc40ef59f2db4baf61108b31ebd1191a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:59:47 +0100 Subject: [PATCH 227/296] Remove peer relayed rendezvous bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 67 ------------------- .../Internal/ZeroTierDataplaneRuntime.cs | 52 -------------- 2 files changed, 119 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 9bf4870..377a8c1 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -599,59 +599,6 @@ public async Task DataplaneRuntime_SendViaRootSockets_FansOutAcrossLocalSockets( Assert.Equal(new[] { 0, 1 }, socketIds); } - [Fact] - public async Task DataplaneRuntime_SendPublicRendezvousViaRoot_AdvertisesPublicSurfaces() - { - await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); - Assert.True(peerIdentity.HasPrivateKey); - Assert.True(peerIdentity.LocallyValidate()); - - var rootNodeId = new NodeId(0x1111111111); - var rootKey = RandomNumberGenerator.GetBytes(48); - await using var runtime = CreateRuntime( - udp, - localIdentity, - rootNodeId, - rootKey, - multipath: new ZeroTierMultipathOptions { Enabled = true }, - planetId: 1, - planetTimestamp: 1, - initialExternalSurfaceObservations: - [ - new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), - new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) - ]); - - runtime.PrimePeerForTests(peerIdentity); - - var sharedKey = new byte[48]; - ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); - - await runtime.SendPublicRendezvousViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); - - var endpoints = udp.GetSendsSnapshot() - .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) - .Select(send => ReadRendezvousEndpoint(send.Payload, sharedKey)) - .Where(static endpoint => endpoint is not null) - .Cast() - .Distinct() - .OrderBy(endpoint => endpoint.Port) - .ToArray(); - - Assert.Equal( - new[] - { - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001) - }, - endpoints); - } - [Fact] public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() { @@ -812,20 +759,6 @@ private static bool TryDecodeVerb(byte[] packet, byte[] sharedKey, out ZeroTierV return surface; } - private static IPEndPoint? ReadRendezvousEndpoint(byte[] packet, byte[] sharedKey) - { - var decoded = packet.ToArray(); - if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || - !ZeroTierPacketCodec.TryDecode(decoded, out var parsed) || - parsed.Header.Verb != ZeroTierVerb.Rendezvous || - !ZeroTierRendezvousCodec.TryParse(parsed.Payload.Span, out var rendezvous)) - { - return null; - } - - return rendezvous.Endpoint; - } - private static byte[] BuildPushDirectPathsPacket( NodeId source, NodeId destination, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index eea8598..5848e68 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -802,7 +802,6 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } @@ -850,7 +849,6 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -1034,53 +1032,6 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } - private async Task SendPublicRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) - { - var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) - .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) - .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) - .ToArray(); - if (advertisements.Length == 0) - { - return; - } - - try - { - var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - for (var i = 0; i < advertisements.Length; i++) - { - var packet = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Rendezvous), - ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, advertisements[i])); - ZeroTierPacketCrypto.Armor( - packet, - ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), - encryptPayload: true); - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisements[i]}."); - } - - await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); - } - } - catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); - } - } - } - private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { var localAdvertisements = GetLocalDirectPathAdvertisements(); @@ -2199,9 +2150,6 @@ internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId internal Task SendHelloViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) => SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken); - internal Task SendPublicRendezvousViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) - => SendPublicRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken); - internal UserSpaceTcpConnectOptions GetTcpConnectOptionsForTests() => GetSuggestedTcpConnectOptions(); From f8fab8e2707619f3dbaf6b194d6eaad6f26b0800 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:03:22 +0100 Subject: [PATCH 228/296] Stop pinning ordinary pushed direct paths to one socket --- .../ZeroTierDirectEndpointManagerPushFlagsTests.cs | 10 +++++----- ...ZeroTierDirectEndpointManagerSocketAffinityTests.cs | 4 ++-- .../ZeroTier/Internal/ZeroTierDirectEndpointManager.cs | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 0479283..9b89482 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -55,7 +55,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_KnownEndpoint_IsRebootstrappedAndUpdatesSocketAffinity() + public async Task PushDirectPaths_KnownEndpoint_IsRebootstrappedWithoutPersistingSocketAffinity() { var udp = new RecordingUdpTransport(); @@ -79,12 +79,12 @@ public async Task PushDirectPaths_KnownEndpoint_IsRebootstrappedAndUpdatesSocket await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 1, CancellationToken.None); Assert.Equal(new[] { endpoint, endpoint }, hintedEndpoints); - Assert.Equal(new[] { 0, 1 }, manager.GetPreferredLocalSocketIds(endpoint).Order().ToArray()); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } [Fact] - public async Task PushDirectPaths_NewEndpoint_RemembersReceivingSocketAffinityWithoutHolePunch() + public async Task PushDirectPaths_NewEndpoint_DoesNotPersistReceivingSocketAffinityOrHolePunch() { var udp = new RecordingUdpTransport(); @@ -98,7 +98,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } @@ -125,7 +125,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Equal(newEndpoint, manager.Endpoints[0]); Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(oldEndpoint)); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(newEndpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(newEndpoint)); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index 40f4d60..7d700c5 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -59,7 +59,7 @@ await manager.HandleRendezvousFromRootAsync( } [Fact] - public async Task PushDirectPaths_UsesReceivingLocalSocket_ForNewEndpoints() + public async Task PushDirectPaths_DoesNotPersistReceivingSocketAffinity_ForNewEndpoints() { await using var udp = new RecordingUdpTransport(); var relay = new IPEndPoint(IPAddress.Loopback, 9999); @@ -72,7 +72,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index f8011c0..9bfc50b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -162,6 +162,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( var forget = new HashSet(StringComparer.Ordinal); var redirect = new List(); var add = new List(); + var preferredSocketEndpoints = new List(); var forceFullHelloByEndpoint = new Dictionary(StringComparer.Ordinal); var useAllEligibleLocalSocketsByEndpoint = new Dictionary(StringComparer.Ordinal); @@ -182,6 +183,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( if ((flags & PushDirectPathsFlagClusterRedirect) != 0) { redirect.Add(endpoint); + preferredSocketEndpoints.Add(endpoint); forceFullHelloByEndpoint[key] = true; useAllEligibleLocalSocketsByEndpoint[key] = false; } @@ -224,7 +226,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( _directEndpoints = endpoints; PrunePinnedRendezvousEndpoints_NoLock(endpoints); PrunePreferredLocalSockets_NoLock(endpoints); - RememberPreferredLocalSockets_NoLock(endpointsToProbe, receivedLocalSocketId); + RememberPreferredLocalSockets_NoLock(preferredSocketEndpoints, receivedLocalSocketId); } if (endpoints.Length == 0 || endpointsToProbe.Length == 0) From 546c1af20b63022e0703c5a19d901a03b5c44198 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:06:25 +0100 Subject: [PATCH 229/296] Retry sticky direct-only hints with periodic HELLO --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 14 +++++++------- .../Internal/ZeroTierDataplaneRuntime.cs | 19 +++++++++++++++---- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 377a8c1..155af3f 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -460,7 +460,7 @@ public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceKeepsHintedEchoOnSelectedSocket_ForModernPeers() + public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHelloOnSelectedSocket_ForModernPeers() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -506,14 +506,14 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceKeepsHintedEchoOnSelect }) .ToArray(); - var echoFanout = hintedSends - .Where(send => send.Verb == ZeroTierVerb.Echo) + var helloFanout = hintedSends + .Where(send => send.Verb == ZeroTierVerb.Hello) .ToArray(); - Assert.Single(echoFanout); - Assert.Equal(new[] { 0 }, echoFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); - Assert.Equal(new[] { hintedEndpoints[0] }, echoFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); - Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Hello); + Assert.Single(helloFanout); + Assert.Equal(new[] { 0 }, helloFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); + Assert.Equal(new[] { hintedEndpoints[0] }, helloFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); + Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Echo); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 5848e68..49f1b97 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -867,6 +867,10 @@ private async Task ProbeHintedDirectEndpointsAsync( : _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { + var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId); + var minIntervalMs = forceFullHello + ? DirectOnlyHintHelloIntervalMs + : DirectHelloMinIntervalMs; var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) ? GetStickyHintedLocalSocketIds(peerNodeId, endpointsToProbe[i]) : _directHintPlanner.GetRotatingSocketIds( @@ -880,8 +884,8 @@ await SendDirectBootstrapProbeAsync( localSocketIds[s], endpointsToProbe[i], sharedKey, - forceFullHello: false, - DirectHelloMinIntervalMs, + forceFullHello, + minIntervalMs, cancellationToken) .ConfigureAwait(false); } @@ -1710,6 +1714,10 @@ private bool ShouldForceFullHelloForHintedCandidate(NodeId peerNodeId) => ShouldFanOutHintedBootstrapAcrossSockets(peerNodeId) && !CanUseEchoForDirectBootstrap(peerNodeId); + private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId) + => ShouldUseStickyHintedPathSelection(peerNodeId) && + CanUseEchoForDirectBootstrap(peerNodeId); + private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) @@ -1765,9 +1773,12 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati for (var c = 0; c < hintedCandidates.Length; c++) { var candidate = hintedCandidates[c]; - var forceFullHello = ShouldForceFullHelloForHintedCandidate(peerNodeId); + var forceFullHello = ShouldForceFullHelloForHintedCandidate(peerNodeId) || + ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId); var minIntervalMs = forceFullHello - ? DirectOnlyHintHelloIntervalMs + ? (ShouldUseStickyHintedPathSelection(peerNodeId) + ? DirectOnlyHintHelloIntervalMs + : DirectHintFullHelloIntervalMs) : DirectHintFullHelloIntervalMs; int[] localSocketIds = [candidate.LocalSocketId]; From c96fc7cabbe7ca584de87588c6c3c0fd54cd7394 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:09:14 +0100 Subject: [PATCH 230/296] Fan out periodic rendezvous HELLO retries --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 5 ++-- .../Internal/ZeroTierDataplaneRuntime.cs | 23 +++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 155af3f..ac39989 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -460,7 +460,7 @@ public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHelloOnSelectedSocket_ForModernPeers() + public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHelloAcrossSockets_ForModernPeers() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -510,8 +510,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHello .Where(send => send.Verb == ZeroTierVerb.Hello) .ToArray(); - Assert.Single(helloFanout); - Assert.Equal(new[] { 0 }, helloFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); + Assert.Equal(new[] { 0, 1 }, helloFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); Assert.Equal(new[] { hintedEndpoints[0] }, helloFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Echo); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 49f1b97..96c223e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -872,11 +872,28 @@ private async Task ProbeHintedDirectEndpointsAsync( ? DirectOnlyHintHelloIntervalMs : DirectHelloMinIntervalMs; var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) - ? GetStickyHintedLocalSocketIds(peerNodeId, endpointsToProbe[i]) + ? (forceFullHello + ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpointsToProbe[i]) + : GetStickyHintedLocalSocketIds(peerNodeId, endpointsToProbe[i])) : _directHintPlanner.GetRotatingSocketIds( peerNodeId, endpointsToProbe[i], includeFallbackLocalSockets: true); + + if (forceFullHello && localSocketIds.Length > 1) + { + await SendHelloPacketAcrossSocketsAsync( + localSocketIds, + peerNodeId, + endpointsToProbe[i], + endpointsToProbe[i], + sharedKey, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + continue; + } + for (var s = 0; s < localSocketIds.Length; s++) { await SendDirectBootstrapProbeAsync( @@ -1780,7 +1797,9 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati ? DirectOnlyHintHelloIntervalMs : DirectHintFullHelloIntervalMs) : DirectHintFullHelloIntervalMs; - int[] localSocketIds = [candidate.LocalSocketId]; + var localSocketIds = forceFullHello && ShouldUseStickyHintedPathSelection(peerNodeId) + ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, candidate.RemoteEndPoint) + : [candidate.LocalSocketId]; if (forceFullHello && localSocketIds.Length > 1) { From 15f83a70c7dc14359a2e2ade20b31a442c76c9a8 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:24:02 +0100 Subject: [PATCH 231/296] Gate direct-only payload on confirmed paths --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 90 ++++------------- .../Internal/ZeroTierDataplaneRuntime.cs | 96 ++++++++++++------- 2 files changed, 80 insertions(+), 106 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index ac39989..a6b1352 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -225,7 +225,7 @@ public async Task DataplaneRuntime_DirectHello_DoesNotConfirmUnknownPath() } [Fact] - public async Task DataplaneRuntime_DirectOnly_CanSelectHintedPath_WithoutConfirmedHop0() + public async Task DataplaneRuntime_DirectOnly_ExposesHintedPathCandidate_WithoutConfirmedHop0() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); @@ -310,72 +310,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_KeepsHintedPathSticky_PerFlow_BeforeConfirmation() - { - await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - var rootNodeId = new NodeId(0x1111111111); - var peerNodeId = new NodeId(0x3333333333); - var hintedEndpoints = new[] - { - new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), - new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) - }; - var rootKey = RandomNumberGenerator.GetBytes(48); - - await using var runtime = CreateRuntime( - udp, - localIdentity, - rootNodeId, - rootKey, - multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, - planetId: 1, - planetTimestamp: 1); - - runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoints); - - Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var first)); - Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var second)); - Assert.Equal(first, second); - } - - [Fact] - public async Task DataplaneRuntime_DirectOnly_PrefersFirstHintedPath_BeforeConfirmation() - { - await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); - - var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); - var rootNodeId = new NodeId(0x1111111111); - var peerNodeId = new NodeId(0x3333333333); - var hintedEndpoints = new[] - { - new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), - new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) - }; - var rootKey = RandomNumberGenerator.GetBytes(48); - - await using var runtime = CreateRuntime( - udp, - localIdentity, - rootNodeId, - rootKey, - multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, - planetId: 1, - planetTimestamp: 1); - - runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoints); - - Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var first)); - Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 1, out var second)); - Assert.Equal(hintedEndpoints[0], first.RemoteEndPoint); - Assert.Equal(first, second); - } - - [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnStickyHintedPath_BeforeConfirmation() + public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedPaths_BeforeConfirmation() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -407,6 +342,9 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnStickyHintedPath runtime.PrimePeerForTests(peerIdentity); runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + var tcp = TcpCodec.Encode( IPAddress.Parse("10.0.0.1"), IPAddress.Parse("10.0.0.2"), @@ -429,12 +367,22 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnStickyHintedPath var payloadSends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) - .Where(send => send.Payload.Length > 4) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) .ToArray(); - var payloadSend = Assert.Single(payloadSends); - Assert.Equal(0, payloadSend.LocalSocketId); - Assert.Equal(hintedEndpoints[0], payloadSend.RemoteEndPoint); + Assert.Equal( + hintedEndpoints + .SelectMany(endpoint => new[] + { + (LocalSocketId: 0, RemoteEndPoint: endpoint), + (LocalSocketId: 1, RemoteEndPoint: endpoint) + }) + .OrderBy(item => item.RemoteEndPoint.Port) + .ThenBy(item => item.LocalSocketId) + .ToArray(), + payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 96c223e..550536b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -471,24 +471,16 @@ private async ValueTask SendToPeerAsync( return; } - await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - - if (_multipath.BondPolicy == ZeroTierBondPolicy.Broadcast) - { - await SendToPeerBroadcastAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); - return; - } + var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); + var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; - if (!TrySelectDirectPath(peerNodeId, flowId, out var direct)) + if (preferHintedDirectFanout && + !_multipath.AllowRootRelayFallback && + !HasConfirmedDirectPath(peerNodeId)) { - EnsureRelayAllowedForPayload(peerNodeId, reason: "no direct peer path is available"); - await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); - return; + await EnsureDirectBootstrapStartedAsync(peerNodeId, cancellationToken).ConfigureAwait(false); } - var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); - var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; - if (preferHintedDirectFanout && TryGetDirectOnlyHintedPayloadFanout(peerNodeId, flowId, out var fanoutPaths)) { @@ -523,6 +515,21 @@ private async ValueTask SendToPeerAsync( EnsureRelayAllowedForPayload(peerNodeId, reason: "direct-only hinted payload fanout failed"); } + await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + + if (_multipath.BondPolicy == ZeroTierBondPolicy.Broadcast) + { + await SendToPeerBroadcastAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + return; + } + + if (!TrySelectDirectPath(peerNodeId, flowId, out var direct)) + { + EnsureRelayAllowedForPayload(peerNodeId, reason: "no direct peer path is available"); + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + return; + } + var confirmed = _peerEcho.TryGetLastRttMs(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, out _); if (_multipath.WarmupDuplicateToRoot && _multipath.AllowRootRelayFallback && !confirmed) { @@ -690,19 +697,29 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return false; } - var endpoint = hinted[0]; - var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); - if (socketIds.Length == 0) + var unique = new HashSet(); + var candidates = new List(hinted.Length * 2); + for (var i = 0; i < hinted.Length; i++) { - socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); + for (var s = 0; s < socketIds.Length; s++) + { + var candidate = new ZeroTierSelectedPeerPath(socketIds[s], hinted[i]); + if (!unique.Add(new ZeroTierPeerPhysicalPathKey(candidate.LocalSocketId, candidate.RemoteEndPoint))) + { + continue; + } + + candidates.Add(candidate); + } } - if (socketIds.Length == 0) + if (candidates.Count == 0) { return false; } - fanoutPaths = [new ZeroTierSelectedPeerPath(socketIds[0], endpoint)]; + fanoutPaths = candidates.ToArray(); return true; } @@ -726,13 +743,13 @@ private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationTok { cancellationToken.ThrowIfCancellationRequested(); - if (!_multipath.Enabled || HasUsableDirectPath(peerNodeId)) + if (!_multipath.Enabled || HasSufficientDirectPath(peerNodeId)) { return; } var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - if (HasUsableDirectPath(peerNodeId)) + if (HasSufficientDirectPath(peerNodeId)) { return; } @@ -743,10 +760,7 @@ private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationTok try { - if (!CanOptimisticallyUseHintedDirectPath(peerNodeId)) - { - await bootstrapTask.WaitAsync(cancellationToken).ConfigureAwait(false); - } + await bootstrapTask.WaitAsync(cancellationToken).ConfigureAwait(false); } finally { @@ -756,7 +770,7 @@ private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationTok } } - if (!HasUsableDirectPath(peerNodeId)) + if (!HasSufficientDirectPath(peerNodeId)) { EnsureRelayAllowedForPayload( peerNodeId, @@ -774,6 +788,26 @@ private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationTok } } + private async ValueTask EnsureDirectBootstrapStartedAsync(NodeId peerNodeId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_multipath.Enabled || HasSufficientDirectPath(peerNodeId)) + { + return; + } + + var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + if (HasSufficientDirectPath(peerNodeId)) + { + return; + } + + _ = _directBootstrapTasks.GetOrAdd( + peerNodeId, + id => BootstrapDirectPathCoreAsync(id, (byte[])sharedKey.Clone(), _cts.Token)); + } + private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -918,9 +952,6 @@ private bool HasSufficientDirectPath(NodeId peerNodeId) ? HasDirectPathCandidate(peerNodeId) : HasConfirmedDirectPath(peerNodeId); - private bool HasUsableDirectPath(NodeId peerNodeId) - => HasSufficientDirectPath(peerNodeId) || CanOptimisticallyUseHintedDirectPath(peerNodeId); - private bool HasConfirmedDirectPath(NodeId peerNodeId) { if (_peerPaths.GetSnapshot(peerNodeId).Length != 0) @@ -962,11 +993,6 @@ private bool HasConfirmedDirectPath(NodeId peerNodeId) return false; } - private bool CanOptimisticallyUseHintedDirectPath(NodeId peerNodeId) - => !_multipath.AllowRootRelayFallback && - _peerPaths.GetSnapshot(peerNodeId).Length == 0 && - GetOrCreateDirectEndpointManager(peerNodeId).Endpoints.Length != 0; - private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { if (ShouldUsePeerRootSocketAffinity(peerNodeId) && From aae36c93607bb19e42d26be3b3e701d9fb3f56b8 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:28:47 +0100 Subject: [PATCH 232/296] Wait for hinted paths before SYN fanout --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 81 ++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 139 +++++++++++++----- 2 files changed, 183 insertions(+), 37 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index a6b1352..2d5049a 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -385,6 +385,87 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_DuringBootstrap() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None).AsTask(); + + await Task.Delay(300); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + await sendTask.WaitAsync(TimeSpan.FromSeconds(2)); + + var payloadSends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) + .ToArray(); + + Assert.Equal( + hintedEndpoints + .SelectMany(endpoint => new[] + { + (LocalSocketId: 0, RemoteEndPoint: endpoint), + (LocalSocketId: 1, RemoteEndPoint: endpoint) + }) + .OrderBy(item => item.RemoteEndPoint.Port) + .ThenBy(item => item.LocalSocketId) + .ToArray(), + payloadSends); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 550536b..01c6f18 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -475,44 +475,10 @@ private async ValueTask SendToPeerAsync( var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; if (preferHintedDirectFanout && - !_multipath.AllowRootRelayFallback && - !HasConfirmedDirectPath(peerNodeId)) + !HasConfirmedDirectPath(peerNodeId) && + await TrySendDirectOnlyHintedPayloadAsync(peerNodeId, packet, flowId, parsedOk ? parsed.PacketId : null, shouldRecord, cancellationToken).ConfigureAwait(false)) { - await EnsureDirectBootstrapStartedAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - } - - if (preferHintedDirectFanout && - TryGetDirectOnlyHintedPayloadFanout(peerNodeId, flowId, out var fanoutPaths)) - { - var directSuccess = 0; - for (var i = 0; i < fanoutPaths.Length; i++) - { - var path = fanoutPaths[i]; - if (shouldRecord) - { - _peerQos.RecordOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); - } - - try - { - await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); - directSuccess++; - } - catch (SocketException) - { - if (shouldRecord) - { - _peerQos.ForgetOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); - } - } - } - - if (directSuccess > 0) - { - return; - } - - EnsureRelayAllowedForPayload(peerNodeId, reason: "direct-only hinted payload fanout failed"); + return; } await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); @@ -723,6 +689,105 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return true; } + private async ValueTask TrySendDirectOnlyHintedPayloadAsync( + NodeId peerNodeId, + ReadOnlyMemory packet, + uint flowId, + ulong? packetId, + bool shouldRecord, + CancellationToken cancellationToken) + { + if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) + { + return false; + } + + await EnsureDirectBootstrapStartedAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (TryGetDirectOnlyHintedPayloadFanout(peerNodeId, flowId, out var fanoutPaths)) + { + var directSuccess = await SendDirectOnlyHintedPayloadFanoutAsync( + peerNodeId, + fanoutPaths, + packet, + packetId, + shouldRecord, + cancellationToken) + .ConfigureAwait(false); + if (directSuccess > 0) + { + return true; + } + + EnsureRelayAllowedForPayload(peerNodeId, reason: "direct-only hinted payload fanout failed"); + } + + if (_directBootstrapTasks.TryGetValue(peerNodeId, out var bootstrapTask)) + { + if (bootstrapTask.IsCompleted) + { + return false; + } + } + else + { + return false; + } + + await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken).ConfigureAwait(false); + } + } + + private async ValueTask SendDirectOnlyHintedPayloadFanoutAsync( + NodeId peerNodeId, + ZeroTierSelectedPeerPath[] fanoutPaths, + ReadOnlyMemory packet, + ulong? packetId, + bool shouldRecord, + CancellationToken cancellationToken) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] TX direct-only payload fanout: peer={peerNodeId} paths={string.Join(", ", fanoutPaths.Select(static path => $"{path.RemoteEndPoint}@{path.LocalSocketId}"))}."); + } + + var directSuccess = 0; + for (var i = 0; i < fanoutPaths.Length; i++) + { + var path = fanoutPaths[i]; + if (shouldRecord && packetId is ulong outboundPacketId) + { + _peerQos.RecordOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, outboundPacketId); + } + + try + { + await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); + directSuccess++; + } + catch (SocketException) + { + if (shouldRecord && packetId is ulong failedPacketId) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, failedPacketId); + } + } + } + + if (directSuccess > 0 && ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct-only payload fanout sent: peer={peerNodeId} successes={directSuccess}/{fanoutPaths.Length}."); + } + + return directSuccess; + } + private void EnsureRelayAllowedForPayload(NodeId peerNodeId, string reason) { if (_multipath.AllowRootRelayFallback) From bb45a149069c0dd43631647baffde46e5f321b35 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:35:21 +0100 Subject: [PATCH 233/296] Prime direct-only hinted paths with HELLO --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 74 +++++++++++++------ .../Internal/ZeroTierDataplaneRuntime.cs | 24 ++++++ 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 2d5049a..b3a9346 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -365,24 +365,37 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); - var payloadSends = udp.GetSendsSnapshot() + var expectedFanout = hintedEndpoints + .SelectMany(endpoint => new[] + { + (LocalSocketId: 0, RemoteEndPoint: endpoint), + (LocalSocketId: 1, RemoteEndPoint: endpoint) + }) + .OrderBy(item => item.RemoteEndPoint.Port) + .ThenBy(item => item.LocalSocketId) + .ToArray(); + + var sends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .ToArray(); + + var helloSends = sends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) + .ToArray(); + + var payloadSends = sends .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) .OrderBy(send => send.RemoteEndPoint.Port) .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal( - hintedEndpoints - .SelectMany(endpoint => new[] - { - (LocalSocketId: 0, RemoteEndPoint: endpoint), - (LocalSocketId: 1, RemoteEndPoint: endpoint) - }) - .OrderBy(item => item.RemoteEndPoint.Port) - .ThenBy(item => item.LocalSocketId) - .ToArray(), - payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); + Assert.Equal(expectedFanout, helloSends); + Assert.Equal(expectedFanout, payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); } [Fact] @@ -444,7 +457,29 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur await sendTask.WaitAsync(TimeSpan.FromSeconds(2)); - var payloadSends = udp.GetSendsSnapshot() + var expectedFanout = hintedEndpoints + .SelectMany(endpoint => new[] + { + (LocalSocketId: 0, RemoteEndPoint: endpoint), + (LocalSocketId: 1, RemoteEndPoint: endpoint) + }) + .OrderBy(item => item.RemoteEndPoint.Port) + .ThenBy(item => item.LocalSocketId) + .ToArray(); + + var sends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .ToArray(); + + var helloSends = sends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) + .ToArray(); + + var payloadSends = sends .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) @@ -453,17 +488,8 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal( - hintedEndpoints - .SelectMany(endpoint => new[] - { - (LocalSocketId: 0, RemoteEndPoint: endpoint), - (LocalSocketId: 1, RemoteEndPoint: endpoint) - }) - .OrderBy(item => item.RemoteEndPoint.Port) - .ThenBy(item => item.LocalSocketId) - .ToArray(), - payloadSends); + Assert.Equal(expectedFanout, helloSends); + Assert.Equal(expectedFanout, payloadSends); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 01c6f18..2d4c723 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -710,6 +710,8 @@ private async ValueTask TrySendDirectOnlyHintedPayloadAsync( if (TryGetDirectOnlyHintedPayloadFanout(peerNodeId, flowId, out var fanoutPaths)) { + await PrimeDirectOnlyHintedPayloadPathsAsync(peerNodeId, fanoutPaths, cancellationToken).ConfigureAwait(false); + var directSuccess = await SendDirectOnlyHintedPayloadFanoutAsync( peerNodeId, fanoutPaths, @@ -742,6 +744,28 @@ private async ValueTask TrySendDirectOnlyHintedPayloadAsync( } } + private async ValueTask PrimeDirectOnlyHintedPayloadPathsAsync( + NodeId peerNodeId, + ZeroTierSelectedPeerPath[] fanoutPaths, + CancellationToken cancellationToken) + { + var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + + for (var i = 0; i < fanoutPaths.Length; i++) + { + var path = fanoutPaths[i]; + await SendHelloPacketAsync( + path.LocalSocketId, + peerNodeId, + path.RemoteEndPoint, + path.RemoteEndPoint, + sharedKey, + minIntervalMs: 0, + cancellationToken) + .ConfigureAwait(false); + } + } + private async ValueTask SendDirectOnlyHintedPayloadFanoutAsync( NodeId peerNodeId, ZeroTierSelectedPeerPath[] fanoutPaths, From 9f6014df4ab18be9ba5e9f37a4995fd30dc32236 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:50:25 +0100 Subject: [PATCH 234/296] Advertise local surfaces in direct-only HELLO --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 84 ++++++++++++++++++- .../Internal/ZeroTierDataplaneRuntime.cs | 33 +++++++- 2 files changed, 110 insertions(+), 7 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index b3a9346..94d9589 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -233,7 +233,11 @@ public async Task DataplaneRuntime_DirectOnly_ExposesHintedPathCandidate_Without var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); var rootNodeId = new NodeId(0x1111111111); var peerNodeId = new NodeId(0x3333333333); - var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime( @@ -245,10 +249,10 @@ public async Task DataplaneRuntime_DirectOnly_ExposesHintedPathCandidate_Without planetId: 1, planetTimestamp: 1); - runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoint); + runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoints); Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var selected)); - Assert.Equal(hintedEndpoint, selected.RemoteEndPoint); + Assert.Equal(hintedEndpoints[0], selected.RemoteEndPoint); Assert.Equal(0, selected.LocalSocketId); } @@ -393,11 +397,83 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP .OrderBy(send => send.RemoteEndPoint.Port) .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal(expectedFanout, helloSends); Assert.Equal(expectedFanout, payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingHello_ReportsLocalSurfacePerSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); + + var helloSurfaces = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + .Select(send => (send.LocalSocketId, Surface: ReadReportedHelloSurface(send.Payload, sharedKey))) + .Distinct() + .OrderBy(send => send.LocalSocketId) + .ToArray(); + + Assert.Contains(helloSurfaces, send => send.LocalSocketId == 0 && Equals(send.Surface, advertisedSurfaces[0].SurfaceAddress)); + Assert.Contains(helloSurfaces, send => send.LocalSocketId == 1 && Equals(send.Surface, advertisedSurfaces[1].SurfaceAddress)); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_DuringBootstrap() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 2d4c723..19bb0c3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -321,6 +321,30 @@ private IPEndPoint[] GetLocalDirectPathAdvertisements() maxEndpoints: ZeroTierProtocolLimits.MaxPushedDirectPaths); } + private IPEndPoint? GetAdvertisedLocalSurfaceForSocket(int localSocketId, IPEndPoint remoteEndPoint) + { + var directCandidates = SelectAdvertisedLocalSurfaces(_surfaceAddresses.GetSnapshot(localSocketId), remoteEndPoint); + if (directCandidates is not null) + { + return directCandidates; + } + + return SelectAdvertisedLocalSurfaces(GetLocalDirectPathAdvertisements(), remoteEndPoint); + } + + private static IPEndPoint? SelectAdvertisedLocalSurfaces(IPEndPoint[] candidates, IPEndPoint remoteEndPoint) + { + for (var i = 0; i < candidates.Length; i++) + { + if (candidates[i].AddressFamily == remoteEndPoint.AddressFamily) + { + return candidates[i]; + } + } + + return candidates.Length > 0 ? candidates[0] : null; + } + private void SeedInitialExternalSurfaceObservations(IReadOnlyList? observations) { if (observations is null || observations.Count == 0) @@ -750,6 +774,7 @@ private async ValueTask PrimeDirectOnlyHintedPayloadPathsAsync( CancellationToken cancellationToken) { var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + await TrySendPeriodicDirectPathPushAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); for (var i = 0; i < fanoutPaths.Length; i++) { @@ -761,7 +786,8 @@ await SendHelloPacketAsync( path.RemoteEndPoint, sharedKey, minIntervalMs: 0, - cancellationToken) + cancellationToken, + reportedSurfaceOverride: GetAdvertisedLocalSurfaceForSocket(path.LocalSocketId, path.RemoteEndPoint)) .ConfigureAwait(false); } } @@ -1444,14 +1470,15 @@ private async Task SendHelloPacketAsync( IPEndPoint sendTo, byte[] sharedKey, long minIntervalMs, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IPEndPoint? reportedSurfaceOverride = null) { if (physicalDestination is not null && !ShouldSendDirectHello(peerNodeId, localSocketId, sendTo, minIntervalMs)) { return; } - var reportedRemoteSurface = physicalDestination; + var reportedRemoteSurface = reportedSurfaceOverride ?? physicalDestination; try { From 32fadbf134916a886d4209d45486f4e5e9dc5d96 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:55:35 +0100 Subject: [PATCH 235/296] Use advertised surfaces in hinted HELLO bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 55 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 17 ++++-- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 94d9589..48b830a 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -646,6 +646,61 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHello Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Echo); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsLocalSurfacePerSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var helloSurfaces = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(hintedEndpoints[0])) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + .Select(send => (send.LocalSocketId, Surface: ReadReportedHelloSurface(send.Payload, sharedKey))) + .Distinct() + .OrderBy(send => send.LocalSocketId) + .ToArray(); + + Assert.Contains(helloSurfaces, send => send.LocalSocketId == 0 && Equals(send.Surface, advertisedSurfaces[0].SurfaceAddress)); + Assert.Contains(helloSurfaces, send => send.LocalSocketId == 1 && Equals(send.Surface, advertisedSurfaces[1].SurfaceAddress)); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 19bb0c3..f0f69a9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1038,7 +1038,8 @@ await SendHelloPacketAcrossSocketsAsync( endpointsToProbe[i], sharedKey, minIntervalMs, - cancellationToken) + cancellationToken, + useAdvertisedSurfaceOverride: true) .ConfigureAwait(false); continue; } @@ -1433,7 +1434,8 @@ private Task SendDirectBootstrapProbeAsync( remoteEndPoint, sharedKey, helloMinIntervalMs, - cancellationToken) + cancellationToken, + reportedSurfaceOverride: GetAdvertisedLocalSurfaceForSocket(localSocketId, remoteEndPoint)) : TrySendEchoDirectProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken); private async Task TrySendEchoDirectProbeAsync( @@ -1544,7 +1546,8 @@ private async Task SendHelloPacketAcrossSocketsAsync( IPEndPoint sendTo, byte[] sharedKey, long minIntervalMs, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool useAdvertisedSurfaceOverride = false) { ArgumentNullException.ThrowIfNull(localSocketIds); @@ -1557,7 +1560,10 @@ await SendHelloPacketAsync( sendTo, sharedKey, minIntervalMs, - cancellationToken) + cancellationToken, + reportedSurfaceOverride: useAdvertisedSurfaceOverride && physicalDestination is not null + ? GetAdvertisedLocalSurfaceForSocket(localSocketIds[i], physicalDestination) + : null) .ConfigureAwait(false); } } @@ -1952,7 +1958,8 @@ await SendHelloPacketAcrossSocketsAsync( candidate.RemoteEndPoint, key, minIntervalMs, - cancellationToken) + cancellationToken, + useAdvertisedSurfaceOverride: true) .ConfigureAwait(false); continue; } From dac5a36a997059a525dbafdf4f3505b68cd8cb92 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:08:24 +0100 Subject: [PATCH 236/296] Tighten direct bootstrap relay control --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 67 ++++++++++++++++++- ...roTierDirectBootstrapControlPolicyTests.cs | 40 +++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 64 ++++++++++++++++-- .../ZeroTierDirectBootstrapControlPolicy.cs | 10 +++ 4 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 48b830a..6514ef2 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -531,7 +531,7 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur await Task.Delay(300); runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); - await sendTask.WaitAsync(TimeSpan.FromSeconds(2)); + await sendTask.WaitAsync(TimeSpan.FromSeconds(5)); var expectedFanout = hintedEndpoints .SelectMany(endpoint => new[] @@ -701,6 +701,58 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsLocalSurfa Assert.Contains(helloSurfaces, send => send.LocalSocketId == 1 && Equals(send.Surface, advertisedSurfaces[1].SurfaceAddress)); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_MaintenanceRelaysRendezvousForAdvertisedSurfaces() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] { new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244) }; + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var rendezvousEndpoints = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Rendezvous) + .Select(send => ReadRendezvousEndpoint(send.Payload, sharedKey)) + .Distinct() + .OrderBy(endpoint => endpoint.Port) + .ToArray(); + + Assert.Equal( + advertisedSurfaces.Select(static observation => observation.SurfaceAddress).OrderBy(endpoint => endpoint.Port).ToArray(), + rendezvousEndpoints); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { @@ -944,6 +996,19 @@ private static bool TryDecodeVerb(byte[] packet, byte[] sharedKey, out ZeroTierV return surface; } + private static IPEndPoint ReadRendezvousEndpoint(byte[] packet, byte[] sharedKey) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed) || + !ZeroTierRendezvousCodec.TryParse(parsed.Payload.Span, out var rendezvous)) + { + throw new InvalidOperationException("Failed to decode RENDEZVOUS packet."); + } + + return rendezvous.Endpoint; + } + private static byte[] BuildPushDirectPathsPacket( NodeId source, NodeId destination, diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs new file mode 100644 index 0000000..8f13e68 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -0,0 +1,40 @@ +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectBootstrapControlPolicyTests +{ + [Theory] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasHintedDirectEndpoints, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( + allowRootRelayFallback, + hasHintedDirectEndpoints)); + } + + [Theory] + [InlineData(true, false, false)] + [InlineData(true, true, false)] + [InlineData(false, true, false)] + [InlineData(false, false, true)] + public void ShouldForceFullHelloForHintedCandidate_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldForceFullHelloForHintedCandidate( + allowRootRelayFallback, + hasConfirmedDirectPath)); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index f0f69a9..7058f82 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -941,7 +941,11 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared cancellationToken.ThrowIfCancellationRequested(); now = Environment.TickCount64; - if (unchecked(now - nextRootHelloAt) >= 0) + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (unchecked(now - nextRootHelloAt) >= 0 && + ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( + _multipath.AllowRootRelayFallback, + hinted.Length != 0)) { await SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); await SendNetworkCredentialsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); @@ -954,7 +958,6 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } - var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; if (hinted.Length > 0 && unchecked(now - nextHintProbeAt) >= 0) { await ProbeHintedDirectEndpointsAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); @@ -998,6 +1001,7 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendRendezvousAdvertisementsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -1203,6 +1207,57 @@ private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); } + private async Task SendRendezvousAdvertisementsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) + { + return; + } + + var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId); + if (advertisements.Length == 0) + { + return; + } + + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + for (var i = 0; i < advertisements.Length; i++) + { + try + { + var payload = ZeroTierRendezvousCodec.BuildPayload(peerNodeId, advertisements[i]); + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + payload); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisements[i]}."); + } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId} ({advertisements[i]}): {ex.GetType().Name}: {ex.Message}"); + } + } + } + } + private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken) { var localSockets = _udp.LocalSockets; @@ -1876,8 +1931,9 @@ private bool ShouldFanOutHintedBootstrapAcrossSockets(NodeId peerNodeId) => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); private bool ShouldForceFullHelloForHintedCandidate(NodeId peerNodeId) - => ShouldFanOutHintedBootstrapAcrossSockets(peerNodeId) && - !CanUseEchoForDirectBootstrap(peerNodeId); + => ZeroTierDirectBootstrapControlPolicy.ShouldForceFullHelloForHintedCandidate( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId)); private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId) => ShouldUseStickyHintedPathSelection(peerNodeId) && diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs new file mode 100644 index 0000000..9a47768 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -0,0 +1,10 @@ +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierDirectBootstrapControlPolicy +{ + public static bool ShouldRefreshRelayedRootControl(bool allowRootRelayFallback, bool hasHintedDirectEndpoints) + => allowRootRelayFallback || !hasHintedDirectEndpoints; + + public static bool ShouldForceFullHelloForHintedCandidate(bool allowRootRelayFallback, bool hasConfirmedDirectPath) + => !allowRootRelayFallback && !hasConfirmedDirectPath; +} From 3f54b8af2fce9e45060c06652fd3f70810cedc10 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:20:08 +0100 Subject: [PATCH 237/296] Prefer pinned rendezvous paths in direct-only mode --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 23 ++++++------- ...roTierDirectBootstrapControlPolicyTests.cs | 31 ++++++++++++++--- .../Internal/ZeroTierDataplaneRuntime.cs | 33 +++++++++++++++---- .../ZeroTierDirectBootstrapControlPolicy.cs | 10 ++++-- .../Internal/ZeroTierDirectEndpointManager.cs | 21 ++++++++++++ 5 files changed, 91 insertions(+), 27 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 6514ef2..f5ece09 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -305,16 +305,15 @@ public async Task DataplaneRuntime_DirectOnly_PrefersRendezvousEndpoint_AfterLat BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(pushedEndpoint)), + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), TimeSpan.FromSeconds(2)); var endpoints = runtime.GetDirectEndpointsForTests(peerIdentity.NodeId); Assert.Equal(rendezvousEndpoint, endpoints[0]); - Assert.Contains(endpoints, endpoint => endpoint.Equals(pushedEndpoint)); } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedPaths_BeforeConfirmation() + public async Task DataplaneRuntime_DirectOnly_SynPayload_UsesFirstHintedPath_BeforeConfirmation() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -369,12 +368,11 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); - var expectedFanout = hintedEndpoints - .SelectMany(endpoint => new[] + var expectedFanout = new[] { - (LocalSocketId: 0, RemoteEndPoint: endpoint), - (LocalSocketId: 1, RemoteEndPoint: endpoint) - }) + (LocalSocketId: 0, RemoteEndPoint: hintedEndpoints[0]), + (LocalSocketId: 1, RemoteEndPoint: hintedEndpoints[0]) + } .OrderBy(item => item.RemoteEndPoint.Port) .ThenBy(item => item.LocalSocketId) .ToArray(); @@ -533,12 +531,11 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur await sendTask.WaitAsync(TimeSpan.FromSeconds(5)); - var expectedFanout = hintedEndpoints - .SelectMany(endpoint => new[] + var expectedFanout = new[] { - (LocalSocketId: 0, RemoteEndPoint: endpoint), - (LocalSocketId: 1, RemoteEndPoint: endpoint) - }) + (LocalSocketId: 0, RemoteEndPoint: hintedEndpoints[0]), + (LocalSocketId: 1, RemoteEndPoint: hintedEndpoints[0]) + } .OrderBy(item => item.RemoteEndPoint.Port) .ThenBy(item => item.LocalSocketId) .ToArray(); diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 8f13e68..001a51f 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -5,20 +5,24 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectBootstrapControlPolicyTests { [Theory] - [InlineData(true, false, true)] - [InlineData(true, true, true)] - [InlineData(false, false, true)] - [InlineData(false, true, false)] + [InlineData(true, false, false, true)] + [InlineData(true, true, false, true)] + [InlineData(true, true, true, true)] + [InlineData(false, false, false, true)] + [InlineData(false, true, false, true)] + [InlineData(false, true, true, false)] public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( bool allowRootRelayFallback, bool hasHintedDirectEndpoints, + bool hasPinnedRendezvousEndpoints, bool expected) { Assert.Equal( expected, ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( allowRootRelayFallback, - hasHintedDirectEndpoints)); + hasHintedDirectEndpoints, + hasPinnedRendezvousEndpoints)); } [Theory] @@ -37,4 +41,21 @@ public void ShouldForceFullHelloForHintedCandidate_ReturnsExpectedValue( allowRootRelayFallback, hasConfirmedDirectPath)); } + + [Theory] + [InlineData(true, false, false)] + [InlineData(true, true, false)] + [InlineData(false, true, false)] + [InlineData(false, false, true)] + public void ShouldRestrictToFirstHintedEndpoint_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldRestrictToFirstHintedEndpoint( + allowRootRelayFallback, + hasConfirmedDirectPath)); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 7058f82..d5a5990 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -681,20 +681,27 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return false; } - var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var hinted = directEndpoints.Endpoints; if (hinted.Length <= 1) { return false; } + var restrictToFirstHintedEndpoint = ZeroTierDirectBootstrapControlPolicy.ShouldRestrictToFirstHintedEndpoint( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId)); + var endpointsToUse = restrictToFirstHintedEndpoint ? [hinted[0]] : hinted; var unique = new HashSet(); - var candidates = new List(hinted.Length * 2); - for (var i = 0; i < hinted.Length; i++) + var candidates = new List(endpointsToUse.Length * 2); + for (var i = 0; i < endpointsToUse.Length; i++) { - var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); + var socketIds = restrictToFirstHintedEndpoint && directEndpoints.IsPinnedRendezvousEndpoint(endpointsToUse[i]) + ? directEndpoints.GetPreferredLocalSocketIds(endpointsToUse[i]) + : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpointsToUse[i]); for (var s = 0; s < socketIds.Length; s++) { - var candidate = new ZeroTierSelectedPeerPath(socketIds[s], hinted[i]); + var candidate = new ZeroTierSelectedPeerPath(socketIds[s], endpointsToUse[i]); if (!unique.Add(new ZeroTierPeerPhysicalPathKey(candidate.LocalSocketId, candidate.RemoteEndPoint))) { continue; @@ -941,11 +948,13 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared cancellationToken.ThrowIfCancellationRequested(); now = Environment.TickCount64; - var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var hinted = directEndpoints.Endpoints; if (unchecked(now - nextRootHelloAt) >= 0 && ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( _multipath.AllowRootRelayFallback, - hinted.Length != 0)) + hinted.Length != 0, + directEndpoints.HasPinnedRendezvousEndpoints)) { await SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); await SendNetworkCredentialsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); @@ -1406,6 +1415,16 @@ private async ValueTask HandleDirectEndpointHintAsync( } forceFullHello |= ShouldForceFullHelloForHintedCandidate(peerNodeId); + if (ZeroTierDirectBootstrapControlPolicy.ShouldRestrictToFirstHintedEndpoint( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId))) + { + var hintedEndpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (hintedEndpoints.Length != 0 && !endpoint.Equals(hintedEndpoints[0])) + { + return; + } + } if (ZeroTierTrace.Enabled) { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index 9a47768..fe56387 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -2,9 +2,15 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierDirectBootstrapControlPolicy { - public static bool ShouldRefreshRelayedRootControl(bool allowRootRelayFallback, bool hasHintedDirectEndpoints) - => allowRootRelayFallback || !hasHintedDirectEndpoints; + public static bool ShouldRefreshRelayedRootControl( + bool allowRootRelayFallback, + bool hasHintedDirectEndpoints, + bool hasPinnedRendezvousEndpoints) + => allowRootRelayFallback || !hasHintedDirectEndpoints || !hasPinnedRendezvousEndpoints; public static bool ShouldForceFullHelloForHintedCandidate(bool allowRootRelayFallback, bool hasConfirmedDirectPath) => !allowRootRelayFallback && !hasConfirmedDirectPath; + + public static bool ShouldRestrictToFirstHintedEndpoint(bool allowRootRelayFallback, bool hasConfirmedDirectPath) + => !allowRootRelayFallback && !hasConfirmedDirectPath; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 9bfc50b..8e65694 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -53,6 +53,17 @@ public ZeroTierDirectEndpointManager( public IPEndPoint[] Endpoints => _directEndpoints; + public bool HasPinnedRendezvousEndpoints + { + get + { + lock (_lock) + { + return _pinnedRendezvousEndpointKeys.Count != 0; + } + } + } + public int[] GetPreferredLocalSocketIds(IPEndPoint endpoint) { ArgumentNullException.ThrowIfNull(endpoint); @@ -81,6 +92,16 @@ public int[] GetPreferredLocalSocketIds(IPEndPoint endpoint) .ToArray(); } + public bool IsPinnedRendezvousEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + lock (_lock) + { + return _pinnedRendezvousEndpointKeys.Contains(FormatEndpointKey(endpoint)); + } + } + public async ValueTask HandleRendezvousFromRootAsync( ReadOnlyMemory payload, int receivedLocalSocketId, From 53259d63bf58b74165d8c728692c525c6cdca7aa Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:37:01 +0100 Subject: [PATCH 238/296] Use echo-first hinted direct bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 71 +++++---- ...roTierDirectBootstrapControlPolicyTests.cs | 34 ---- ...TierDirectEndpointManagerPushFlagsTests.cs | 2 +- ...irectEndpointManagerSocketAffinityTests.cs | 2 +- .../Internal/ZeroTierDataplaneRuntime.cs | 149 +++--------------- .../ZeroTierDirectBootstrapControlPolicy.cs | 6 - .../Internal/ZeroTierDirectEndpointManager.cs | 4 +- 7 files changed, 63 insertions(+), 205 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index f5ece09..8817c64 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -99,7 +99,7 @@ await WaitForConditionAsync( var firstBootstrapSend = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) .OrderBy(send => send.LocalSocketId) .First(); @@ -313,7 +313,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_UsesFirstHintedPath_BeforeConfirmation() + public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedPaths_BeforeConfirmation() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -368,11 +368,12 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_UsesFirstHintedPath_Bef await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); - var expectedFanout = new[] + var expectedFanout = hintedEndpoints + .SelectMany(endpoint => new[] { - (LocalSocketId: 0, RemoteEndPoint: hintedEndpoints[0]), - (LocalSocketId: 1, RemoteEndPoint: hintedEndpoints[0]) - } + (LocalSocketId: 0, RemoteEndPoint: endpoint), + (LocalSocketId: 1, RemoteEndPoint: endpoint) + }) .OrderBy(item => item.RemoteEndPoint.Port) .ThenBy(item => item.LocalSocketId) .ToArray(); @@ -381,8 +382,8 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_UsesFirstHintedPath_Bef .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .ToArray(); - var helloSends = sends - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + var echoSends = sends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) .Distinct() .OrderBy(send => send.RemoteEndPoint.Port) @@ -395,12 +396,12 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_UsesFirstHintedPath_Bef .OrderBy(send => send.RemoteEndPoint.Port) .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal(expectedFanout, helloSends); + Assert.Equal(expectedFanout, echoSends); Assert.Equal(expectedFanout, payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingHello_ReportsLocalSurfacePerSocket() + public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingEcho_UsesHintedPeerEndpoint() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -460,16 +461,26 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingHello_ReportsLoc await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); - var helloSurfaces = udp.GetSendsSnapshot() + var echoSends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) - .Select(send => (send.LocalSocketId, Surface: ReadReportedHelloSurface(send.Payload, sharedKey))) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) .Distinct() - .OrderBy(send => send.LocalSocketId) + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Contains(helloSurfaces, send => send.LocalSocketId == 0 && Equals(send.Surface, advertisedSurfaces[0].SurfaceAddress)); - Assert.Contains(helloSurfaces, send => send.LocalSocketId == 1 && Equals(send.Surface, advertisedSurfaces[1].SurfaceAddress)); + Assert.Equal( + hintedEndpoints + .SelectMany(endpoint => new[] + { + (LocalSocketId: 0, RemoteEndPoint: endpoint), + (LocalSocketId: 1, RemoteEndPoint: endpoint) + }) + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) + .ToArray(), + echoSends); } [Fact] @@ -531,11 +542,12 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur await sendTask.WaitAsync(TimeSpan.FromSeconds(5)); - var expectedFanout = new[] + var expectedFanout = hintedEndpoints + .SelectMany(endpoint => new[] { - (LocalSocketId: 0, RemoteEndPoint: hintedEndpoints[0]), - (LocalSocketId: 1, RemoteEndPoint: hintedEndpoints[0]) - } + (LocalSocketId: 0, RemoteEndPoint: endpoint), + (LocalSocketId: 1, RemoteEndPoint: endpoint) + }) .OrderBy(item => item.RemoteEndPoint.Port) .ThenBy(item => item.LocalSocketId) .ToArray(); @@ -544,8 +556,8 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .ToArray(); - var helloSends = sends - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + var echoSends = sends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) .Distinct() .OrderBy(send => send.RemoteEndPoint.Port) @@ -561,7 +573,7 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal(expectedFanout, helloSends); + Assert.Equal(expectedFanout, echoSends); Assert.Equal(expectedFanout, payloadSends); } @@ -644,7 +656,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHello } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsLocalSurfacePerSocket() + public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsHintedPeerEndpoint() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -689,17 +701,16 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsLocalSurfa var helloSurfaces = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(hintedEndpoints[0])) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) - .Select(send => (send.LocalSocketId, Surface: ReadReportedHelloSurface(send.Payload, sharedKey))) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint, Surface: ReadReportedHelloSurface(send.Payload, sharedKey))) .Distinct() .OrderBy(send => send.LocalSocketId) .ToArray(); - Assert.Contains(helloSurfaces, send => send.LocalSocketId == 0 && Equals(send.Surface, advertisedSurfaces[0].SurfaceAddress)); - Assert.Contains(helloSurfaces, send => send.LocalSocketId == 1 && Equals(send.Surface, advertisedSurfaces[1].SurfaceAddress)); + Assert.All(helloSurfaces, send => Assert.Equal(send.RemoteEndPoint, send.Surface)); } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceRelaysRendezvousForAdvertisedSurfaces() + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvousForAdvertisedSurfaces() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -745,9 +756,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceRelaysRendezvousForAdve .OrderBy(endpoint => endpoint.Port) .ToArray(); - Assert.Equal( - advertisedSurfaces.Select(static observation => observation.SurfaceAddress).OrderBy(endpoint => endpoint.Port).ToArray(), - rendezvousEndpoints); + Assert.Empty(rendezvousEndpoints); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 001a51f..500672c 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -24,38 +24,4 @@ public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( hasHintedDirectEndpoints, hasPinnedRendezvousEndpoints)); } - - [Theory] - [InlineData(true, false, false)] - [InlineData(true, true, false)] - [InlineData(false, true, false)] - [InlineData(false, false, true)] - public void ShouldForceFullHelloForHintedCandidate_ReturnsExpectedValue( - bool allowRootRelayFallback, - bool hasConfirmedDirectPath, - bool expected) - { - Assert.Equal( - expected, - ZeroTierDirectBootstrapControlPolicy.ShouldForceFullHelloForHintedCandidate( - allowRootRelayFallback, - hasConfirmedDirectPath)); - } - - [Theory] - [InlineData(true, false, false)] - [InlineData(true, true, false)] - [InlineData(false, true, false)] - [InlineData(false, false, true)] - public void ShouldRestrictToFirstHintedEndpoint_ReturnsExpectedValue( - bool allowRootRelayFallback, - bool hasConfirmedDirectPath, - bool expected) - { - Assert.Equal( - expected, - ZeroTierDirectBootstrapControlPolicy.ShouldRestrictToFirstHintedEndpoint( - allowRootRelayFallback, - hasConfirmedDirectPath)); - } } diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 9b89482..eae8816 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -180,7 +180,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, false, 1), hints[0]); + Assert.Equal((endpoint, false, true, 1), hints[0]); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index 7d700c5..f9e946a 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -55,7 +55,7 @@ await manager.HandleRendezvousFromRootAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, true, false, 1), hints[0]); + Assert.Equal((endpoint, false, false, 1), hints[0]); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d5a5990..fbae194 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -321,30 +321,6 @@ private IPEndPoint[] GetLocalDirectPathAdvertisements() maxEndpoints: ZeroTierProtocolLimits.MaxPushedDirectPaths); } - private IPEndPoint? GetAdvertisedLocalSurfaceForSocket(int localSocketId, IPEndPoint remoteEndPoint) - { - var directCandidates = SelectAdvertisedLocalSurfaces(_surfaceAddresses.GetSnapshot(localSocketId), remoteEndPoint); - if (directCandidates is not null) - { - return directCandidates; - } - - return SelectAdvertisedLocalSurfaces(GetLocalDirectPathAdvertisements(), remoteEndPoint); - } - - private static IPEndPoint? SelectAdvertisedLocalSurfaces(IPEndPoint[] candidates, IPEndPoint remoteEndPoint) - { - for (var i = 0; i < candidates.Length; i++) - { - if (candidates[i].AddressFamily == remoteEndPoint.AddressFamily) - { - return candidates[i]; - } - } - - return candidates.Length > 0 ? candidates[0] : null; - } - private void SeedInitialExternalSurfaceObservations(IReadOnlyList? observations) { if (observations is null || observations.Count == 0) @@ -683,25 +659,19 @@ private bool TryGetDirectOnlyHintedPayloadFanout( var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); var hinted = directEndpoints.Endpoints; - if (hinted.Length <= 1) + if (hinted.Length == 0) { return false; } - var restrictToFirstHintedEndpoint = ZeroTierDirectBootstrapControlPolicy.ShouldRestrictToFirstHintedEndpoint( - _multipath.AllowRootRelayFallback, - HasConfirmedDirectPath(peerNodeId)); - var endpointsToUse = restrictToFirstHintedEndpoint ? [hinted[0]] : hinted; var unique = new HashSet(); - var candidates = new List(endpointsToUse.Length * 2); - for (var i = 0; i < endpointsToUse.Length; i++) + var candidates = new List(hinted.Length * 2); + for (var i = 0; i < hinted.Length; i++) { - var socketIds = restrictToFirstHintedEndpoint && directEndpoints.IsPinnedRendezvousEndpoint(endpointsToUse[i]) - ? directEndpoints.GetPreferredLocalSocketIds(endpointsToUse[i]) - : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpointsToUse[i]); + var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); for (var s = 0; s < socketIds.Length; s++) { - var candidate = new ZeroTierSelectedPeerPath(socketIds[s], endpointsToUse[i]); + var candidate = new ZeroTierSelectedPeerPath(socketIds[s], hinted[i]); if (!unique.Add(new ZeroTierPeerPhysicalPathKey(candidate.LocalSocketId, candidate.RemoteEndPoint))) { continue; @@ -786,15 +756,14 @@ private async ValueTask PrimeDirectOnlyHintedPayloadPathsAsync( for (var i = 0; i < fanoutPaths.Length; i++) { var path = fanoutPaths[i]; - await SendHelloPacketAsync( - path.LocalSocketId, + await SendDirectBootstrapProbeAsync( peerNodeId, - path.RemoteEndPoint, + path.LocalSocketId, path.RemoteEndPoint, sharedKey, - minIntervalMs: 0, - cancellationToken, - reportedSurfaceOverride: GetAdvertisedLocalSurfaceForSocket(path.LocalSocketId, path.RemoteEndPoint)) + forceFullHello: false, + helloMinIntervalMs: 0, + cancellationToken) .ConfigureAwait(false); } } @@ -1010,7 +979,6 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendRendezvousAdvertisementsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -1051,8 +1019,7 @@ await SendHelloPacketAcrossSocketsAsync( endpointsToProbe[i], sharedKey, minIntervalMs, - cancellationToken, - useAdvertisedSurfaceOverride: true) + cancellationToken) .ConfigureAwait(false); continue; } @@ -1216,57 +1183,6 @@ private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); } - private async Task SendRendezvousAdvertisementsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) - { - if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) - { - return; - } - - var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId); - if (advertisements.Length == 0) - { - return; - } - - var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - for (var i = 0; i < advertisements.Length; i++) - { - try - { - var payload = ZeroTierRendezvousCodec.BuildPayload(peerNodeId, advertisements[i]); - var packet = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Rendezvous), - payload); - ZeroTierPacketCrypto.Armor( - packet, - ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), - encryptPayload: true); - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {advertisements[i]}."); - } - - await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine( - $"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId} ({advertisements[i]}): {ex.GetType().Name}: {ex.Message}"); - } - } - } - } - private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken) { var localSockets = _udp.LocalSockets; @@ -1414,18 +1330,6 @@ private async ValueTask HandleDirectEndpointHintAsync( return; } - forceFullHello |= ShouldForceFullHelloForHintedCandidate(peerNodeId); - if (ZeroTierDirectBootstrapControlPolicy.ShouldRestrictToFirstHintedEndpoint( - _multipath.AllowRootRelayFallback, - HasConfirmedDirectPath(peerNodeId))) - { - var hintedEndpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; - if (hintedEndpoints.Length != 0 && !endpoint.Equals(hintedEndpoints[0])) - { - return; - } - } - if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine( @@ -1508,8 +1412,7 @@ private Task SendDirectBootstrapProbeAsync( remoteEndPoint, sharedKey, helloMinIntervalMs, - cancellationToken, - reportedSurfaceOverride: GetAdvertisedLocalSurfaceForSocket(localSocketId, remoteEndPoint)) + cancellationToken) : TrySendEchoDirectProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken); private async Task TrySendEchoDirectProbeAsync( @@ -1546,27 +1449,24 @@ private async Task SendHelloPacketAsync( IPEndPoint sendTo, byte[] sharedKey, long minIntervalMs, - CancellationToken cancellationToken, - IPEndPoint? reportedSurfaceOverride = null) + CancellationToken cancellationToken) { if (physicalDestination is not null && !ShouldSendDirectHello(peerNodeId, localSocketId, sendTo, minIntervalMs)) { return; } - var reportedRemoteSurface = reportedSurfaceOverride ?? physicalDestination; - try { if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, reported={reportedRemoteSurface})."); + ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, reported={physicalDestination})."); } var packet = ZeroTierHelloPacketBuilder.BuildPacket( _localIdentity, peerNodeId, - reportedRemoteSurface, + physicalDestination, timestamp: (ulong)Environment.TickCount64, _planetId, _planetTimestamp, @@ -1620,8 +1520,7 @@ private async Task SendHelloPacketAcrossSocketsAsync( IPEndPoint sendTo, byte[] sharedKey, long minIntervalMs, - CancellationToken cancellationToken, - bool useAdvertisedSurfaceOverride = false) + CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(localSocketIds); @@ -1634,10 +1533,7 @@ await SendHelloPacketAsync( sendTo, sharedKey, minIntervalMs, - cancellationToken, - reportedSurfaceOverride: useAdvertisedSurfaceOverride && physicalDestination is not null - ? GetAdvertisedLocalSurfaceForSocket(localSocketIds[i], physicalDestination) - : null) + cancellationToken) .ConfigureAwait(false); } } @@ -1949,11 +1845,6 @@ private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEnd private bool ShouldFanOutHintedBootstrapAcrossSockets(NodeId peerNodeId) => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); - private bool ShouldForceFullHelloForHintedCandidate(NodeId peerNodeId) - => ZeroTierDirectBootstrapControlPolicy.ShouldForceFullHelloForHintedCandidate( - _multipath.AllowRootRelayFallback, - HasConfirmedDirectPath(peerNodeId)); - private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId) => ShouldUseStickyHintedPathSelection(peerNodeId) && CanUseEchoForDirectBootstrap(peerNodeId); @@ -2013,8 +1904,7 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati for (var c = 0; c < hintedCandidates.Length; c++) { var candidate = hintedCandidates[c]; - var forceFullHello = ShouldForceFullHelloForHintedCandidate(peerNodeId) || - ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId); + var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId); var minIntervalMs = forceFullHello ? (ShouldUseStickyHintedPathSelection(peerNodeId) ? DirectOnlyHintHelloIntervalMs @@ -2033,8 +1923,7 @@ await SendHelloPacketAcrossSocketsAsync( candidate.RemoteEndPoint, key, minIntervalMs, - cancellationToken, - useAdvertisedSurfaceOverride: true) + cancellationToken) .ConfigureAwait(false); continue; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index fe56387..4b0dca7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -7,10 +7,4 @@ public static bool ShouldRefreshRelayedRootControl( bool hasHintedDirectEndpoints, bool hasPinnedRendezvousEndpoints) => allowRootRelayFallback || !hasHintedDirectEndpoints || !hasPinnedRendezvousEndpoints; - - public static bool ShouldForceFullHelloForHintedCandidate(bool allowRootRelayFallback, bool hasConfirmedDirectPath) - => !allowRootRelayFallback && !hasConfirmedDirectPath; - - public static bool ShouldRestrictToFirstHintedEndpoint(bool allowRootRelayFallback, bool hasConfirmedDirectPath) - => !allowRootRelayFallback && !hasConfirmedDirectPath; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 8e65694..de30b6d 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -142,7 +142,7 @@ await _handleDirectEndpointHintAsync( _remoteNodeId, receivedLocalSocketId, endpoint, - true, + false, false, cancellationToken) .ConfigureAwait(false); @@ -212,7 +212,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); } } From cd6238fb3a07de0e0cb557c4c82bfba7bc2546ff Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:51:38 +0100 Subject: [PATCH 239/296] Refresh pinned rendezvous hole punches --- ...irectEndpointManagerSocketAffinityTests.cs | 24 +++++++++++++++++++ ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs | 4 ++-- .../ZeroTierDataplanePeerDatagramProcessor.cs | 12 +++++++++- .../Internal/ZeroTierDataplaneRuntime.cs | 14 +++++++++++ .../Internal/ZeroTierDirectEndpointManager.cs | 16 +++++++++++++ .../Internal/ZeroTierPeerEchoManager.cs | 7 +++--- 6 files changed, 71 insertions(+), 6 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index f9e946a..8a55d33 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -58,6 +58,30 @@ await manager.HandleRendezvousFromRootAsync( Assert.Equal((endpoint, false, false, 1), hints[0]); } + [Fact] + public async Task RefreshPinnedRendezvousHolePunch_UsesRequestedSocket() + { + await using var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + + manager.RefreshPinnedRendezvousHolePunch(endpoint, preferredLocalSocketIds: [0]); + + Assert.Equal(2, udp.Sends.Count); + var refreshSend = udp.Sends[1]; + Assert.Equal(0, refreshSend.LocalSocketId); + Assert.Equal(endpoint, refreshSend.RemoteEndPoint); + Assert.Equal(2, refreshSend.HopLimit); + } + [Fact] public async Task PushDirectPaths_DoesNotPersistReceivingSocketAffinity_ForNewEndpoints() { diff --git a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs index 30036e9..8c0215d 100644 --- a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs @@ -36,7 +36,7 @@ public async Task EchoProbe_UsesOnWirePacketId_AndUpdatesRttOnOk() now += 25; Span okTail = stackalloc byte[0]; - mgr.HandleEchoOk(peerNodeId, localSocketId: 1, endpoint, onWireEchoPacketId, okTail); + Assert.True(mgr.HandleEchoOk(peerNodeId, localSocketId: 1, endpoint, onWireEchoPacketId, okTail)); Assert.True(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, endpoint, out var rttMs)); Assert.Equal(25, rttMs); @@ -136,7 +136,7 @@ public async Task EchoOk_AcceptsReplyKeyMatch_FromRemappedEndpoint() var remappedReplyPacketId = (onWireEchoPacketId & 0xffffffff00000000UL) | 0x00000000000000a5UL; now += 25; - mgr.HandleEchoOk(peerNodeId, localSocketId: 1, replyEndpoint, remappedReplyPacketId, ReadOnlySpan.Empty); + Assert.True(mgr.HandleEchoOk(peerNodeId, localSocketId: 1, replyEndpoint, remappedReplyPacketId, ReadOnlySpan.Empty)); Assert.True(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, replyEndpoint, out var rttMs)); Assert.Equal(25, rttMs); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs index 3dc399a..ca7c4ca 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerDatagramProcessor.cs @@ -238,7 +238,17 @@ await _peerEcho if (inReVerb == ZeroTierVerb.Echo) { var inRePacketId = BinaryPrimitives.ReadUInt64BigEndian(payloadSpan.Slice(1, 8)); - _peerEcho.HandleEchoOk(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, inRePacketId, payloadSpan.Slice(1 + 8)); + var matchedPending = _peerEcho.HandleEchoOk( + peerNodeId, + datagram.LocalSocketId, + datagram.RemoteEndPoint, + inRePacketId, + payloadSpan.Slice(1 + 8)); + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] RX OK(ECHO): peer={peerNodeId} via={datagram.RemoteEndPoint} hop={decoded.Header.HopCount} inRe=0x{inRePacketId:x16} localSocket={datagram.LocalSocketId} matched={matchedPending}."); + } return; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index fbae194..6ab68b0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1010,6 +1010,8 @@ private async Task ProbeHintedDirectEndpointsAsync( endpointsToProbe[i], includeFallbackLocalSockets: true); + RefreshPinnedRendezvousHolePunch(peerNodeId, endpointsToProbe[i], localSocketIds); + if (forceFullHello && localSocketIds.Length > 1) { await SendHelloPacketAcrossSocketsAsync( @@ -1667,6 +1669,16 @@ private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoi return [socketIds[0]]; } + private void RefreshPinnedRendezvousHolePunch(NodeId peerNodeId, IPEndPoint endpoint, int[] localSocketIds) + { + if (_multipath.AllowRootRelayFallback || localSocketIds.Length == 0) + { + return; + } + + GetOrCreateDirectEndpointManager(peerNodeId).RefreshPinnedRendezvousHolePunch(endpoint, localSocketIds); + } + private int? GetPathLatencyMsOrNull(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) { if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) @@ -1914,6 +1926,8 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, candidate.RemoteEndPoint) : [candidate.LocalSocketId]; + RefreshPinnedRendezvousHolePunch(peerNodeId, candidate.RemoteEndPoint, localSocketIds); + if (forceFullHello && localSocketIds.Length > 1) { await SendHelloPacketAcrossSocketsAsync( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index de30b6d..7835d8c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -102,6 +102,22 @@ public bool IsPinnedRendezvousEndpoint(IPEndPoint endpoint) } } + public void RefreshPinnedRendezvousHolePunch(IPEndPoint endpoint, int[] preferredLocalSocketIds) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentNullException.ThrowIfNull(preferredLocalSocketIds); + + if (!IsPinnedRendezvousEndpoint(endpoint)) + { + return; + } + + TrySendHolePunch( + endpoint, + preferredLocalSocketIds, + hopLimit: RendezvousHolePunchHopLimit); + } + public async ValueTask HandleRendezvousFromRootAsync( ReadOnlyMemory payload, int receivedLocalSocketId, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs index adad13c..2f29bf1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs @@ -151,7 +151,7 @@ public async ValueTask HandleEchoRequestAsync( await _udp.SendAsync(localSocketId, remoteEndPoint, packet, cancellationToken).ConfigureAwait(false); } - public void HandleEchoOk( + public bool HandleEchoOk( NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, @@ -162,18 +162,19 @@ public void HandleEchoOk( if (!TryTakePendingEcho(peerNodeId, inRePacketId, out var pending)) { - return; + return false; } var now = _nowUnixMs(); var rtt = unchecked(now - pending.TimestampUnixMs); if (rtt < 0 || rtt > int.MaxValue) { - return; + return false; } var observedPath = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); _lastRttMsByPath[observedPath] = new RttEntry((int)rtt, LastUpdatedMs: now); + return true; } private bool TryTakePendingEcho(NodeId peerNodeId, ulong inRePacketId, out PendingEcho pending) From 659b0128a9b1364191e0cd6b7e19229ccd29f843 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:56:08 +0100 Subject: [PATCH 240/296] Clamp direct-only traffic to pinned rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 93 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 35 ++++++- 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 8817c64..67b056d 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -400,6 +400,99 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP Assert.Equal(expectedFanout, payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnPinnedRendezvousPath() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), + TimeSpan.FromSeconds(2)); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); + + var directSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) + .ToArray(); + + Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); + + var echoSends = directSends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .ToArray(); + var payloadSends = directSends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .ToArray(); + + Assert.Equal([(1, rendezvousEndpoint)], echoSends); + Assert.Equal([(1, rendezvousEndpoint)], payloadSends); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingEcho_UsesHintedPeerEndpoint() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 6ab68b0..96cd15d 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -665,13 +665,19 @@ private bool TryGetDirectOnlyHintedPayloadFanout( } var unique = new HashSet(); - var candidates = new List(hinted.Length * 2); - for (var i = 0; i < hinted.Length; i++) + var restrictToPinnedRendezvous = ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); + var selectedEndpoints = restrictToPinnedRendezvous + ? [hinted[0]] + : hinted; + var candidates = new List(selectedEndpoints.Length * 2); + for (var i = 0; i < selectedEndpoints.Length; i++) { - var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); + var socketIds = restrictToPinnedRendezvous + ? GetStickyHintedLocalSocketIds(peerNodeId, selectedEndpoints[i]) + : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, selectedEndpoints[i]); for (var s = 0; s < socketIds.Length; s++) { - var candidate = new ZeroTierSelectedPeerPath(socketIds[s], hinted[i]); + var candidate = new ZeroTierSelectedPeerPath(socketIds[s], selectedEndpoints[i]); if (!unique.Add(new ZeroTierPeerPhysicalPathKey(candidate.LocalSocketId, candidate.RemoteEndPoint))) { continue; @@ -1316,6 +1322,23 @@ private async ValueTask HandleDirectEndpointHintAsync( return; } + if (ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (!directEndpoints.IsPinnedRendezvousEndpoint(endpoint)) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap skipped: peer={peerNodeId} endpoint={endpoint} reason=pinned rendezvous path is preferred."); + } + + return; + } + + useAllEligibleLocalSockets = false; + } + byte[] sharedKey; try { @@ -1653,6 +1676,10 @@ private bool TrySelectConfirmedHintedDirectPath( private bool ShouldUseStickyHintedPathSelection(NodeId peerNodeId) => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + private bool ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(NodeId peerNodeId) + => ShouldUseStickyHintedPathSelection(peerNodeId) && + GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints; + private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoint) { var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); From 713a298e505f270f362934130358bb4b9fec30d4 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:01:27 +0100 Subject: [PATCH 241/296] Keep pinned rendezvous maintenance on one socket --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 82 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 20 +++-- 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 67b056d..2970c81 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -493,6 +493,88 @@ await WaitForConditionAsync( Assert.Equal([(1, rendezvousEndpoint)], payloadSends); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_MaintenanceStaysOnPinnedRendezvousSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var directSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) + .ToArray(); + + Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); + + var verbs = directSends + .Where(send => send.Payload.Length > 4) + .Select(send => new + { + send.LocalSocketId, + send.RemoteEndPoint, + Verb = TryDecodeVerb(send.Payload, sharedKey, out var verb) ? verb : (ZeroTierVerb?)null + }) + .Where(send => send.Verb is ZeroTierVerb.Echo or ZeroTierVerb.Hello) + .ToArray(); + + Assert.NotEmpty(verbs); + Assert.All(verbs, send => + { + Assert.Equal(1, send.LocalSocketId); + Assert.Equal(rendezvousEndpoint, send.RemoteEndPoint); + }); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingEcho_UsesHintedPeerEndpoint() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 96cd15d..c3b987b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1008,9 +1008,7 @@ private async Task ProbeHintedDirectEndpointsAsync( ? DirectOnlyHintHelloIntervalMs : DirectHelloMinIntervalMs; var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) - ? (forceFullHello - ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpointsToProbe[i]) - : GetStickyHintedLocalSocketIds(peerNodeId, endpointsToProbe[i])) + ? GetStickyDirectHintProbeSocketIds(peerNodeId, endpointsToProbe[i], forceFullHello) : _directHintPlanner.GetRotatingSocketIds( peerNodeId, endpointsToProbe[i], @@ -1680,6 +1678,18 @@ private bool ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(NodeId peer => ShouldUseStickyHintedPathSelection(peerNodeId) && GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints; + private int[] GetStickyDirectHintProbeSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) + { + if (ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + } + + return forceFullHello + ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint) + : GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + } + private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoint) { var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); @@ -1949,8 +1959,8 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati ? DirectOnlyHintHelloIntervalMs : DirectHintFullHelloIntervalMs) : DirectHintFullHelloIntervalMs; - var localSocketIds = forceFullHello && ShouldUseStickyHintedPathSelection(peerNodeId) - ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, candidate.RemoteEndPoint) + var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) + ? GetStickyDirectHintProbeSocketIds(peerNodeId, candidate.RemoteEndPoint, forceFullHello) : [candidate.LocalSocketId]; RefreshPinnedRendezvousHolePunch(peerNodeId, candidate.RemoteEndPoint, localSocketIds); From fc8f0fd9204e91ad64745e78a706339c5b736b12 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:13:00 +0100 Subject: [PATCH 242/296] Report local surface in direct HELLO --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 36 ++++++------- .../Internal/ZeroTierDataplaneRuntime.cs | 50 ++++++++++++++++++- 2 files changed, 66 insertions(+), 20 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 2970c81..7b7a34b 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -107,7 +107,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint() + public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsAdvertisedLocalSurface() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -120,6 +120,9 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint( var rootNodeId = new NodeId(0x1111111111); var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var advertisedSurface = new ZeroTierExternalSurfaceObservation( + 1, + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)); var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime( @@ -129,7 +132,8 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint( rootKey, multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, planetId: 1, - planetTimestamp: 1); + planetTimestamp: 1, + initialExternalSurfaceObservations: [advertisedSurface]); runtime.PrimePeerForTests(peerIdentity, peerProtocolVersion: 4); @@ -157,7 +161,7 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint( verb == ZeroTierVerb.Hello, TimeSpan.FromSeconds(2)); - Assert.Equal(rendezvousEndpoint, ReadReportedHelloSurface(helloSend.Payload, sharedKey)); + Assert.Equal(advertisedSurface.SurfaceAddress, ReadReportedHelloSurface(helloSend.Payload, sharedKey)); } [Fact] @@ -556,19 +560,8 @@ await WaitForConditionAsync( Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); - var verbs = directSends - .Where(send => send.Payload.Length > 4) - .Select(send => new - { - send.LocalSocketId, - send.RemoteEndPoint, - Verb = TryDecodeVerb(send.Payload, sharedKey, out var verb) ? verb : (ZeroTierVerb?)null - }) - .Where(send => send.Verb is ZeroTierVerb.Echo or ZeroTierVerb.Hello) - .ToArray(); - - Assert.NotEmpty(verbs); - Assert.All(verbs, send => + Assert.NotEmpty(directSends); + Assert.All(directSends, send => { Assert.Equal(1, send.LocalSocketId); Assert.Equal(rendezvousEndpoint, send.RemoteEndPoint); @@ -831,7 +824,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHello } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsHintedPeerEndpoint() + public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsAdvertisedLocalSurface() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -881,7 +874,14 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsHintedPeer .OrderBy(send => send.LocalSocketId) .ToArray(); - Assert.All(helloSurfaces, send => Assert.Equal(send.RemoteEndPoint, send.Surface)); + Assert.Equal( + advertisedSurfaces + .Select(observation => (observation.LocalSocketId, Surface: (IPEndPoint?)observation.SurfaceAddress)) + .OrderBy(entry => entry.LocalSocketId) + .ToArray(), + helloSurfaces + .Select(send => (send.LocalSocketId, send.Surface)) + .ToArray()); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c3b987b..dc90a47 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1189,6 +1189,48 @@ private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); } + private IPEndPoint? GetReportedDirectHelloSurface(int localSocketId, AddressFamily remoteAddressFamily) + { + var observedSurfaces = _surfaceAddresses.GetSnapshot(localSocketId); + for (var i = 0; i < observedSurfaces.Length; i++) + { + var surface = observedSurfaces[i]; + if (surface.Port != 0 && surface.Address.AddressFamily == remoteAddressFamily) + { + return surface; + } + } + + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return null; + } + + for (var i = 0; i < localSockets.Count; i++) + { + var socket = localSockets[i]; + if (socket.Id != localSocketId) + { + continue; + } + + var advertisements = _localDirectPathAdvertisementsSource.GetSnapshot([socket]); + for (var a = 0; a < advertisements.Length; a++) + { + var advertisement = advertisements[a]; + if (advertisement.Port != 0 && advertisement.Address.AddressFamily == remoteAddressFamily) + { + return advertisement; + } + } + + break; + } + + return null; + } + private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken) { var localSockets = _udp.LocalSockets; @@ -1481,15 +1523,19 @@ private async Task SendHelloPacketAsync( try { + var reportedSurface = physicalDestination is null + ? null + : GetReportedDirectHelloSurface(localSocketId, sendTo.Address.AddressFamily); + if (ZeroTierTrace.Enabled) { - ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, reported={physicalDestination})."); + ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, reported={reportedSurface})."); } var packet = ZeroTierHelloPacketBuilder.BuildPacket( _localIdentity, peerNodeId, - physicalDestination, + reportedSurface, timestamp: (ulong)Environment.TickCount64, _planetId, _planetTimestamp, From 61ef90480d4592b9a94cc88cca535144f4827112 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:17:12 +0100 Subject: [PATCH 243/296] Clamp direct ads to pinned rendezvous socket --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 88 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 24 +++++ 2 files changed, 112 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 7b7a34b..34373b7 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -934,6 +934,79 @@ public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvous Assert.Empty(rendezvousEndpoints); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_AdvertisesPinnedRendezvousSurfaceOnly() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var pushDirectSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.PushDirectPaths) + .Select(send => ReadPushedDirectPathEndpoints(send.Payload, sharedKey)) + .ToArray(); + + Assert.NotEmpty(pushDirectSends); + Assert.All(pushDirectSends, paths => Assert.Equal([advertisedSurfaces[1].SurfaceAddress], paths)); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { @@ -1190,6 +1263,21 @@ private static IPEndPoint ReadRendezvousEndpoint(byte[] packet, byte[] sharedKey return rendezvous.Endpoint; } + private static IPEndPoint[] ReadPushedDirectPathEndpoints(byte[] packet, byte[] sharedKey) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed) || + !ZeroTierPushDirectPathsCodec.TryParse(parsed.Payload.Span, out var paths)) + { + throw new InvalidOperationException("Failed to decode PUSH_DIRECT_PATHS packet."); + } + + return paths + .Select(path => path.Endpoint) + .ToArray(); + } + private static byte[] BuildPushDirectPathsPacket( NodeId source, NodeId destination, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index dc90a47..f731aa9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1185,10 +1185,34 @@ private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId { var localAdvertisements = GetLocalDirectPathAdvertisements(); var hintedPeerEndpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + var pinnedAdvertisements = GetPinnedRendezvousLocalDirectPathAdvertisements(peerNodeId, hintedPeerEndpoints); + if (pinnedAdvertisements.Length != 0) + { + return pinnedAdvertisements; + } + var observedPeerPaths = _peerPaths.GetSnapshot(peerNodeId); return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); } + private IPEndPoint[] GetPinnedRendezvousLocalDirectPathAdvertisements(NodeId peerNodeId, IPEndPoint[] hintedPeerEndpoints) + { + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) || hintedPeerEndpoints.Length == 0) + { + return Array.Empty(); + } + + var pinnedEndpoint = hintedPeerEndpoints[0]; + var stickySocketIds = GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoint); + if (stickySocketIds.Length == 0) + { + return Array.Empty(); + } + + var reportedSurface = GetReportedDirectHelloSurface(stickySocketIds[0], pinnedEndpoint.Address.AddressFamily); + return reportedSurface is null ? Array.Empty() : [reportedSurface]; + } + private IPEndPoint? GetReportedDirectHelloSurface(int localSocketId, AddressFamily remoteAddressFamily) { var observedSurfaces = _surfaceAddresses.GetSnapshot(localSocketId); From 97a52683cc6696f7a93440108353285a514abc2c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:20:00 +0100 Subject: [PATCH 244/296] Pin relayed control to rendezvous socket --- ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs | 8 ++++++-- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 34373b7..051f033 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -1000,11 +1000,15 @@ await WaitForConditionAsync( .Skip(initialSendCount) .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.PushDirectPaths) - .Select(send => ReadPushedDirectPathEndpoints(send.Payload, sharedKey)) + .Select(send => (send.LocalSocketId, Paths: ReadPushedDirectPathEndpoints(send.Payload, sharedKey))) .ToArray(); Assert.NotEmpty(pushDirectSends); - Assert.All(pushDirectSends, paths => Assert.Equal([advertisedSurfaces[1].SurfaceAddress], paths)); + Assert.All(pushDirectSends, send => + { + Assert.Equal(1, send.LocalSocketId); + Assert.Equal([advertisedSurfaces[1].SurfaceAddress], send.Paths); + }); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index f731aa9..4a3127a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1282,7 +1282,9 @@ private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet } private bool ShouldUsePeerRootSocketAffinity(NodeId peerNodeId) - => _multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId); + => _multipath.AllowRootRelayFallback || + HasConfirmedDirectPath(peerNodeId) || + ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { From 3ba2f505c83e1ae202212129f2a37904b02d1576 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:23:41 +0100 Subject: [PATCH 245/296] Restore upstream HELLO endpoint semantics --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 21 +++++-------------- .../Internal/ZeroTierDataplaneRuntime.cs | 4 +--- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 051f033..e0ef4fd 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -107,7 +107,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsAdvertisedLocalSurface() + public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -120,9 +120,6 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsAdvertisedLoc var rootNodeId = new NodeId(0x1111111111); var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); - var advertisedSurface = new ZeroTierExternalSurfaceObservation( - 1, - new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)); var rootKey = RandomNumberGenerator.GetBytes(48); await using var runtime = CreateRuntime( @@ -132,8 +129,7 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsAdvertisedLoc rootKey, multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, planetId: 1, - planetTimestamp: 1, - initialExternalSurfaceObservations: [advertisedSurface]); + planetTimestamp: 1); runtime.PrimePeerForTests(peerIdentity, peerProtocolVersion: 4); @@ -161,7 +157,7 @@ public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsAdvertisedLoc verb == ZeroTierVerb.Hello, TimeSpan.FromSeconds(2)); - Assert.Equal(advertisedSurface.SurfaceAddress, ReadReportedHelloSurface(helloSend.Payload, sharedKey)); + Assert.Equal(rendezvousEndpoint, ReadReportedHelloSurface(helloSend.Payload, sharedKey)); } [Fact] @@ -824,7 +820,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHello } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsAdvertisedLocalSurface() + public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsHintedPeerEndpoint() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -874,14 +870,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsAdvertised .OrderBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal( - advertisedSurfaces - .Select(observation => (observation.LocalSocketId, Surface: (IPEndPoint?)observation.SurfaceAddress)) - .OrderBy(entry => entry.LocalSocketId) - .ToArray(), - helloSurfaces - .Select(send => (send.LocalSocketId, send.Surface)) - .ToArray()); + Assert.All(helloSurfaces, send => Assert.Equal(send.RemoteEndPoint, send.Surface)); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 4a3127a..39f12d6 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1549,9 +1549,7 @@ private async Task SendHelloPacketAsync( try { - var reportedSurface = physicalDestination is null - ? null - : GetReportedDirectHelloSurface(localSocketId, sendTo.Address.AddressFamily); + var reportedSurface = physicalDestination; if (ZeroTierTrace.Enabled) { From 06485dd277125178d00740ef530161d897aa5d5c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:29:47 +0100 Subject: [PATCH 246/296] Match upstream rendezvous hole-punch size --- .../ZeroTierDataplaneRuntimeDirectPathTests.cs | 16 ++++++++-------- .../Internal/ZeroTierDirectEndpointManager.cs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index e0ef4fd..18a356d 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -43,7 +43,7 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() var holePunch = await WaitForSendAsync( udp, - send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4, + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 8, TimeSpan.FromSeconds(2)); Assert.Equal(0, holePunch.LocalSocketId); @@ -97,13 +97,13 @@ await WaitForConditionAsync( () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length > 4), TimeSpan.FromSeconds(2)); - var firstBootstrapSend = udp.GetSendsSnapshot() + var bootstrapSends = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) - .OrderBy(send => send.LocalSocketId) - .First(); + .Where(send => send.Payload.Length > 8) + .ToArray(); - Assert.Equal(1, firstBootstrapSend.LocalSocketId); + Assert.NotEmpty(bootstrapSends); + Assert.All(bootstrapSends, send => Assert.Equal(1, send.LocalSocketId)); } [Fact] @@ -449,7 +449,7 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnPinnedRendezvous BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 8), TimeSpan.FromSeconds(2)); var tcp = TcpCodec.Encode( @@ -542,7 +542,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceStaysOnPinnedRendezvous BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 8), TimeSpan.FromSeconds(2)); var initialSendCount = udp.GetSendsSnapshot().Length; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 7835d8c..7e8a4a8 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -379,7 +379,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId var now = Environment.TickCount64; _holePunchLimiter.CleanupIfNeeded(now); - var junk = new byte[4]; + var junk = new byte[8]; RandomNumberGenerator.Fill(junk); if (preferredLocalSocketIds is { Length: > 0 }) From c395f93dd9c50c3fd469999eaa6ef8d21ccedab9 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:34:32 +0100 Subject: [PATCH 247/296] Reduce ordinary push-direct socket fanout --- ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs | 2 +- ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index eae8816..9b89482 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -180,7 +180,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Equal((endpoint, false, false, 1), hints[0]); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 7e8a4a8..773a097 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -228,7 +228,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); } } From 99d680425739f8d753bf2dbd5b4b86b7f24b1233 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:39:43 +0100 Subject: [PATCH 248/296] Honor single-socket ordinary direct hints --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 4 +-- .../ZeroTierDirectHintPathPlannerTests.cs | 25 +++++++++++++++++++ .../Internal/ZeroTierDirectHintPathPlanner.cs | 14 ++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 18a356d..93eeeb0 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -94,12 +94,12 @@ public async Task DataplaneRuntime_RendezvousBootstrap_UsesReceivingSocket_ForMo udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length > 4), + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.HopLimit is null), TimeSpan.FromSeconds(2)); var bootstrapSends = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) - .Where(send => send.Payload.Length > 8) + .Where(send => send.HopLimit is null) .ToArray(); Assert.NotEmpty(bootstrapSends); diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs index a36dc23..fb2e9a3 100644 --- a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs @@ -1,5 +1,6 @@ using System.Net; using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; namespace ZTSharp.Tests; @@ -74,6 +75,30 @@ public void GetPreferredAndFallbackSocketIds_ReturnsEmpty_WhenNoSocketCanReachEn Assert.Empty(sockets); } + [Fact] + public async Task GetPreferredSocketIds_ReturnsOnlyPersistedPreferredSockets() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner(udp, _ => manager); + + Assert.Empty(planner.GetPreferredSocketIds(peerNodeId, endpoint)); + + await manager.HandleRendezvousFromRootAsync( + ZeroTierRendezvousCodec.BuildPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: new IPEndPoint(IPAddress.Loopback, 9993), + CancellationToken.None); + + Assert.Equal([1], planner.GetPreferredSocketIds(peerNodeId, endpoint)); + Assert.Equal([1, 0], planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint)); + } + private sealed class StubUdpTransport : IZeroTierUdpTransport { public StubUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs index 7350f90..05e12a9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -41,7 +41,19 @@ public int[] GetRotatingSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool i } public int[] GetPreferredSocketIds(NodeId peerNodeId, IPEndPoint endpoint) - => GetSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: false); + { + endpoint = Canonicalize(endpoint); + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return Array.Empty(); + } + + var preferred = new List(localSockets.Count); + var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); + AddSocketIds(preferred, preferredFromHints, localSockets, endpoint, _socketAdmissibility); + return preferred.ToArray(); + } public int[] GetPreferredAndFallbackSocketIds(NodeId peerNodeId, IPEndPoint endpoint) => GetSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true); From 4a0bab818dd719c924a5b7d4c4cf306b9bbf7c4e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:43:59 +0100 Subject: [PATCH 249/296] Narrow unresolved direct-only payload fanout --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 45 +++++-------------- .../Internal/ZeroTierDataplaneRuntime.cs | 18 ++++++-- 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 93eeeb0..2f66691 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -368,16 +368,6 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); - var expectedFanout = hintedEndpoints - .SelectMany(endpoint => new[] - { - (LocalSocketId: 0, RemoteEndPoint: endpoint), - (LocalSocketId: 1, RemoteEndPoint: endpoint) - }) - .OrderBy(item => item.RemoteEndPoint.Port) - .ThenBy(item => item.LocalSocketId) - .ToArray(); - var sends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .ToArray(); @@ -396,8 +386,10 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP .OrderBy(send => send.RemoteEndPoint.Port) .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal(expectedFanout, echoSends); - Assert.Equal(expectedFanout, payloadSends.Select(send => (send.LocalSocketId, send.RemoteEndPoint)).ToArray()); + Assert.Equal(hintedEndpoints, echoSends.Select(send => send.RemoteEndPoint).ToArray()); + Assert.Equal(hintedEndpoints, payloadSends.Select(send => send.RemoteEndPoint).ToArray()); + Assert.All(echoSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); + Assert.All(payloadSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); } [Fact] @@ -634,17 +626,8 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingEcho_UsesHintedP .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal( - hintedEndpoints - .SelectMany(endpoint => new[] - { - (LocalSocketId: 0, RemoteEndPoint: endpoint), - (LocalSocketId: 1, RemoteEndPoint: endpoint) - }) - .OrderBy(send => send.RemoteEndPoint.Port) - .ThenBy(send => send.LocalSocketId) - .ToArray(), - echoSends); + Assert.Equal(hintedEndpoints, echoSends.Select(send => send.RemoteEndPoint).ToArray()); + Assert.All(echoSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); } [Fact] @@ -706,16 +689,6 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur await sendTask.WaitAsync(TimeSpan.FromSeconds(5)); - var expectedFanout = hintedEndpoints - .SelectMany(endpoint => new[] - { - (LocalSocketId: 0, RemoteEndPoint: endpoint), - (LocalSocketId: 1, RemoteEndPoint: endpoint) - }) - .OrderBy(item => item.RemoteEndPoint.Port) - .ThenBy(item => item.LocalSocketId) - .ToArray(); - var sends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .ToArray(); @@ -737,8 +710,10 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal(expectedFanout, echoSends); - Assert.Equal(expectedFanout, payloadSends); + Assert.Equal(hintedEndpoints, echoSends.Select(send => send.RemoteEndPoint).ToArray()); + Assert.Equal(hintedEndpoints, payloadSends.Select(send => send.RemoteEndPoint).ToArray()); + Assert.All(echoSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); + Assert.All(payloadSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 39f12d6..3f1edcb 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -672,9 +672,10 @@ private bool TryGetDirectOnlyHintedPayloadFanout( var candidates = new List(selectedEndpoints.Length * 2); for (var i = 0; i < selectedEndpoints.Length; i++) { - var socketIds = restrictToPinnedRendezvous - ? GetStickyHintedLocalSocketIds(peerNodeId, selectedEndpoints[i]) - : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, selectedEndpoints[i]); + var socketIds = GetDirectOnlyHintedPayloadSocketIds( + peerNodeId, + selectedEndpoints[i], + restrictToPinnedRendezvous); for (var s = 0; s < socketIds.Length; s++) { var candidate = new ZeroTierSelectedPeerPath(socketIds[s], selectedEndpoints[i]); @@ -696,6 +697,17 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return true; } + private int[] GetDirectOnlyHintedPayloadSocketIds( + NodeId peerNodeId, + IPEndPoint endpoint, + bool restrictToPinnedRendezvous) + => restrictToPinnedRendezvous + ? GetStickyHintedLocalSocketIds(peerNodeId, endpoint) + : _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true); + private async ValueTask TrySendDirectOnlyHintedPayloadAsync( NodeId peerNodeId, ReadOnlyMemory packet, From e3b99025f8e055dcc381bbc543d6aeb92691bcc0 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:46:40 +0100 Subject: [PATCH 250/296] Keep unresolved direct-only hinted HELLOs on one socket --- ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs | 4 ++-- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 2f66691..af20fc2 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -739,7 +739,7 @@ public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() } [Fact] - public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHelloAcrossSockets_ForModernPeers() + public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHelloOnSingleSocket_ForModernPeers() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -789,7 +789,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHello .Where(send => send.Verb == ZeroTierVerb.Hello) .ToArray(); - Assert.Equal(new[] { 0, 1 }, helloFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); + Assert.Equal(new[] { 0 }, helloFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); Assert.Equal(new[] { hintedEndpoints[0] }, helloFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Echo); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 3f1edcb..fb3cc7e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1767,9 +1767,7 @@ private int[] GetStickyDirectHintProbeSocketIds(NodeId peerNodeId, IPEndPoint en return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); } - return forceFullHello - ? _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint) - : GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); } private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoint) From 32317343410ce85d615e40f6c595dbc477c215a5 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:50:49 +0100 Subject: [PATCH 251/296] Keep ordinary direct hints on root-affine sockets --- ...TierDirectEndpointManagerPushFlagsTests.cs | 11 +++++---- ...irectEndpointManagerSocketAffinityTests.cs | 4 ++-- .../Internal/ZeroTierDataplaneRuntime.cs | 23 ++++++++++++++----- .../Internal/ZeroTierDirectEndpointManager.cs | 1 + 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 9b89482..1898c81 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -55,7 +55,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_KnownEndpoint_IsRebootstrappedWithoutPersistingSocketAffinity() + public async Task PushDirectPaths_KnownEndpoint_ReusesObservedSocketAffinity() { var udp = new RecordingUdpTransport(); @@ -79,12 +79,12 @@ public async Task PushDirectPaths_KnownEndpoint_IsRebootstrappedWithoutPersistin await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 1, CancellationToken.None); Assert.Equal(new[] { endpoint, endpoint }, hintedEndpoints); - Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal(new[] { 0, 1 }, manager.GetPreferredLocalSocketIds(endpoint).Order().ToArray()); Assert.Empty(udp.Sends); } [Fact] - public async Task PushDirectPaths_NewEndpoint_DoesNotPersistReceivingSocketAffinityOrHolePunch() + public async Task PushDirectPaths_NewEndpoint_RemembersReceivingSocketWithoutHolePunch() { var udp = new RecordingUdpTransport(); @@ -98,7 +98,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } @@ -125,7 +125,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Equal(newEndpoint, manager.Endpoints[0]); Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(oldEndpoint)); - Assert.Empty(manager.GetPreferredLocalSocketIds(newEndpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(newEndpoint)); } [Fact] @@ -181,6 +181,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Single(hints); Assert.Equal((endpoint, false, false, 1), hints[0]); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index 8a55d33..dd3bed6 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -83,7 +83,7 @@ await manager.HandleRendezvousFromRootAsync( } [Fact] - public async Task PushDirectPaths_DoesNotPersistReceivingSocketAffinity_ForNewEndpoints() + public async Task PushDirectPaths_RemembersReceivingSocketAffinity_ForNewEndpoints() { await using var udp = new RecordingUdpTransport(); var relay = new IPEndPoint(IPAddress.Loopback, 9999); @@ -96,7 +96,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); } private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index fb3cc7e..a3f2df4 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -701,12 +701,23 @@ private int[] GetDirectOnlyHintedPayloadSocketIds( NodeId peerNodeId, IPEndPoint endpoint, bool restrictToPinnedRendezvous) - => restrictToPinnedRendezvous - ? GetStickyHintedLocalSocketIds(peerNodeId, endpoint) - : _directHintPlanner.GetRotatingSocketIds( - peerNodeId, - endpoint, - includeFallbackLocalSockets: true); + { + if (restrictToPinnedRendezvous) + { + return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + } + + var preferredSocketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); + if (preferredSocketIds.Length != 0) + { + return [preferredSocketIds[0]]; + } + + return _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true); + } private async ValueTask TrySendDirectOnlyHintedPayloadAsync( NodeId peerNodeId, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 773a097..0e84133 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -227,6 +227,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( else { add.Add(endpoint); + preferredSocketEndpoints.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); } From 6f588eb96b8c54cb33e9b52aff80b49bec39d61d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:56:06 +0100 Subject: [PATCH 252/296] Match upstream rendezvous hole-punch size --- ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs | 6 +++--- ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs | 2 +- ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index af20fc2..d9e65e2 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -43,7 +43,7 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() var holePunch = await WaitForSendAsync( udp, - send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 8, + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4, TimeSpan.FromSeconds(2)); Assert.Equal(0, holePunch.LocalSocketId); @@ -441,7 +441,7 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnPinnedRendezvous BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 8), + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), TimeSpan.FromSeconds(2)); var tcp = TcpCodec.Encode( @@ -534,7 +534,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceStaysOnPinnedRendezvous BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 8), + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), TimeSpan.FromSeconds(2)); var initialSendCount = udp.GetSendsSnapshot().Length; diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs index 44f7322..f481fb3 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs @@ -14,7 +14,7 @@ public async Task Rendezvous_UsesLowTtlHolePunch() var udp = new RecordingUdpTransport(); var relay = new IPEndPoint(IPAddress.Loopback, 9999); var peerNodeId = new NodeId(0x1111111111); - var endpoint = new IPEndPoint(IPAddress.Loopback, 4242); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); await manager.HandleRendezvousFromRootAsync( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 0e84133..dd7e212 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -380,7 +380,7 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId var now = Environment.TickCount64; _holePunchLimiter.CleanupIfNeeded(now); - var junk = new byte[8]; + var junk = new byte[4]; RandomNumberGenerator.Fill(junk); if (preferredLocalSocketIds is { Length: > 0 }) From e325c284c0dc57d264e12e646735c6c41daa6521 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:02:11 +0100 Subject: [PATCH 253/296] Probe alternate public hints while payload stays pinned --- .../ZeroTierDataplaneRuntimeDirectPathTests.cs | 14 ++++++++------ .../ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 14 ++++---------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index d9e65e2..ca0ee0d 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -468,8 +468,6 @@ await WaitForConditionAsync( .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) .ToArray(); - Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); - var echoSends = directSends .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) @@ -481,7 +479,8 @@ await WaitForConditionAsync( .Distinct() .ToArray(); - Assert.Equal([(1, rendezvousEndpoint)], echoSends); + Assert.Contains((1, rendezvousEndpoint), echoSends); + Assert.All(echoSends, send => Assert.Equal(1, send.LocalSocketId)); Assert.Equal([(1, rendezvousEndpoint)], payloadSends); } @@ -546,14 +545,17 @@ await WaitForConditionAsync( .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) .ToArray(); - Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); - Assert.NotEmpty(directSends); Assert.All(directSends, send => { Assert.Equal(1, send.LocalSocketId); - Assert.Equal(rendezvousEndpoint, send.RemoteEndPoint); }); + Assert.Contains(directSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint)); + Assert.DoesNotContain( + directSends, + send => send.RemoteEndPoint.Equals(pushedEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.ExtFrame); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index a3f2df4..b56de23 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1414,18 +1414,12 @@ private async ValueTask HandleDirectEndpointHintAsync( if (ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) { var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - if (!directEndpoints.IsPinnedRendezvousEndpoint(endpoint)) + useAllEligibleLocalSockets = false; + if (!directEndpoints.IsPinnedRendezvousEndpoint(endpoint) && ZeroTierTrace.Enabled) { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine( - $"[zerotier] Direct hint bootstrap skipped: peer={peerNodeId} endpoint={endpoint} reason=pinned rendezvous path is preferred."); - } - - return; + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} mode=alternate-hint payloadPinned=True."); } - - useAllEligibleLocalSockets = false; } byte[] sharedKey; From beeed042ff9e092cc1409c00998671a3f5410fd4 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:10:29 +0100 Subject: [PATCH 254/296] Restore upstream direct bootstrap advertisements --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 6 +- ...roTierDirectBootstrapControlPolicyTests.cs | 16 ++--- .../Internal/ZeroTierDataplaneRuntime.cs | 69 +------------------ .../ZeroTierDirectBootstrapControlPolicy.cs | 5 +- 4 files changed, 13 insertions(+), 83 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index ca0ee0d..a1b14df 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -901,7 +901,7 @@ public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvous } [Fact] - public async Task DataplaneRuntime_DirectOnly_Maintenance_AdvertisesPinnedRendezvousSurfaceOnly() + public async Task DataplaneRuntime_DirectOnly_Maintenance_AdvertisesAllApplicableSurfaces_WhilePayloadStaysPinned() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -973,7 +973,9 @@ await WaitForConditionAsync( Assert.All(pushDirectSends, send => { Assert.Equal(1, send.LocalSocketId); - Assert.Equal([advertisedSurfaces[1].SurfaceAddress], send.Paths); + Assert.Equal( + advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), + send.Paths); }); } diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 500672c..6971c02 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -5,23 +5,19 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectBootstrapControlPolicyTests { [Theory] - [InlineData(true, false, false, true)] - [InlineData(true, true, false, true)] - [InlineData(true, true, true, true)] - [InlineData(false, false, false, true)] - [InlineData(false, true, false, true)] - [InlineData(false, true, true, false)] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( bool allowRootRelayFallback, - bool hasHintedDirectEndpoints, - bool hasPinnedRendezvousEndpoints, + bool hasConfirmedDirectPath, bool expected) { Assert.Equal( expected, ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( allowRootRelayFallback, - hasHintedDirectEndpoints, - hasPinnedRendezvousEndpoints)); + hasConfirmedDirectPath)); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index b56de23..2ae36af 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -951,8 +951,7 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextRootHelloAt) >= 0 && ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( _multipath.AllowRootRelayFallback, - hinted.Length != 0, - directEndpoints.HasPinnedRendezvousEndpoints)) + HasConfirmedDirectPath(peerNodeId))) { await SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); await SendNetworkCredentialsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); @@ -1208,76 +1207,10 @@ private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId { var localAdvertisements = GetLocalDirectPathAdvertisements(); var hintedPeerEndpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; - var pinnedAdvertisements = GetPinnedRendezvousLocalDirectPathAdvertisements(peerNodeId, hintedPeerEndpoints); - if (pinnedAdvertisements.Length != 0) - { - return pinnedAdvertisements; - } - var observedPeerPaths = _peerPaths.GetSnapshot(peerNodeId); return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); } - private IPEndPoint[] GetPinnedRendezvousLocalDirectPathAdvertisements(NodeId peerNodeId, IPEndPoint[] hintedPeerEndpoints) - { - if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) || hintedPeerEndpoints.Length == 0) - { - return Array.Empty(); - } - - var pinnedEndpoint = hintedPeerEndpoints[0]; - var stickySocketIds = GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoint); - if (stickySocketIds.Length == 0) - { - return Array.Empty(); - } - - var reportedSurface = GetReportedDirectHelloSurface(stickySocketIds[0], pinnedEndpoint.Address.AddressFamily); - return reportedSurface is null ? Array.Empty() : [reportedSurface]; - } - - private IPEndPoint? GetReportedDirectHelloSurface(int localSocketId, AddressFamily remoteAddressFamily) - { - var observedSurfaces = _surfaceAddresses.GetSnapshot(localSocketId); - for (var i = 0; i < observedSurfaces.Length; i++) - { - var surface = observedSurfaces[i]; - if (surface.Port != 0 && surface.Address.AddressFamily == remoteAddressFamily) - { - return surface; - } - } - - var localSockets = _udp.LocalSockets; - if (localSockets.Count == 0) - { - return null; - } - - for (var i = 0; i < localSockets.Count; i++) - { - var socket = localSockets[i]; - if (socket.Id != localSocketId) - { - continue; - } - - var advertisements = _localDirectPathAdvertisementsSource.GetSnapshot([socket]); - for (var a = 0; a < advertisements.Length; a++) - { - var advertisement = advertisements[a]; - if (advertisement.Port != 0 && advertisement.Address.AddressFamily == remoteAddressFamily) - { - return advertisement; - } - } - - break; - } - - return null; - } - private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken) { var localSockets = _udp.LocalSockets; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index 4b0dca7..f9ed9f0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -4,7 +4,6 @@ internal static class ZeroTierDirectBootstrapControlPolicy { public static bool ShouldRefreshRelayedRootControl( bool allowRootRelayFallback, - bool hasHintedDirectEndpoints, - bool hasPinnedRendezvousEndpoints) - => allowRootRelayFallback || !hasHintedDirectEndpoints || !hasPinnedRendezvousEndpoints; + bool hasConfirmedDirectPath) + => allowRootRelayFallback || !hasConfirmedDirectPath; } From 6e0ab6c62a9e243265a2661cad01f6fa4e120359 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:15:42 +0100 Subject: [PATCH 255/296] Restore upstream ordinary direct hint fanout --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 35 ++++++++++++++++--- ...TierDirectEndpointManagerPushFlagsTests.cs | 12 +++---- ...irectEndpointManagerSocketAffinityTests.cs | 4 +-- .../Internal/ZeroTierDataplaneRuntime.cs | 7 ++-- .../Internal/ZeroTierDirectEndpointManager.cs | 3 +- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index a1b14df..f13549a 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -307,9 +307,34 @@ public async Task DataplaneRuntime_DirectOnly_PrefersRendezvousEndpoint_AfterLat await WaitForConditionAsync( () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), TimeSpan.FromSeconds(2)); + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + send.RemoteEndPoint.Equals(pushedEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo), + TimeSpan.FromSeconds(2)); var endpoints = runtime.GetDirectEndpointsForTests(peerIdentity.NodeId); Assert.Equal(rendezvousEndpoint, endpoints[0]); + + var pushedBootstrapSockets = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + Assert.Equal(new[] { 0, 1 }, pushedBootstrapSockets); + + var rendezvousBootstrapSockets = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + (verb == ZeroTierVerb.Echo || verb == ZeroTierVerb.Hello)) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + Assert.Equal(new[] { 1 }, rendezvousBootstrapSockets); } [Fact] @@ -480,7 +505,8 @@ await WaitForConditionAsync( .ToArray(); Assert.Contains((1, rendezvousEndpoint), echoSends); - Assert.All(echoSends, send => Assert.Equal(1, send.LocalSocketId)); + Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); + Assert.Contains((0, pushedEndpoint), echoSends); Assert.Equal([(1, rendezvousEndpoint)], payloadSends); } @@ -546,11 +572,10 @@ await WaitForConditionAsync( .ToArray(); Assert.NotEmpty(directSends); - Assert.All(directSends, send => - { - Assert.Equal(1, send.LocalSocketId); - }); Assert.Contains(directSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint)); + Assert.DoesNotContain( + directSends, + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint) && diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 1898c81..965cbb0 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -79,12 +79,12 @@ public async Task PushDirectPaths_KnownEndpoint_ReusesObservedSocketAffinity() await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 1, CancellationToken.None); Assert.Equal(new[] { endpoint, endpoint }, hintedEndpoints); - Assert.Equal(new[] { 0, 1 }, manager.GetPreferredLocalSocketIds(endpoint).Order().ToArray()); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } [Fact] - public async Task PushDirectPaths_NewEndpoint_RemembersReceivingSocketWithoutHolePunch() + public async Task PushDirectPaths_NewEndpoint_DoesNotRememberReceivingSocketWithoutHolePunch() { var udp = new RecordingUdpTransport(); @@ -98,7 +98,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } @@ -125,7 +125,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Equal(newEndpoint, manager.Endpoints[0]); Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(oldEndpoint)); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(newEndpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(newEndpoint)); } [Fact] @@ -180,8 +180,8 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, false, 1), hints[0]); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs index dd3bed6..79ee033 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -83,7 +83,7 @@ await manager.HandleRendezvousFromRootAsync( } [Fact] - public async Task PushDirectPaths_RemembersReceivingSocketAffinity_ForNewEndpoints() + public async Task PushDirectPaths_DoesNotRememberReceivingSocketAffinity_ForNewEndpoints() { await using var udp = new RecordingUdpTransport(); var relay = new IPEndPoint(IPAddress.Loopback, 9999); @@ -96,7 +96,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 2ae36af..30689db 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1347,8 +1347,11 @@ private async ValueTask HandleDirectEndpointHintAsync( if (ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) { var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - useAllEligibleLocalSockets = false; - if (!directEndpoints.IsPinnedRendezvousEndpoint(endpoint) && ZeroTierTrace.Enabled) + if (directEndpoints.IsPinnedRendezvousEndpoint(endpoint)) + { + useAllEligibleLocalSockets = false; + } + else if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine( $"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} mode=alternate-hint payloadPinned=True."); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index dd7e212..7835d8c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -227,9 +227,8 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( else { add.Add(endpoint); - preferredSocketEndpoints.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); } } From 2f67b44293c0c9988fd5eb23031abad6d1e37bab Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:21:08 +0100 Subject: [PATCH 256/296] Pin relayed root control to observed socket --- ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs | 4 ++-- ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index f13549a..4620e61 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -1127,7 +1127,7 @@ public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() } [Fact] - public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithoutConfirmedPath_FansOutAcrossRootSockets() + public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithObservedRootPath_UsesPeerRootSocketAffinity() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), @@ -1167,7 +1167,7 @@ public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithoutConfirmedPa .Order() .ToArray(); - Assert.Equal(new[] { 0, 1 }, socketIds); + Assert.Equal(new[] { 1 }, socketIds); } private static ZeroTierDataplaneRuntime CreateRuntime( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 30689db..2d66e18 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1238,7 +1238,8 @@ private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet } private bool ShouldUsePeerRootSocketAffinity(NodeId peerNodeId) - => _multipath.AllowRootRelayFallback || + => _peerRootSocketAffinity.TryGet(peerNodeId, out _) || + _multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId) || ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); From 8cad9d7e337c61a063fb1e36d3f6f8bc6632d15a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:25:28 +0100 Subject: [PATCH 257/296] Fan out direct-only payload beyond rendezvous --- .../ZeroTierDataplaneRuntimeDirectPathTests.cs | 6 ++++-- .../Internal/ZeroTierDataplaneRuntime.cs | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 4620e61..c72a52f 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -418,7 +418,7 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_StaysOnPinnedRendezvousPath() + public async Task DataplaneRuntime_DirectOnly_SynPayload_PrefersPinnedRendezvous_AndFansOutOnStickySocket() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -507,7 +507,9 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); Assert.Contains((0, pushedEndpoint), echoSends); - Assert.Equal([(1, rendezvousEndpoint)], payloadSends); + Assert.Contains((1, rendezvousEndpoint), payloadSends); + Assert.Contains((1, pushedEndpoint), payloadSends); + Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 2d66e18..2b5f2a9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -666,9 +666,7 @@ private bool TryGetDirectOnlyHintedPayloadFanout( var unique = new HashSet(); var restrictToPinnedRendezvous = ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); - var selectedEndpoints = restrictToPinnedRendezvous - ? [hinted[0]] - : hinted; + var selectedEndpoints = hinted; var candidates = new List(selectedEndpoints.Length * 2); for (var i = 0; i < selectedEndpoints.Length; i++) { @@ -704,7 +702,17 @@ private int[] GetDirectOnlyHintedPayloadSocketIds( { if (restrictToPinnedRendezvous) { - return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (directEndpoints.IsPinnedRendezvousEndpoint(endpoint)) + { + return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + } + + var pinnedEndpoints = directEndpoints.Endpoints; + if (pinnedEndpoints.Length != 0 && directEndpoints.IsPinnedRendezvousEndpoint(pinnedEndpoints[0])) + { + return GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoints[0]); + } } var preferredSocketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); From c3a9a26789a3eb6faab997dc8aab1c9b301bfb8e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:28:50 +0100 Subject: [PATCH 258/296] Force direct-only HELLO on alternate hints --- .../ZeroTierDataplaneRuntimeDirectPathTests.cs | 6 ++++++ .../ZeroTier/Internal/ZeroTierDataplaneRuntime.cs | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index c72a52f..f4e14e3 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -507,6 +507,12 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); Assert.Contains((0, pushedEndpoint), echoSends); + Assert.Contains( + directSends, + send => send.RemoteEndPoint.Equals(pushedEndpoint) && + send.LocalSocketId == 1 && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); Assert.Contains((1, rendezvousEndpoint), payloadSends); Assert.Contains((1, pushedEndpoint), payloadSends); Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 2b5f2a9..0fe8e0f 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -793,12 +793,13 @@ private async ValueTask PrimeDirectOnlyHintedPayloadPathsAsync( for (var i = 0; i < fanoutPaths.Length; i++) { var path = fanoutPaths[i]; + var forceFullHello = ShouldForceFullHelloForDirectOnlyPayloadPrime(peerNodeId, path.RemoteEndPoint); await SendDirectBootstrapProbeAsync( peerNodeId, path.LocalSocketId, path.RemoteEndPoint, sharedKey, - forceFullHello: false, + forceFullHello, helloMinIntervalMs: 0, cancellationToken) .ConfigureAwait(false); @@ -1928,6 +1929,17 @@ private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId) => ShouldUseStickyHintedPathSelection(peerNodeId) && CanUseEchoForDirectBootstrap(peerNodeId); + private bool ShouldForceFullHelloForDirectOnlyPayloadPrime(NodeId peerNodeId, IPEndPoint endpoint) + { + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) || + !CanUseEchoForDirectBootstrap(peerNodeId)) + { + return false; + } + + return !GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint); + } + private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) From a2498a35e920a20795e242e04d21457fd16d4501 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:32:56 +0100 Subject: [PATCH 259/296] Accept CGNAT direct hints on shared space --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 2 +- .../ZeroTierDirectEndpointPolicyTests.cs | 28 ++++++++++++++++++ .../Internal/ZeroTierDirectEndpointPolicy.cs | 29 +++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index f4e14e3..36ed276 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -324,7 +324,7 @@ await WaitForConditionAsync( .Distinct() .Order() .ToArray(); - Assert.Equal(new[] { 0, 1 }, pushedBootstrapSockets); + Assert.Contains(0, pushedBootstrapSockets); var rendezvousBootstrapSockets = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs index 88ffb13..e42ab65 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs @@ -93,6 +93,34 @@ public void ShouldAccept_RejectsSharedEndpoint_OutsideLocalNetworks() Assert.False(accepted); } + [Fact] + public void ShouldAccept_AcceptsSharedEndpoint_WhenLocalAddressIsAlsoInSharedSpace() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("100.74.185.14"), 32) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAdvertiseLocalEndpoint_AcceptsSharedSpacePeerOutsideExactSubnet() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("100.74.185.14"), 32) + ]); + + var accepted = policy.ShouldAdvertiseLocalEndpoint( + new IPEndPoint(IPAddress.Parse("100.74.185.14"), 50060), + [new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)]); + + Assert.True(accepted); + } + [Fact] public void ShouldAccept_AcceptsUlaEndpoint_OnMatchingLocalNetwork() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs index 364e611..2bc36d7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs @@ -19,6 +19,7 @@ internal sealed class ZeroTierDirectEndpointPolicy private readonly LocalNetwork[] _localNetworks; private readonly bool _hasLocalIpv4; private readonly bool _hasLocalIpv6; + private readonly bool _hasLocalIpv4SharedSpace; public ZeroTierDirectEndpointPolicy() : this(GetLocalNetworks()) @@ -34,6 +35,10 @@ internal ZeroTierDirectEndpointPolicy(IReadOnlyList localNetworks) if (_localNetworks[i].Address.AddressFamily == AddressFamily.InterNetwork) { _hasLocalIpv4 = true; + if (IsIpv4SharedSpace(_localNetworks[i].Address)) + { + _hasLocalIpv4SharedSpace = true; + } } else if (_localNetworks[i].Address.AddressFamily == AddressFamily.InterNetworkV6) { @@ -62,6 +67,13 @@ public bool ShouldAccept(IPEndPoint endpoint) return ShouldAcceptPublicFamily(endpoint.AddressFamily); } + if (endpoint.Address.AddressFamily == AddressFamily.InterNetwork && + IsIpv4SharedSpace(endpoint.Address) && + _hasLocalIpv4SharedSpace) + { + return true; + } + for (var i = 0; i < _localNetworks.Length; i++) { if (_localNetworks[i].Contains(endpoint.Address)) @@ -128,6 +140,14 @@ private bool SharesLocalNetwork(IPAddress localAddress, IPAddress peerAddress) return false; } + if (localAddress.AddressFamily == AddressFamily.InterNetwork && + IsIpv4SharedSpace(localAddress) && + IsIpv4SharedSpace(peerAddress) && + _hasLocalIpv4SharedSpace) + { + return true; + } + for (var i = 0; i < _localNetworks.Length; i++) { var network = _localNetworks[i]; @@ -304,6 +324,15 @@ private static bool IsIpv4LinkLocal(IPAddress address) return bytes is [169, 254, _, _]; } + private static bool IsIpv4SharedSpace(IPAddress address) + { + var bytes = address.GetAddressBytes(); + return bytes.Length == 4 && + bytes[0] == 100 && + bytes[1] >= 64 && + bytes[1] <= 127; + } + private static IPEndPoint Canonicalize(IPEndPoint endpoint) => new(Canonicalize(endpoint.Address)!, endpoint.Port); From d5332d7593bca4bb2ca192f04dfd959492b81d65 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:44:24 +0100 Subject: [PATCH 260/296] Accept routed private hints on allowed sockets --- ...oTierDirectPathSocketAdmissibilityTests.cs | 30 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 17 +++++++++-- .../ZeroTierDirectPathSocketAdmissibility.cs | 16 +++++----- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs b/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs index 72a932e..c1fa17c 100644 --- a/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs @@ -37,6 +37,36 @@ public void ShouldUsePath_FallsBackToAddressFamily_WhenRouteUnknown() Assert.True(accepted); } + [Fact] + public void ShouldUsePath_AcceptsSharedEndpoint_WhenRouteUsesAcceptedLocalAddress() + { + var admissibility = CreateAdmissibility(endpoint => + endpoint.Address.Equals(IPAddress.Parse("100.85.196.109")) + ? IPAddress.Parse("10.0.0.112") + : null); + + var accepted = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldUsePath_RejectsSharedEndpoint_WhenRouteUsesUnacceptedLocalAddress() + { + var admissibility = CreateAdmissibility(endpoint => + endpoint.Address.Equals(IPAddress.Parse("100.85.196.109")) + ? IPAddress.Parse("100.74.185.14") + : null); + + var rejected = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.False(rejected); + } + private static ZeroTierDirectPathSocketAdmissibility CreateAdmissibility(Func resolveLocalAddress) => new( new ZeroTierDirectEndpointPolicy( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 0fe8e0f..c123ea0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -53,6 +53,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerRootSocketAffinity _peerRootSocketAffinity; private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; private readonly ZeroTierDirectEndpointPolicy _directEndpointPolicy; + private readonly ZeroTierDirectPathSocketAdmissibility _directPathSocketAdmissibility; private readonly ZeroTierLocalDirectPathAdvertisementPlanner _localDirectAdvertisementPlanner; private readonly ZeroTierLocalDirectPathAdvertisementSource _localDirectPathAdvertisementsSource; private readonly ZeroTierMultipathOptions _multipath; @@ -205,13 +206,14 @@ public ZeroTierDataplaneRuntime( _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); _peerRootSocketAffinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); + _directPathSocketAdmissibility = new ZeroTierDirectPathSocketAdmissibility( + _directEndpointPolicy, + new ZeroTierDirectRouteResolver()); _localDirectAdvertisementPlanner = new ZeroTierLocalDirectPathAdvertisementPlanner(_directEndpointPolicy); _directHintPlanner = new ZeroTierDirectHintPathPlanner( udp, GetOrCreateDirectEndpointManager, - new ZeroTierDirectPathSocketAdmissibility( - _directEndpointPolicy, - new ZeroTierDirectRouteResolver())); + _directPathSocketAdmissibility); var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); @@ -2323,6 +2325,15 @@ private bool ShouldAcceptDirectEndpoint(IPEndPoint endpoint) return true; } + var localSockets = _udp.LocalSockets; + for (var i = 0; i < localSockets.Count; i++) + { + if (_directPathSocketAdmissibility.ShouldUsePath(localSockets[i], endpoint)) + { + return true; + } + } + if (ZeroTierTrace.Enabled) { ZeroTierTrace.WriteLine($"[zerotier] Drop unusable hinted endpoint: {endpoint}."); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs index 8eda624..6a37719 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs @@ -22,23 +22,25 @@ public bool ShouldUsePath(ZeroTierUdpLocalSocket localSocket, IPEndPoint remoteE ArgumentNullException.ThrowIfNull(remoteEndpoint); remoteEndpoint = Canonicalize(remoteEndpoint); - if (!_endpointPolicy.ShouldAccept(remoteEndpoint)) + var localAddress = Canonicalize(localSocket.LocalEndpoint.Address); + if (!AddressFamiliesAreCompatible(localAddress, remoteEndpoint.Address)) { return false; } - var localAddress = Canonicalize(localSocket.LocalEndpoint.Address); - if (!AddressFamiliesAreCompatible(localAddress, remoteEndpoint.Address)) + if (!_routeResolver.TryResolve(remoteEndpoint, out var routedLocalAddress)) { - return false; + return _endpointPolicy.ShouldAccept(remoteEndpoint); } - if (IsWildcard(localAddress) || !_routeResolver.TryResolve(remoteEndpoint, out var routedLocalAddress)) + routedLocalAddress = Canonicalize(routedLocalAddress); + if (!IsWildcard(localAddress) && !localAddress.Equals(routedLocalAddress)) { - return true; + return false; } - return localAddress.Equals(Canonicalize(routedLocalAddress)); + return _endpointPolicy.ShouldAccept(remoteEndpoint) || + _endpointPolicy.ShouldAccept(new IPEndPoint(routedLocalAddress, remoteEndpoint.Port)); } private static bool IsWildcard(IPAddress address) From b4e6d8742e6479d53b6e37c9d8763eb61a92929d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:51:31 +0100 Subject: [PATCH 261/296] Advertise local private paths for private peers --- ...oTierLocalDirectPathAdvertisementPlannerTests.cs | 13 +++++++++++++ .../ZeroTierLocalDirectPathAdvertisementPlanner.cs | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs index ab204a8..1276ede 100644 --- a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs @@ -34,6 +34,19 @@ [new IPEndPoint(IPAddress.Parse("10.0.0.7"), 14849)], Assert.Equal([PublicLocal, PrivateLocal], selected); } + [Fact] + public void SelectForPeer_KeepsPrivateLocalAdvertisements_ForSharedOrPrivatePeerHints() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)], + []); + + Assert.Equal([PublicLocal, PrivateLocal], selected); + } + [Fact] public void SelectForPeer_DropsPrivateLocalAdvertisements_WhenPeerEndpointsUnknown() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs index e99d369..3523937 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs @@ -22,6 +22,11 @@ public IPEndPoint[] SelectForPeer( ArgumentNullException.ThrowIfNull(observedPeerPaths); var peerEndpoints = BuildPeerEndpoints(hintedPeerEndpoints, observedPeerPaths); + if (peerEndpoints.Any(static endpoint => !ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint))) + { + return localAdvertisements.ToArray(); + } + var selected = new List(localAdvertisements.Count); for (var i = 0; i < localAdvertisements.Count; i++) { From 95ece29a74828f6f942d817ed0a2104e13da2409 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:57:45 +0100 Subject: [PATCH 262/296] Pin private direct hints to root socket --- ...TierDirectEndpointManagerPushFlagsTests.cs | 29 +++++++++++++++++++ .../Internal/ZeroTierDirectEndpointManager.cs | 7 ++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 965cbb0..65f1257 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -184,6 +184,35 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } + [Fact] + public async Task PushDirectPaths_PrivateNormalHint_UsesReceivingSocketOnly() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, false, false, 1), hints[0]); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + } + [Fact] public async Task PushDirectPaths_ClusterRedirect_UsesReceivingSocketAndForcesFullHello() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 7835d8c..161be9a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -228,7 +228,12 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); + var useAllEligibleLocalSockets = ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, useAllEligibleLocalSockets); + if (!useAllEligibleLocalSockets) + { + preferredSocketEndpoints.Add(endpoint); + } } } From abceb726eb6e634314fa10edefe1ade07c9ccd0c Mon Sep 17 00:00:00 2001 From: Jonas Kamsker Date: Mon, 16 Mar 2026 14:03:56 +0100 Subject: [PATCH 263/296] Pin ordinary direct hints to observed socket --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 6 +- ...TierDirectEndpointManagerPushFlagsTests.cs | 410 ++++++------ .../ZeroTierDirectHintPathPlannerTests.cs | 2 +- .../Internal/ZeroTierDirectEndpointManager.cs | 616 +++++++++--------- .../Internal/ZeroTierDirectHintPathPlanner.cs | 11 +- 5 files changed, 523 insertions(+), 522 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 36ed276..15ed1f7 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -324,7 +324,7 @@ await WaitForConditionAsync( .Distinct() .Order() .ToArray(); - Assert.Contains(0, pushedBootstrapSockets); + Assert.Equal(new[] { 1 }, pushedBootstrapSockets); var rendezvousBootstrapSockets = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) @@ -506,7 +506,7 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); - Assert.Contains((0, pushedEndpoint), echoSends); + Assert.Contains((1, pushedEndpoint), echoSends); Assert.Contains( directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint) && @@ -1379,4 +1379,4 @@ private static async Task WaitForConditionAsync(Func condition, TimeSpan t throw new TimeoutException("Timed out waiting for condition."); } } - + diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 65f1257..64d72ee 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -1,104 +1,104 @@ -using System.Buffers.Binary; -using System.Net; -using ZTSharp.ZeroTier.Internal; -using ZTSharp.ZeroTier.Protocol; -using ZTSharp.ZeroTier.Transport; - -namespace ZTSharp.Tests; - -public sealed class ZeroTierDirectEndpointManagerPushFlagsTests -{ - [Fact] - public async Task PushDirectPaths_ForgetFlag_RemovesEndpoint() - { - await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - - var relay = new IPEndPoint(IPAddress.Loopback, 9999); - var peerNodeId = new NodeId(0x1111111111); - var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); - - var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); - await manager.HandlePushDirectPathsFromRemoteAsync( - BuildPushDirectPathsPayload(endpoint, flags: 0), - receivedLocalSocketId: 0, - CancellationToken.None); - - Assert.Contains(manager.Endpoints, ep => ep.Equals(endpoint)); - - await manager.HandlePushDirectPathsFromRemoteAsync( - BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagForgetPath), - receivedLocalSocketId: 0, - CancellationToken.None); - - Assert.DoesNotContain(manager.Endpoints, ep => ep.Equals(endpoint)); - } - - [Fact] - public async Task PushDirectPaths_RejectedEndpoint_IsIgnored() - { - await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - - var relay = new IPEndPoint(IPAddress.Loopback, 9999); - var peerNodeId = new NodeId(0x1111111111); - var manager = new ZeroTierDirectEndpointManager( - udp, - relay, - peerNodeId, - shouldAcceptEndpoint: static endpoint => !endpoint.Address.ToString().StartsWith("172.", StringComparison.Ordinal)); - - await manager.HandlePushDirectPathsFromRemoteAsync( - BuildPushDirectPathsPayload(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993), flags: 0), - receivedLocalSocketId: 0, - CancellationToken.None); - - Assert.Empty(manager.Endpoints); - } - - [Fact] - public async Task PushDirectPaths_KnownEndpoint_ReusesObservedSocketAffinity() +using System.Buffers.Binary; +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointManagerPushFlagsTests +{ + [Fact] + public async Task PushDirectPaths_ForgetFlag_RemovesEndpoint() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.Contains(manager.Endpoints, ep => ep.Equals(endpoint)); + + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagForgetPath), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.DoesNotContain(manager.Endpoints, ep => ep.Equals(endpoint)); + } + + [Fact] + public async Task PushDirectPaths_RejectedEndpoint_IsIgnored() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + shouldAcceptEndpoint: static endpoint => !endpoint.Address.ToString().StartsWith("172.", StringComparison.Ordinal)); + + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993), flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.Empty(manager.Endpoints); + } + + [Fact] + public async Task PushDirectPaths_KnownEndpoint_AccumulatesObservedSocketAffinity() { var udp = new RecordingUdpTransport(); - - var relay = new IPEndPoint(IPAddress.Loopback, 9999); - var peerNodeId = new NodeId(0x1111111111); - var hintedEndpoints = new List(); - var manager = new ZeroTierDirectEndpointManager( - udp, - relay, - peerNodeId, - handleDirectEndpointHintAsync: (_, _, endpoint, _, _, _) => - { - hintedEndpoints.Add(endpoint); - return ValueTask.CompletedTask; - }); - - var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); - var payload = BuildPushDirectPathsPayload(endpoint, flags: 0); - + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new List(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, _, endpoint, _, _, _) => + { + hintedEndpoints.Add(endpoint); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + var payload = BuildPushDirectPathsPayload(endpoint, flags: 0); + await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 1, CancellationToken.None); Assert.Equal(new[] { endpoint, endpoint }, hintedEndpoints); - Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal(new[] { 0, 1 }, manager.GetPreferredLocalSocketIds(endpoint).Order()); Assert.Empty(udp.Sends); } [Fact] - public async Task PushDirectPaths_NewEndpoint_DoesNotRememberReceivingSocketWithoutHolePunch() + public async Task PushDirectPaths_NewEndpoint_RemembersReceivingSocket() { var udp = new RecordingUdpTransport(); - - var relay = new IPEndPoint(IPAddress.Loopback, 9999); - var peerNodeId = new NodeId(0x1111111111); - var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); - - var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); await manager.HandlePushDirectPathsFromRemoteAsync( BuildPushDirectPathsPayload(endpoint, flags: 0), receivedLocalSocketId: 1, CancellationToken.None); - Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } @@ -125,7 +125,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Equal(newEndpoint, manager.Endpoints[0]); Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(oldEndpoint)); - Assert.Empty(manager.GetPreferredLocalSocketIds(newEndpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(newEndpoint)); } [Fact] @@ -156,32 +156,32 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_NormalHint_UsesReceivingSocketFirst() + public async Task PushDirectPaths_NormalHint_UsesReceivingSocketOnly() { var udp = new RecordingUdpTransport(); - - var relay = new IPEndPoint(IPAddress.Loopback, 9999); - var peerNodeId = new NodeId(0x1111111111); - var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); - var manager = new ZeroTierDirectEndpointManager( - udp, - relay, - peerNodeId, - handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => - { - hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); - return ValueTask.CompletedTask; - }); - - var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); await manager.HandlePushDirectPathsFromRemoteAsync( BuildPushDirectPathsPayload(endpoint, flags: 0), receivedLocalSocketId: 1, CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, true, 1), hints[0]); - Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal((endpoint, false, false, 1), hints[0]); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); } [Fact] @@ -212,105 +212,105 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Equal((endpoint, false, false, 1), hints[0]); Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); } - - [Fact] - public async Task PushDirectPaths_ClusterRedirect_UsesReceivingSocketAndForcesFullHello() - { - var udp = new RecordingUdpTransport(); - - var relay = new IPEndPoint(IPAddress.Loopback, 9999); - var peerNodeId = new NodeId(0x1111111111); - var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); - var manager = new ZeroTierDirectEndpointManager( - udp, - relay, - peerNodeId, - handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => - { - hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); - return ValueTask.CompletedTask; - }); - - var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); - await manager.HandlePushDirectPathsFromRemoteAsync( - BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagClusterRedirect), - receivedLocalSocketId: 1, - CancellationToken.None); - - Assert.Single(hints); - Assert.Equal((endpoint, true, false, 1), hints[0]); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); - Assert.Empty(udp.Sends); - } - - private const byte ZtPushDirectPathsFlagForgetPath = 0x01; - private const byte ZtPushDirectPathsFlagClusterRedirect = 0x02; - - private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flags) - { - if (endpoint.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork) - { - throw new ArgumentOutOfRangeException(nameof(endpoint), "Test helper supports IPv4 only."); - } - - var addressBytes = endpoint.Address.GetAddressBytes(); - if (addressBytes.Length != 4) - { - throw new ArgumentOutOfRangeException(nameof(endpoint), "Invalid IPv4 address bytes."); - } - - var payload = new byte[2 + 1 + 2 + 1 + 1 + 6]; - var span = payload.AsSpan(); - - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(0, 2), 1); - - var ptr = 2; - span[ptr++] = flags; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), 0); - ptr += 2; - - span[ptr++] = 4; - span[ptr++] = 6; - - addressBytes.CopyTo(span.Slice(ptr, 4)); - ptr += 4; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), (ushort)endpoint.Port); - - return payload; - } - - private sealed class RecordingUdpTransport : IZeroTierUdpTransport - { - public List<(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit)> Sends { get; } = new(); - - public IReadOnlyList LocalSockets { get; } = - [ - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Loopback, 0)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Loopback, 0)) - ]; - - public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) - => throw new NotSupportedException(); - - public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) - => throw new NotSupportedException(); - - public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) - => SendAsync(0, remoteEndpoint, payload, cancellationToken); - - public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) - { - Sends.Add((localSocketId, remoteEndpoint, null)); - return Task.CompletedTask; - } - - public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) - { - Sends.Add((localSocketId, remoteEndpoint, hopLimit)); - return Task.CompletedTask; - } - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; - } -} - + + [Fact] + public async Task PushDirectPaths_ClusterRedirect_UsesReceivingSocketAndForcesFullHello() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagClusterRedirect), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, true, false, 1), hints[0]); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(udp.Sends); + } + + private const byte ZtPushDirectPathsFlagForgetPath = 0x01; + private const byte ZtPushDirectPathsFlagClusterRedirect = 0x02; + + private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flags) + { + if (endpoint.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork) + { + throw new ArgumentOutOfRangeException(nameof(endpoint), "Test helper supports IPv4 only."); + } + + var addressBytes = endpoint.Address.GetAddressBytes(); + if (addressBytes.Length != 4) + { + throw new ArgumentOutOfRangeException(nameof(endpoint), "Invalid IPv4 address bytes."); + } + + var payload = new byte[2 + 1 + 2 + 1 + 1 + 6]; + var span = payload.AsSpan(); + + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(0, 2), 1); + + var ptr = 2; + span[ptr++] = flags; + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), 0); + ptr += 2; + + span[ptr++] = 4; + span[ptr++] = 6; + + addressBytes.CopyTo(span.Slice(ptr, 4)); + ptr += 4; + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), (ushort)endpoint.Port); + + return payload; + } + + private sealed class RecordingUdpTransport : IZeroTierUdpTransport + { + public List<(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit)> Sends { get; } = new(); + + public IReadOnlyList LocalSockets { get; } = + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Loopback, 0)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Loopback, 0)) + ]; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => SendAsync(0, remoteEndpoint, payload, cancellationToken); + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + Sends.Add((localSocketId, remoteEndpoint, null)); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) + { + Sends.Add((localSocketId, remoteEndpoint, hopLimit)); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} + diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs index fb2e9a3..bed5b42 100644 --- a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs @@ -96,7 +96,7 @@ await manager.HandleRendezvousFromRootAsync( CancellationToken.None); Assert.Equal([1], planner.GetPreferredSocketIds(peerNodeId, endpoint)); - Assert.Equal([1, 0], planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint)); + Assert.Equal([1], planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint)); } private sealed class StubUdpTransport : IZeroTierUdpTransport diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 161be9a..029fcd5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -1,14 +1,14 @@ -using System.Net; -using System.Collections.Concurrent; -using System.Globalization; -using System.Net.Sockets; -using System.Security.Cryptography; -using System.Linq; -using ZTSharp.ZeroTier.Protocol; -using ZTSharp.ZeroTier.Transport; - -namespace ZTSharp.ZeroTier.Internal; - +using System.Net; +using System.Collections.Concurrent; +using System.Globalization; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Linq; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + internal sealed class ZeroTierDirectEndpointManager { private const int MaxEndpoints = 64; @@ -16,13 +16,13 @@ internal sealed class ZeroTierDirectEndpointManager private const int RendezvousHolePunchHopLimit = 2; private const long PushDirectPathsCutoffTimeMs = 30_000; private const int PushDirectPathsCutoffLimit = 8; - - private const byte PushDirectPathsFlagForgetPath = 0x01; - private const byte PushDirectPathsFlagClusterRedirect = 0x02; - - private readonly IZeroTierUdpTransport _udp; - private readonly IPEndPoint _relayEndpoint; - private readonly NodeId _remoteNodeId; + + private const byte PushDirectPathsFlagForgetPath = 0x01; + private const byte PushDirectPathsFlagClusterRedirect = 0x02; + + private readonly IZeroTierUdpTransport _udp; + private readonly IPEndPoint _relayEndpoint; + private readonly NodeId _remoteNodeId; private readonly Func? _handleDirectEndpointHintAsync; private readonly Func? _shouldAcceptEndpoint; private readonly object _lock = new(); @@ -33,24 +33,24 @@ internal sealed class ZeroTierDirectEndpointManager private readonly HashSet _pinnedRendezvousEndpointKeys = new(StringComparer.Ordinal); private long _lastDirectPathPushReceiveMs; private int _directPathPushCutoffCount; - - public ZeroTierDirectEndpointManager( - IZeroTierUdpTransport udp, - IPEndPoint relayEndpoint, - NodeId remoteNodeId, - Func? handleDirectEndpointHintAsync = null, - Func? shouldAcceptEndpoint = null) - { - ArgumentNullException.ThrowIfNull(udp); - ArgumentNullException.ThrowIfNull(relayEndpoint); - - _udp = udp; - _relayEndpoint = relayEndpoint; - _remoteNodeId = remoteNodeId; - _handleDirectEndpointHintAsync = handleDirectEndpointHintAsync; - _shouldAcceptEndpoint = shouldAcceptEndpoint; - } - + + public ZeroTierDirectEndpointManager( + IZeroTierUdpTransport udp, + IPEndPoint relayEndpoint, + NodeId remoteNodeId, + Func? handleDirectEndpointHintAsync = null, + Func? shouldAcceptEndpoint = null) + { + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(relayEndpoint); + + _udp = udp; + _relayEndpoint = relayEndpoint; + _remoteNodeId = remoteNodeId; + _handleDirectEndpointHintAsync = handleDirectEndpointHintAsync; + _shouldAcceptEndpoint = shouldAcceptEndpoint; + } + public IPEndPoint[] Endpoints => _directEndpoints; public bool HasPinnedRendezvousEndpoints @@ -67,27 +67,27 @@ public bool HasPinnedRendezvousEndpoints public int[] GetPreferredLocalSocketIds(IPEndPoint endpoint) { ArgumentNullException.ThrowIfNull(endpoint); - - int[] preferredSocketIds; + + int[] preferredSocketIds; lock (_lock) { if (!_preferredLocalSocketsByEndpoint.TryGetValue(FormatEndpointKey(endpoint), out var socketIds) || socketIds.Count == 0) { - return Array.Empty(); - } - - preferredSocketIds = socketIds.ToArray(); - } - - var localSockets = _udp.LocalSockets; - if (localSockets.Count == 0) - { - return preferredSocketIds.Contains(0) ? new[] { 0 } : Array.Empty(); - } - - var available = localSockets.Select(static socket => socket.Id).ToHashSet(); - return preferredSocketIds - .Where(available.Contains) + return Array.Empty(); + } + + preferredSocketIds = socketIds.ToArray(); + } + + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return preferredSocketIds.Contains(0) ? new[] { 0 } : Array.Empty(); + } + + var available = localSockets.Select(static socket => socket.Id).ToHashSet(); + return preferredSocketIds + .Where(available.Contains) .Distinct() .ToArray(); } @@ -119,39 +119,39 @@ public void RefreshPinnedRendezvousHolePunch(IPEndPoint endpoint, int[] preferre } public async ValueTask HandleRendezvousFromRootAsync( - ReadOnlyMemory payload, - int receivedLocalSocketId, - IPEndPoint receivedVia, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) && rendezvous.With == _remoteNodeId) - { - var endpoints = ZeroTierDirectEndpointSelection.Normalize( - [rendezvous.Endpoint], - _relayEndpoint, - maxEndpoints: MaxEndpoints, - _shouldAcceptEndpoint, - maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); - } - + ReadOnlyMemory payload, + int receivedLocalSocketId, + IPEndPoint receivedVia, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) && rendezvous.With == _remoteNodeId) + { + var endpoints = ZeroTierDirectEndpointSelection.Normalize( + [rendezvous.Endpoint], + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); + } + lock (_lock) { _directEndpoints = endpoints; ReplacePinnedRendezvousEndpoints_NoLock(endpoints); RememberPreferredLocalSockets_NoLock(endpoints, receivedLocalSocketId); } - - foreach (var endpoint in endpoints) - { - TrySendHolePunch( - endpoint, - preferredLocalSocketIds: new[] { receivedLocalSocketId }, - hopLimit: RendezvousHolePunchHopLimit); + + foreach (var endpoint in endpoints) + { + TrySendHolePunch( + endpoint, + preferredLocalSocketIds: new[] { receivedLocalSocketId }, + hopLimit: RendezvousHolePunchHopLimit); if (_handleDirectEndpointHintAsync is not null) { await _handleDirectEndpointHintAsync( @@ -163,60 +163,60 @@ await _handleDirectEndpointHintAsync( cancellationToken) .ConfigureAwait(false); } - } - - return; - } - - ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS (ignored) via {receivedVia}."); - return; - } - - public async ValueTask HandlePushDirectPathsFromRemoteAsync( - ReadOnlyMemory payload, - int receivedLocalSocketId, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (!ZeroTierPushDirectPathsCodec.TryParse(payload.Span, out var paths) || paths.Length == 0) - { - ZeroTierTrace.WriteLine("[zerotier] Drop: failed to parse PUSH_DIRECT_PATHS payload."); - return; - } - - var now = Environment.TickCount64; - if (!RateGatePushDirectPaths(now)) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] Drop: PUSH_DIRECT_PATHS rate-gated (peer={_remoteNodeId})."); - } - - return; - } - + } + + return; + } + + ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS (ignored) via {receivedVia}."); + return; + } + + public async ValueTask HandlePushDirectPathsFromRemoteAsync( + ReadOnlyMemory payload, + int receivedLocalSocketId, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!ZeroTierPushDirectPathsCodec.TryParse(payload.Span, out var paths) || paths.Length == 0) + { + ZeroTierTrace.WriteLine("[zerotier] Drop: failed to parse PUSH_DIRECT_PATHS payload."); + return; + } + + var now = Environment.TickCount64; + if (!RateGatePushDirectPaths(now)) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Drop: PUSH_DIRECT_PATHS rate-gated (peer={_remoteNodeId})."); + } + + return; + } + var forget = new HashSet(StringComparer.Ordinal); var redirect = new List(); var add = new List(); var preferredSocketEndpoints = new List(); var forceFullHelloByEndpoint = new Dictionary(StringComparer.Ordinal); var useAllEligibleLocalSocketsByEndpoint = new Dictionary(StringComparer.Ordinal); - - for (var i = 0; i < paths.Length; i++) - { - var flags = paths[i].Flags; - var endpoint = paths[i].Endpoint; - var key = FormatEndpointKey(endpoint); - - if ((flags & PushDirectPathsFlagForgetPath) != 0) - { - forget.Add(key); - forceFullHelloByEndpoint.Remove(key); - useAllEligibleLocalSocketsByEndpoint.Remove(key); - continue; - } - + + for (var i = 0; i < paths.Length; i++) + { + var flags = paths[i].Flags; + var endpoint = paths[i].Endpoint; + var key = FormatEndpointKey(endpoint); + + if ((flags & PushDirectPathsFlagForgetPath) != 0) + { + forget.Add(key); + forceFullHelloByEndpoint.Remove(key); + useAllEligibleLocalSocketsByEndpoint.Remove(key); + continue; + } + if ((flags & PushDirectPathsFlagClusterRedirect) != 0) { redirect.Add(endpoint); @@ -228,15 +228,11 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - var useAllEligibleLocalSockets = ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, useAllEligibleLocalSockets); - if (!useAllEligibleLocalSockets) - { - preferredSocketEndpoints.Add(endpoint); - } + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); + preferredSocketEndpoints.Add(endpoint); } } - + IPEndPoint[] endpoints; IPEndPoint[] endpointsToProbe; lock (_lock) @@ -259,7 +255,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( endpoints = ZeroTierDirectEndpointSelection.Normalize( merged, _relayEndpoint, - maxEndpoints: MaxEndpoints, + maxEndpoints: MaxEndpoints, _shouldAcceptEndpoint, maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); endpointsToProbe = endpoints @@ -270,125 +266,125 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( PrunePreferredLocalSockets_NoLock(endpoints); RememberPreferredLocalSockets_NoLock(preferredSocketEndpoints, receivedLocalSocketId); } - - if (endpoints.Length == 0 || endpointsToProbe.Length == 0) - { - return; - } - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RX PUSH_DIRECT_PATHS: {FormatPaths(paths)} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)}."); - } - - foreach (var endpoint in endpointsToProbe) - { - if (_handleDirectEndpointHintAsync is not null) - { - var forceFullHello = forceFullHelloByEndpoint[FormatEndpointKey(endpoint)]; - var useAllEligibleLocalSockets = useAllEligibleLocalSocketsByEndpoint[FormatEndpointKey(endpoint)]; - await _handleDirectEndpointHintAsync( - _remoteNodeId, - receivedLocalSocketId, - endpoint, - forceFullHello, - useAllEligibleLocalSockets, - cancellationToken) - .ConfigureAwait(false); - } - } - - return; - } - - public void SeedEndpoints(IEnumerable endpoints) - { - ArgumentNullException.ThrowIfNull(endpoints); - - IPEndPoint[] normalized; - lock (_lock) - { - normalized = ZeroTierDirectEndpointSelection.Normalize( - _directEndpoints.Concat(endpoints), - _relayEndpoint, - maxEndpoints: MaxEndpoints, - _shouldAcceptEndpoint, - maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); - _directEndpoints = normalized; - PrunePreferredLocalSockets_NoLock(normalized); - } - - if (ZeroTierTrace.Enabled && normalized.Length > 0) - { - ZeroTierTrace.WriteLine($"[zerotier] Seed direct endpoints for {_remoteNodeId}: {ZeroTierDirectEndpointSelection.Format(normalized)}."); - } - - foreach (var endpoint in normalized) - { - TrySendHolePunch(endpoint); - } - } - - private bool RateGatePushDirectPaths(long nowMs) - { - lock (_lock) - { - if (unchecked(nowMs - _lastDirectPathPushReceiveMs) <= PushDirectPathsCutoffTimeMs) - { - _directPathPushCutoffCount++; - } - else - { - _directPathPushCutoffCount = 0; - } - - _lastDirectPathPushReceiveMs = nowMs; - return _directPathPushCutoffCount < PushDirectPathsCutoffLimit; - } - } - - private static string FormatPaths(ZeroTierPushedDirectPath[] paths) - { - if (paths.Length == 0) - { - return ""; - } - - return string.Join(", ", paths.Select(static path => $"{FormatFlags(path.Flags)}{path.Endpoint}")); - } - - private static string FormatFlags(byte flags) - { - if (flags == 0) - { - return string.Empty; - } - - var parts = new List(2); - if ((flags & PushDirectPathsFlagForgetPath) != 0) - { - parts.Add("forget"); - } - - if ((flags & PushDirectPathsFlagClusterRedirect) != 0) - { - parts.Add("redirect"); - } - - return "[" + string.Join("|", parts) + "] "; - } - - private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketIds = null, int? hopLimit = null) - { + + if (endpoints.Length == 0 || endpointsToProbe.Length == 0) + { + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RX PUSH_DIRECT_PATHS: {FormatPaths(paths)} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)}."); + } + + foreach (var endpoint in endpointsToProbe) + { + if (_handleDirectEndpointHintAsync is not null) + { + var forceFullHello = forceFullHelloByEndpoint[FormatEndpointKey(endpoint)]; + var useAllEligibleLocalSockets = useAllEligibleLocalSocketsByEndpoint[FormatEndpointKey(endpoint)]; + await _handleDirectEndpointHintAsync( + _remoteNodeId, + receivedLocalSocketId, + endpoint, + forceFullHello, + useAllEligibleLocalSockets, + cancellationToken) + .ConfigureAwait(false); + } + } + + return; + } + + public void SeedEndpoints(IEnumerable endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + IPEndPoint[] normalized; + lock (_lock) + { + normalized = ZeroTierDirectEndpointSelection.Normalize( + _directEndpoints.Concat(endpoints), + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); + _directEndpoints = normalized; + PrunePreferredLocalSockets_NoLock(normalized); + } + + if (ZeroTierTrace.Enabled && normalized.Length > 0) + { + ZeroTierTrace.WriteLine($"[zerotier] Seed direct endpoints for {_remoteNodeId}: {ZeroTierDirectEndpointSelection.Format(normalized)}."); + } + + foreach (var endpoint in normalized) + { + TrySendHolePunch(endpoint); + } + } + + private bool RateGatePushDirectPaths(long nowMs) + { + lock (_lock) + { + if (unchecked(nowMs - _lastDirectPathPushReceiveMs) <= PushDirectPathsCutoffTimeMs) + { + _directPathPushCutoffCount++; + } + else + { + _directPathPushCutoffCount = 0; + } + + _lastDirectPathPushReceiveMs = nowMs; + return _directPathPushCutoffCount < PushDirectPathsCutoffLimit; + } + } + + private static string FormatPaths(ZeroTierPushedDirectPath[] paths) + { + if (paths.Length == 0) + { + return ""; + } + + return string.Join(", ", paths.Select(static path => $"{FormatFlags(path.Flags)}{path.Endpoint}")); + } + + private static string FormatFlags(byte flags) + { + if (flags == 0) + { + return string.Empty; + } + + var parts = new List(2); + if ((flags & PushDirectPathsFlagForgetPath) != 0) + { + parts.Add("forget"); + } + + if ((flags & PushDirectPathsFlagClusterRedirect) != 0) + { + parts.Add("redirect"); + } + + return "[" + string.Join("|", parts) + "] "; + } + + private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketIds = null, int? hopLimit = null) + { var localSockets = _udp.LocalSockets; var now = Environment.TickCount64; _holePunchLimiter.CleanupIfNeeded(now); - + var junk = new byte[4]; RandomNumberGenerator.Fill(junk); - - if (preferredLocalSocketIds is { Length: > 0 }) - { + + if (preferredLocalSocketIds is { Length: > 0 }) + { for (var i = 0; i < preferredLocalSocketIds.Length; i++) { var socketId = preferredLocalSocketIds[i]; @@ -396,24 +392,24 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId { continue; } - - TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); - } - - return; - } - + + TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); + } + + return; + } + if (localSockets.Count == 0) { if (!_holePunchLimiter.ShouldSend(localSocketId: 0, endpoint, now)) { return; } - - TrySendHolePunchCore(localSocketId: 0, endpoint, junk, hopLimit); - return; - } - + + TrySendHolePunchCore(localSocketId: 0, endpoint, junk, hopLimit); + return; + } + for (var i = 0; i < localSockets.Count; i++) { var socketId = localSockets[i].Id; @@ -421,30 +417,30 @@ private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketId { continue; } - - TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); - } - } - + + TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); + } + } + private void TrySendHolePunchCore(int localSocketId, IPEndPoint endpoint, byte[] junk, int? hopLimit) - { - Task sendTask; - try - { - ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint} (socket={localSocketId}, hopLimit={(hopLimit?.ToString(CultureInfo.InvariantCulture) ?? "default")})."); - sendTask = hopLimit is int ttl - ? _udp.SendWithHopLimitAsync(localSocketId, endpoint, junk, ttl, CancellationToken.None) - : _udp.SendAsync(localSocketId, endpoint, junk, CancellationToken.None); - } - catch (Exception ex) when (ex is ObjectDisposedException or SocketException or OperationCanceledException) - { - return; - } - - _ = sendTask.ContinueWith( - static t => _ = t.Exception, - CancellationToken.None, - TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, + { + Task sendTask; + try + { + ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint} (socket={localSocketId}, hopLimit={(hopLimit?.ToString(CultureInfo.InvariantCulture) ?? "default")})."); + sendTask = hopLimit is int ttl + ? _udp.SendWithHopLimitAsync(localSocketId, endpoint, junk, ttl, CancellationToken.None) + : _udp.SendAsync(localSocketId, endpoint, junk, CancellationToken.None); + } + catch (Exception ex) when (ex is ObjectDisposedException or SocketException or OperationCanceledException) + { + return; + } + + _ = sendTask.ContinueWith( + static t => _ = t.Exception, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); } @@ -457,33 +453,33 @@ private void ReplacePinnedRendezvousEndpoints_NoLock(IEnumerable end } } - private void RememberPreferredLocalSockets_NoLock(IEnumerable endpoints, int localSocketId) - { - foreach (var endpoint in endpoints) - { - var key = FormatEndpointKey(endpoint); - if (!_preferredLocalSocketsByEndpoint.TryGetValue(key, out var socketIds)) - { - socketIds = new HashSet(); - _preferredLocalSocketsByEndpoint[key] = socketIds; - } - - socketIds.Add(localSocketId); - } - } - - private void PrunePreferredLocalSockets_NoLock(IEnumerable endpoints) - { - var keep = endpoints - .Select(FormatEndpointKey) - .ToHashSet(StringComparer.Ordinal); - + private void RememberPreferredLocalSockets_NoLock(IEnumerable endpoints, int localSocketId) + { + foreach (var endpoint in endpoints) + { + var key = FormatEndpointKey(endpoint); + if (!_preferredLocalSocketsByEndpoint.TryGetValue(key, out var socketIds)) + { + socketIds = new HashSet(); + _preferredLocalSocketsByEndpoint[key] = socketIds; + } + + socketIds.Add(localSocketId); + } + } + + private void PrunePreferredLocalSockets_NoLock(IEnumerable endpoints) + { + var keep = endpoints + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); + foreach (var key in _preferredLocalSocketsByEndpoint.Keys.ToArray()) { if (!keep.Contains(key)) - { - _preferredLocalSocketsByEndpoint.Remove(key); - } + { + _preferredLocalSocketsByEndpoint.Remove(key); + } } } @@ -498,4 +494,4 @@ private void PrunePinnedRendezvousEndpoints_NoLock(IEnumerable endpo private static string FormatEndpointKey(IPEndPoint endpoint) => ZeroTierDirectEndpointKey.Format(endpoint); } - + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs index 05e12a9..a8f76d9 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -167,7 +167,12 @@ private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeF var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); AddSocketIds(preferred, preferredFromHints, localSockets, endpoint, _socketAdmissibility); - if (includeFallbackLocalSockets || preferred.Count == 0) + if (preferred.Count != 0) + { + return preferred.ToArray(); + } + + if (includeFallbackLocalSockets) { AddSocketIds(preferred, GetAdmissibleSocketIds(localSockets, endpoint, _socketAdmissibility), localSockets, endpoint, _socketAdmissibility); } @@ -306,5 +311,5 @@ public int TakeNextSocketIndex(IPEndPoint endpoint, int socketCount) return index; } } -} - +} + From 3f6d3a22d86bb12db8981e55aa0cc54a138b16ff Mon Sep 17 00:00:00 2001 From: Jonas Kamsker Date: Mon, 16 Mar 2026 14:08:08 +0100 Subject: [PATCH 264/296] Clamp direct-only fanout to pinned rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 4 +- .../Internal/ZeroTierDataplaneRuntime.cs | 955 +++++++++--------- 2 files changed, 485 insertions(+), 474 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 15ed1f7..ed4040e 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -507,14 +507,14 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); Assert.Contains((1, pushedEndpoint), echoSends); - Assert.Contains( + Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint) && send.LocalSocketId == 1 && TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello); Assert.Contains((1, rendezvousEndpoint), payloadSends); - Assert.Contains((1, pushedEndpoint), payloadSends); + Assert.DoesNotContain(payloadSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c123ea0..520dae3 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -2,16 +2,16 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Net; -using System.Net.Sockets; -using System.Threading.Channels; -using ZTSharp.Transport.Internal; -using ZTSharp.ZeroTier.Net; -using ZTSharp.ZeroTier.Protocol; -using ZTSharp.ZeroTier.Transport; - -namespace ZTSharp.ZeroTier.Internal; - +using System.Net; +using System.Net.Sockets; +using System.Threading.Channels; +using ZTSharp.Transport.Internal; +using ZTSharp.ZeroTier.Net; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable { private const long DirectEndpointManagerTtlMs = 600_000; @@ -27,12 +27,12 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const long DirectOnlyHintHelloIntervalMs = 5_000; private readonly IZeroTierUdpTransport _udp; - private readonly NodeId _rootNodeId; - private readonly IPEndPoint _rootEndpoint; - private readonly byte[] _rootKey; - private readonly byte _rootProtocolVersion; - private readonly ZeroTierIdentity _localIdentity; - private readonly ulong _networkId; + private readonly NodeId _rootNodeId; + private readonly IPEndPoint _rootEndpoint; + private readonly byte[] _rootKey; + private readonly byte _rootProtocolVersion; + private readonly ZeroTierIdentity _localIdentity; + private readonly ulong _networkId; private readonly byte[] _inlineCom; private readonly ulong _planetId; private readonly ulong _planetTimestamp; @@ -44,8 +44,8 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ConcurrentDictionary _directEndpoints = new(); private readonly ConcurrentDictionary _directEndpointLastUsedMs = new(); private long _lastDirectEndpointCleanupMs; - private readonly ZeroTierPeerPhysicalPathTracker _peerPaths; - private readonly ZeroTierPeerEchoManager _peerEcho; + private readonly ZeroTierPeerPhysicalPathTracker _peerPaths; + private readonly ZeroTierPeerEchoManager _peerEcho; private readonly ZeroTierExternalSurfaceAddressTracker _surfaceAddresses; private readonly ZeroTierPeerQosManager _peerQos; private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; @@ -64,36 +64,36 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); private readonly ConcurrentDictionary _pendingHelloProbes = new(); private long _lastPendingHelloCleanupMs; - + private readonly Channel _peerQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) { FullMode = BoundedChannelFullMode.Wait, SingleReader = false, SingleWriter = true }); - private long _peerQueueDropCount; - private readonly CancellationTokenSource _cts = new(); - private readonly Task _dispatcherLoop; - private readonly Task _peerLoop; - private readonly Task? _multipathMaintenanceLoop; - - private readonly ZeroTierDataplaneRootClient _rootClient; - private readonly ZeroTierDataplanePeerSecurity _peerSecurity; - private readonly ManagedIpToNodeIdCache _managedIpToNodeId = new(); - private readonly ZeroTierDataplaneRouteRegistry _routes; - private readonly ZeroTierDataplanePeerPacketHandler _peerPackets; - private readonly ZeroTierDataplanePeerDatagramProcessor _peerDatagrams; - private readonly ZeroTierDataplaneRxLoops _rxLoops; - - private bool _disposed; - - public ZeroTierDataplaneRuntime( - IZeroTierUdpTransport udp, - NodeId rootNodeId, - IPEndPoint rootEndpoint, - byte[] rootKey, - byte rootProtocolVersion, - ZeroTierIdentity localIdentity, + private long _peerQueueDropCount; + private readonly CancellationTokenSource _cts = new(); + private readonly Task _dispatcherLoop; + private readonly Task _peerLoop; + private readonly Task? _multipathMaintenanceLoop; + + private readonly ZeroTierDataplaneRootClient _rootClient; + private readonly ZeroTierDataplanePeerSecurity _peerSecurity; + private readonly ManagedIpToNodeIdCache _managedIpToNodeId = new(); + private readonly ZeroTierDataplaneRouteRegistry _routes; + private readonly ZeroTierDataplanePeerPacketHandler _peerPackets; + private readonly ZeroTierDataplanePeerDatagramProcessor _peerDatagrams; + private readonly ZeroTierDataplaneRxLoops _rxLoops; + + private bool _disposed; + + public ZeroTierDataplaneRuntime( + IZeroTierUdpTransport udp, + NodeId rootNodeId, + IPEndPoint rootEndpoint, + byte[] rootKey, + byte rootProtocolVersion, + ZeroTierIdentity localIdentity, ulong networkId, IReadOnlyList localManagedIpsV4, IReadOnlyList localManagedIpsV6, @@ -115,14 +115,14 @@ public ZeroTierDataplaneRuntime( localExternalSurfaceAddress: null) { } - - public ZeroTierDataplaneRuntime( - IZeroTierUdpTransport udp, - NodeId rootNodeId, - IPEndPoint rootEndpoint, - byte[] rootKey, - byte rootProtocolVersion, - ZeroTierIdentity localIdentity, + + public ZeroTierDataplaneRuntime( + IZeroTierUdpTransport udp, + NodeId rootNodeId, + IPEndPoint rootEndpoint, + byte[] rootKey, + byte rootProtocolVersion, + ZeroTierIdentity localIdentity, ulong networkId, IReadOnlyList localManagedIpsV4, IReadOnlyList localManagedIpsV6, @@ -133,68 +133,68 @@ public ZeroTierDataplaneRuntime( IPEndPoint? localExternalSurfaceAddress = null, IReadOnlyList? initialExternalSurfaceObservations = null) { - ArgumentNullException.ThrowIfNull(udp); - ArgumentNullException.ThrowIfNull(rootEndpoint); - ArgumentNullException.ThrowIfNull(rootKey); - ArgumentNullException.ThrowIfNull(localIdentity); - ArgumentNullException.ThrowIfNull(localManagedIpsV6); - ArgumentNullException.ThrowIfNull(inlineCom); - ArgumentNullException.ThrowIfNull(multipath); - - // Some local test harnesses pass wildcard-bound socket endpoints (0.0.0.0/::) as a "reachable" root endpoint. - // Normalize those to loopback to keep the dataplane's remote root endpoint concrete. - rootEndpoint = UdpEndpointNormalization.NormalizeForAdvertisement(rootEndpoint); - - if (localManagedIpsV4.Count == 0 && localManagedIpsV6.Count == 0) - { - throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "At least one managed IP (IPv4 or IPv6) is required."); - } - - for (var i = 0; i < localManagedIpsV4.Count; i++) - { - if (localManagedIpsV4[i].AddressFamily != AddressFamily.InterNetwork) - { - throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "All IPv4 managed IPs must be IPv4."); - } - } - - for (var i = 0; i < localManagedIpsV6.Count; i++) - { - if (localManagedIpsV6[i].AddressFamily != AddressFamily.InterNetworkV6) - { - throw new ArgumentOutOfRangeException(nameof(localManagedIpsV6), "All IPv6 managed IPs must be IPv6."); - } - } - - _udp = udp; - _rootNodeId = rootNodeId; - _rootEndpoint = rootEndpoint; - _rootKey = rootKey; - _rootProtocolVersion = rootProtocolVersion; - _localIdentity = localIdentity; + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(rootEndpoint); + ArgumentNullException.ThrowIfNull(rootKey); + ArgumentNullException.ThrowIfNull(localIdentity); + ArgumentNullException.ThrowIfNull(localManagedIpsV6); + ArgumentNullException.ThrowIfNull(inlineCom); + ArgumentNullException.ThrowIfNull(multipath); + + // Some local test harnesses pass wildcard-bound socket endpoints (0.0.0.0/::) as a "reachable" root endpoint. + // Normalize those to loopback to keep the dataplane's remote root endpoint concrete. + rootEndpoint = UdpEndpointNormalization.NormalizeForAdvertisement(rootEndpoint); + + if (localManagedIpsV4.Count == 0 && localManagedIpsV6.Count == 0) + { + throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "At least one managed IP (IPv4 or IPv6) is required."); + } + + for (var i = 0; i < localManagedIpsV4.Count; i++) + { + if (localManagedIpsV4[i].AddressFamily != AddressFamily.InterNetwork) + { + throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "All IPv4 managed IPs must be IPv4."); + } + } + + for (var i = 0; i < localManagedIpsV6.Count; i++) + { + if (localManagedIpsV6[i].AddressFamily != AddressFamily.InterNetworkV6) + { + throw new ArgumentOutOfRangeException(nameof(localManagedIpsV6), "All IPv6 managed IPs must be IPv6."); + } + } + + _udp = udp; + _rootNodeId = rootNodeId; + _rootEndpoint = rootEndpoint; + _rootKey = rootKey; + _rootProtocolVersion = rootProtocolVersion; + _localIdentity = localIdentity; _networkId = networkId; _inlineCom = inlineCom; _planetId = planetId; _planetTimestamp = planetTimestamp; _multipath = multipath; - _localManagedIpsV4 = localManagedIpsV4.Count == 0 ? Array.Empty() : localManagedIpsV4.ToArray(); + _localManagedIpsV4 = localManagedIpsV4.Count == 0 ? Array.Empty() : localManagedIpsV4.ToArray(); _localManagedIpsV4Bytes = _localManagedIpsV4.Length == 0 ? Array.Empty() : _localManagedIpsV4.Select(ip => ip.GetAddressBytes()).ToArray(); _localManagedIpsV6 = localManagedIpsV6.Count == 0 ? Array.Empty() : localManagedIpsV6.ToArray(); _localDirectPathAdvertisements = BuildLocalDirectPathAdvertisements(udp, localExternalSurfaceAddress); _localMac = ZeroTierMac.FromAddress(localIdentity.NodeId, networkId); - - _routes = new ZeroTierDataplaneRouteRegistry(this); - _rootClient = new ZeroTierDataplaneRootClient( - udp, - rootNodeId, - rootEndpoint, - rootKey, - rootProtocolVersion, - localIdentity.NodeId, - networkId, - inlineCom); + + _routes = new ZeroTierDataplaneRouteRegistry(this); + _rootClient = new ZeroTierDataplaneRootClient( + udp, + rootNodeId, + rootEndpoint, + rootKey, + rootProtocolVersion, + localIdentity.NodeId, + networkId, + inlineCom); _peerSecurity = new ZeroTierDataplanePeerSecurity(udp, _rootClient, localIdentity); _peerPaths = new ZeroTierPeerPhysicalPathTracker(ttl: TimeSpan.FromSeconds(30)); _peerEcho = new ZeroTierPeerEchoManager(udp, localIdentity.NodeId, _peerSecurity.GetPeerProtocolVersionOrDefault); @@ -217,17 +217,17 @@ public ZeroTierDataplaneRuntime( var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); - var ip = new ZeroTierDataplaneIpHandler( - sender: this, - routes: _routes, - managedIpToNodeId: _managedIpToNodeId, - icmpv6: icmpv6, - networkId: _networkId, - localMac: _localMac, - localManagedIpsV4: _localManagedIpsV4, - localManagedIpsV4Bytes: _localManagedIpsV4Bytes, - localManagedIpsV6: _localManagedIpsV6); - _peerPackets = new ZeroTierDataplanePeerPacketHandler(_networkId, _localMac, ip, handleControlAsync: HandlePeerControlPacketAsync); + var ip = new ZeroTierDataplaneIpHandler( + sender: this, + routes: _routes, + managedIpToNodeId: _managedIpToNodeId, + icmpv6: icmpv6, + networkId: _networkId, + localMac: _localMac, + localManagedIpsV4: _localManagedIpsV4, + localManagedIpsV4Bytes: _localManagedIpsV4Bytes, + localManagedIpsV6: _localManagedIpsV6); + _peerPackets = new ZeroTierDataplanePeerPacketHandler(_networkId, _localMac, ip, handleControlAsync: HandlePeerControlPacketAsync); _peerDatagrams = new ZeroTierDataplanePeerDatagramProcessor( localIdentity.NodeId, _peerSecurity, @@ -257,11 +257,11 @@ public ZeroTierDataplaneRuntime( { Interlocked.Increment(ref _peerQueueDropCount); if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine("[zerotier] Drop: peer queue is full."); - } - }); - + { + ZeroTierTrace.WriteLine("[zerotier] Drop: peer queue is full."); + } + }); + _dispatcherLoop = Task.Run(() => _rxLoops.DispatcherLoopAsync(_peerQueue, _cts.Token), CancellationToken.None); _peerLoop = Task.Run(() => _rxLoops.PeerLoopAsync(_peerQueue.Reader, _cts.Token), CancellationToken.None); _multipathMaintenanceLoop = multipath.Enabled @@ -352,111 +352,111 @@ private void SeedInitialExternalSurfaceObservations(IReadOnlyList _localIdentity.NodeId; - - public IPEndPoint LocalUdp => _udp.LocalSockets[0].LocalEndpoint; - - public long PeerQueueDropCount => Interlocked.Read(ref _peerQueueDropCount); - - public IZeroTierRoutedIpLink RegisterTcpRoute(NodeId peerNodeId, IPEndPoint localEndpoint, IPEndPoint remoteEndpoint) - { - ObjectDisposedException.ThrowIf(_disposed, this); - return _routes.RegisterTcpRoute(peerNodeId, localEndpoint, remoteEndpoint); - } - - public void UnregisterRoute(ZeroTierTcpRouteKey routeKey) - => _routes.UnregisterRoute(routeKey); - - public void UnregisterRoute(ZeroTierTcpRouteKeyV6 routeKey) - => _routes.UnregisterRoute(routeKey); - - public bool TryRegisterTcpListener( - IPAddress localAddress, - ushort localPort, - Func, CancellationToken, Task> onSyn) - => _routes.TryRegisterTcpListener(localAddress, localPort, onSyn); - - public void UnregisterTcpListener(IPAddress localAddress, ushort localPort) - => _routes.UnregisterTcpListener(localAddress, localPort); - - public bool TryRegisterUdpPort(AddressFamily addressFamily, ushort localPort, ChannelWriter handler) - => _routes.TryRegisterUdpPort(addressFamily, localPort, handler); - - public void UnregisterUdpPort(AddressFamily addressFamily, ushort localPort) - => _routes.UnregisterUdpPort(addressFamily, localPort); - - public async Task ResolveNodeIdAsync(IPAddress managedIp, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(managedIp); - cancellationToken.ThrowIfCancellationRequested(); - ObjectDisposedException.ThrowIf(_disposed, this); - return await _rootClient.ResolveNodeIdAsync(managedIp, _managedIpToNodeId, cancellationToken).ConfigureAwait(false); - } - - public async ValueTask SendIpv4Async(NodeId peerNodeId, ReadOnlyMemory ipv4Packet, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - ObjectDisposedException.ThrowIf(_disposed, this); - - var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); - var packet = ZeroTierExtFramePacketBuilder.BuildPacket( - packetId, - destination: peerNodeId, - source: _localIdentity.NodeId, - networkId: _networkId, - inlineCom: _inlineCom, - to: remoteMac, - from: _localMac, - etherType: ZeroTierFrameCodec.EtherTypeIpv4, - frame: ipv4Packet.Span, - sharedKey: key, - remoteProtocolVersion: peerProtocolVersion); - + + public NodeId NodeId => _localIdentity.NodeId; + + public IPEndPoint LocalUdp => _udp.LocalSockets[0].LocalEndpoint; + + public long PeerQueueDropCount => Interlocked.Read(ref _peerQueueDropCount); + + public IZeroTierRoutedIpLink RegisterTcpRoute(NodeId peerNodeId, IPEndPoint localEndpoint, IPEndPoint remoteEndpoint) + { + ObjectDisposedException.ThrowIf(_disposed, this); + return _routes.RegisterTcpRoute(peerNodeId, localEndpoint, remoteEndpoint); + } + + public void UnregisterRoute(ZeroTierTcpRouteKey routeKey) + => _routes.UnregisterRoute(routeKey); + + public void UnregisterRoute(ZeroTierTcpRouteKeyV6 routeKey) + => _routes.UnregisterRoute(routeKey); + + public bool TryRegisterTcpListener( + IPAddress localAddress, + ushort localPort, + Func, CancellationToken, Task> onSyn) + => _routes.TryRegisterTcpListener(localAddress, localPort, onSyn); + + public void UnregisterTcpListener(IPAddress localAddress, ushort localPort) + => _routes.UnregisterTcpListener(localAddress, localPort); + + public bool TryRegisterUdpPort(AddressFamily addressFamily, ushort localPort, ChannelWriter handler) + => _routes.TryRegisterUdpPort(addressFamily, localPort, handler); + + public void UnregisterUdpPort(AddressFamily addressFamily, ushort localPort) + => _routes.UnregisterUdpPort(addressFamily, localPort); + + public async Task ResolveNodeIdAsync(IPAddress managedIp, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(managedIp); + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); + return await _rootClient.ResolveNodeIdAsync(managedIp, _managedIpToNodeId, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask SendIpv4Async(NodeId peerNodeId, ReadOnlyMemory ipv4Packet, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); + + var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var packet = ZeroTierExtFramePacketBuilder.BuildPacket( + packetId, + destination: peerNodeId, + source: _localIdentity.NodeId, + networkId: _networkId, + inlineCom: _inlineCom, + to: remoteMac, + from: _localMac, + etherType: ZeroTierFrameCodec.EtherTypeIpv4, + frame: ipv4Packet.Span, + sharedKey: key, + remoteProtocolVersion: peerProtocolVersion); + var flowId = _multipath.Enabled ? ZeroTierFlowId.Derive(ipv4Packet.Span) : 0; var preferHintedDirectFanout = ZeroTierTcpHandshakeClassifier.IsInitialSyn(ipv4Packet.Span); await SendToPeerAsync(peerNodeId, packet, flowId, preferHintedDirectFanout, cancellationToken).ConfigureAwait(false); - } - - public async ValueTask SendEthernetFrameAsync( - NodeId peerNodeId, - ushort etherType, - ReadOnlyMemory frame, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - ObjectDisposedException.ThrowIf(_disposed, this); - - var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); - var packet = ZeroTierExtFramePacketBuilder.BuildPacket( - packetId, - destination: peerNodeId, - source: _localIdentity.NodeId, - networkId: _networkId, - inlineCom: _inlineCom, - to: remoteMac, - from: _localMac, - etherType: etherType, - frame: frame.Span, - sharedKey: key, - remoteProtocolVersion: peerProtocolVersion); - - var flowId = (_multipath.Enabled && (etherType == ZeroTierFrameCodec.EtherTypeIpv4 || etherType == ZeroTierFrameCodec.EtherTypeIpv6)) - ? ZeroTierFlowId.Derive(frame.Span) - : 0; - + } + + public async ValueTask SendEthernetFrameAsync( + NodeId peerNodeId, + ushort etherType, + ReadOnlyMemory frame, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); + + var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var packet = ZeroTierExtFramePacketBuilder.BuildPacket( + packetId, + destination: peerNodeId, + source: _localIdentity.NodeId, + networkId: _networkId, + inlineCom: _inlineCom, + to: remoteMac, + from: _localMac, + etherType: etherType, + frame: frame.Span, + sharedKey: key, + remoteProtocolVersion: peerProtocolVersion); + + var flowId = (_multipath.Enabled && (etherType == ZeroTierFrameCodec.EtherTypeIpv4 || etherType == ZeroTierFrameCodec.EtherTypeIpv6)) + ? ZeroTierFlowId.Derive(frame.Span) + : 0; + var preferHintedDirectFanout = (etherType == ZeroTierFrameCodec.EtherTypeIpv4 || etherType == ZeroTierFrameCodec.EtherTypeIpv6) && ZeroTierTcpHandshakeClassifier.IsInitialSyn(frame.Span); await SendToPeerAsync(peerNodeId, packet, flowId, preferHintedDirectFanout, cancellationToken).ConfigureAwait(false); - } - + } + private async ValueTask SendToPeerAsync( NodeId peerNodeId, ReadOnlyMemory packet, @@ -464,8 +464,8 @@ private async ValueTask SendToPeerAsync( bool preferHintedDirectFanout, CancellationToken cancellationToken) { - cancellationToken.ThrowIfCancellationRequested(); - + cancellationToken.ThrowIfCancellationRequested(); + if (!_multipath.Enabled) { EnsureRelayAllowedForPayload(peerNodeId, reason: "multipath direct paths are disabled"); @@ -556,7 +556,7 @@ await TrySendDirectOnlyHintedPayloadAsync(peerNodeId, packet, flowId, parsedOk ? await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); } } - + private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemory packet, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -565,14 +565,14 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo var observed = _peerPaths.GetSnapshot(peerNodeId); var hinted = observed.Length == 0 ? GetOrCreateDirectEndpointManager(peerNodeId).Endpoints : Array.Empty(); - + if (observed.Length == 0 && hinted.Length == 0) { EnsureRelayAllowedForPayload(peerNodeId, reason: "no direct peer paths are available for broadcast"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; } - + var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; @@ -580,28 +580,28 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo var directSuccess = 0; if (observed.Length > 0) - { - var broadcast = ZeroTierPeerBondPolicyEngine.GetBroadcastPaths(observed); - for (var i = 0; i < broadcast.Length; i++) - { - var path = broadcast[i]; - if (_peerEcho.TryGetLastRttMs(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out _)) - { - anyConfirmed = true; - } - + { + var broadcast = ZeroTierPeerBondPolicyEngine.GetBroadcastPaths(observed); + for (var i = 0; i < broadcast.Length; i++) + { + var path = broadcast[i]; + if (_peerEcho.TryGetLastRttMs(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out _)) + { + anyConfirmed = true; + } + if (shouldRecord) { _peerQos.RecordOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); } - - try - { - await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); - directSuccess++; - } - catch (SocketException) - { + + try + { + await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); + directSuccess++; + } + catch (SocketException) + { if (shouldRecord) { _peerQos.ForgetOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); @@ -634,7 +634,7 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo } } } - + if (_multipath.WarmupDuplicateToRoot && _multipath.AllowRootRelayFallback && !anyConfirmed) { await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); @@ -666,9 +666,11 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return false; } - var unique = new HashSet(); var restrictToPinnedRendezvous = ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); - var selectedEndpoints = hinted; + var selectedEndpoints = restrictToPinnedRendezvous + ? SelectPinnedRendezvousEndpoints(peerNodeId, hinted) + : hinted; + var unique = new HashSet(); var candidates = new List(selectedEndpoints.Length * 2); for (var i = 0; i < selectedEndpoints.Length; i++) { @@ -697,6 +699,15 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return true; } + private IPEndPoint[] SelectPinnedRendezvousEndpoints(NodeId peerNodeId, IPEndPoint[] hinted) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var pinned = hinted + .Where(directEndpoints.IsPinnedRendezvousEndpoint) + .ToArray(); + return pinned.Length != 0 ? pinned : hinted; + } + private int[] GetDirectOnlyHintedPayloadSocketIds( NodeId peerNodeId, IPEndPoint endpoint, @@ -1748,19 +1759,19 @@ private void RefreshPinnedRendezvousHolePunch(NodeId peerNodeId, IPEndPoint endp GetOrCreateDirectEndpointManager(peerNodeId).RefreshPinnedRendezvousHolePunch(endpoint, localSocketIds); } - + private int? GetPathLatencyMsOrNull(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) { - if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) - { - return rttMs; - } - - if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) - { - return (int)Math.Min((long)latencyMs * 2, int.MaxValue); - } - + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) + { + return rttMs; + } + + if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) + { + return (int)Math.Min((long)latencyMs * 2, int.MaxValue); + } + return null; } @@ -1920,7 +1931,7 @@ private void CleanupPendingHellosIfNeeded(long nowMs) } } } - + private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) => _peerNegotiation.TryGetRemoteUtility(peerNodeId, localSocketId, remoteEndPoint, out var util) ? util : (short)0; @@ -1941,29 +1952,29 @@ private bool ShouldForceFullHelloForDirectOnlyPayloadPrime(NodeId peerNodeId, IP return !GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint); } - - private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) - { - try - { - await RunMultipathMaintenanceOnceAsync(cancellationToken).ConfigureAwait(false); - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - return; - } -#pragma warning disable CA1031 // Maintenance loop must survive per-iteration faults. - catch (Exception ex) -#pragma warning restore CA1031 - { - ZeroTierTrace.WriteLine($"[zerotier] Multipath maintenance fault: {ex.GetType().Name}: {ex.Message}"); - } - } - } - + + private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await RunMultipathMaintenanceOnceAsync(cancellationToken).ConfigureAwait(false); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } +#pragma warning disable CA1031 // Maintenance loop must survive per-iteration faults. + catch (Exception ex) +#pragma warning restore CA1031 + { + ZeroTierTrace.WriteLine($"[zerotier] Multipath maintenance fault: {ex.GetType().Name}: {ex.Message}"); + } + } + } + private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellationToken) { var nowMs = Environment.TickCount64; @@ -1975,7 +1986,7 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati { return; } - + for (var i = 0; i < peers.Length; i++) { var peerNodeId = peers[i]; @@ -2039,123 +2050,123 @@ await SendDirectBootstrapProbeAsync( for (var p = 0; p < paths.Length; p++) { - var path = paths[p]; - - await _peerEcho - .TrySendEchoProbeAsync(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, key, cancellationToken) - .ConfigureAwait(false); - - if (!_peerQos.TryBuildOutboundPayload(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out var qosPayload)) - { - continue; - } - - await SendPeerControlAsync( - peerNodeId, - path.LocalSocketId, - path.RemoteEndPoint, - ZeroTierVerb.QosMeasurement, - qosPayload, - key, - peerProtocolVersion, - cancellationToken) - .ConfigureAwait(false); - } - + var path = paths[p]; + + await _peerEcho + .TrySendEchoProbeAsync(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, key, cancellationToken) + .ConfigureAwait(false); + + if (!_peerQos.TryBuildOutboundPayload(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out var qosPayload)) + { + continue; + } + + await SendPeerControlAsync( + peerNodeId, + path.LocalSocketId, + path.RemoteEndPoint, + ZeroTierVerb.QosMeasurement, + qosPayload, + key, + peerProtocolVersion, + cancellationToken) + .ConfigureAwait(false); + } + if (_multipath.BondPolicy != ZeroTierBondPolicy.ActiveBackup || paths.Length < 2) { continue; } - - var bestPathIndex = -1; - var bestScore = int.MinValue; - var secondScore = int.MinValue; - - for (var p = 0; p < paths.Length; p++) - { - var path = paths[p]; - var score = ComputePathQualityScore(peerNodeId, path.LocalSocketId, path.RemoteEndPoint); - - if (score > bestScore) - { - secondScore = bestScore; - bestScore = score; - bestPathIndex = p; - } - else if (score > secondScore) - { - secondScore = score; - } - } - - if (bestPathIndex < 0) - { - continue; - } - - var bestPath = paths[bestPathIndex]; - if (!_peerNegotiation.TryMarkSent(peerNodeId, bestPath.LocalSocketId, bestPath.RemoteEndPoint)) - { - continue; - } - - var delta = bestScore - Math.Max(secondScore, 0); - var utility = (short)Math.Clamp(delta, short.MinValue, short.MaxValue); - var payload = new byte[2]; - BinaryPrimitives.WriteInt16BigEndian(payload, utility); - - await SendPeerControlAsync( - peerNodeId, - bestPath.LocalSocketId, - bestPath.RemoteEndPoint, - ZeroTierVerb.PathNegotiationRequest, - payload, - key, - peerProtocolVersion, - cancellationToken) - .ConfigureAwait(false); - } - } - + + var bestPathIndex = -1; + var bestScore = int.MinValue; + var secondScore = int.MinValue; + + for (var p = 0; p < paths.Length; p++) + { + var path = paths[p]; + var score = ComputePathQualityScore(peerNodeId, path.LocalSocketId, path.RemoteEndPoint); + + if (score > bestScore) + { + secondScore = bestScore; + bestScore = score; + bestPathIndex = p; + } + else if (score > secondScore) + { + secondScore = score; + } + } + + if (bestPathIndex < 0) + { + continue; + } + + var bestPath = paths[bestPathIndex]; + if (!_peerNegotiation.TryMarkSent(peerNodeId, bestPath.LocalSocketId, bestPath.RemoteEndPoint)) + { + continue; + } + + var delta = bestScore - Math.Max(secondScore, 0); + var utility = (short)Math.Clamp(delta, short.MinValue, short.MaxValue); + var payload = new byte[2]; + BinaryPrimitives.WriteInt16BigEndian(payload, utility); + + await SendPeerControlAsync( + peerNodeId, + bestPath.LocalSocketId, + bestPath.RemoteEndPoint, + ZeroTierVerb.PathNegotiationRequest, + payload, + key, + peerProtocolVersion, + cancellationToken) + .ConfigureAwait(false); + } + } + private int ComputePathQualityScore(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) - { - if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) - { - return 32767 - Math.Clamp(rttMs, 0, 32767); - } - - if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) - { - var estRtt = (int)Math.Min((long)latencyMs * 2, 32767L); - return 32767 - estRtt; - } - - return 0; - } - + { + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) + { + return 32767 - Math.Clamp(rttMs, 0, 32767); + } + + if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) + { + var estRtt = (int)Math.Min((long)latencyMs * 2, 32767L); + return 32767 - estRtt; + } + + return 0; + } + private async ValueTask SendPeerControlAsync( - NodeId peerNodeId, - int localSocketId, - IPEndPoint remoteEndPoint, - ZeroTierVerb verb, - ReadOnlyMemory payload, - byte[] sharedKey, - byte remoteProtocolVersion, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); - var header = new ZeroTierPacketHeader( - PacketId: packetId, - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)verb); - - var packet = ZeroTierPacketCodec.Encode(header, payload.Span); - ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), encryptPayload: false); + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + ZeroTierVerb verb, + ReadOnlyMemory payload, + byte[] sharedKey, + byte remoteProtocolVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var header = new ZeroTierPacketHeader( + PacketId: packetId, + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)verb); + + var packet = ZeroTierPacketCodec.Encode(header, payload.Span); + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), encryptPayload: false); await _udp.SendAsync(localSocketId, remoteEndPoint, packet, cancellationToken).ConfigureAwait(false); } @@ -2196,91 +2207,91 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer return candidates.ToArray(); } - - private static bool TryGetPacketIdAndVerb(ReadOnlyMemory packet, out (ulong PacketId, ZeroTierVerb Verb) parsed) - { - if (packet.Length < ZeroTierPacketHeader.Length) - { - parsed = default; - return false; - } - - var span = packet.Span; - var packetId = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(ZeroTierPacketHeader.IndexPacketId, 8)); - var verb = (ZeroTierVerb)(span[ZeroTierPacketHeader.IndexVerb] & 0x1F); - parsed = (packetId, verb); - return true; - } - - public async ValueTask DisposeAsync() - { - if (_disposed) - { - return; - } - - _disposed = true; - _peerQueue.Writer.TryComplete(); - await _cts.CancelAsync().ConfigureAwait(false); - - if (_multipathMaintenanceLoop is not null) - { - try - { - await _multipathMaintenanceLoop.ConfigureAwait(false); - } - catch (OperationCanceledException) when (_cts.IsCancellationRequested) - { - } - } - - await _udp.DisposeAsync().ConfigureAwait(false); - - try - { - await _dispatcherLoop.ConfigureAwait(false); - } - catch (OperationCanceledException) when (_cts.IsCancellationRequested) - { - } - catch (ChannelClosedException) when (_cts.IsCancellationRequested) - { - } - - try - { - await _peerLoop.ConfigureAwait(false); - } - catch (OperationCanceledException) when (_cts.IsCancellationRequested) - { - } - catch (ChannelClosedException) when (_cts.IsCancellationRequested) - { - } - - _cts.Dispose(); - _peerSecurity.Dispose(); - } - - private Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken cancellationToken) - => _peerSecurity.GetPeerKeyAsync(peerNodeId, cancellationToken); - + + private static bool TryGetPacketIdAndVerb(ReadOnlyMemory packet, out (ulong PacketId, ZeroTierVerb Verb) parsed) + { + if (packet.Length < ZeroTierPacketHeader.Length) + { + parsed = default; + return false; + } + + var span = packet.Span; + var packetId = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(ZeroTierPacketHeader.IndexPacketId, 8)); + var verb = (ZeroTierVerb)(span[ZeroTierPacketHeader.IndexVerb] & 0x1F); + parsed = (packetId, verb); + return true; + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + _peerQueue.Writer.TryComplete(); + await _cts.CancelAsync().ConfigureAwait(false); + + if (_multipathMaintenanceLoop is not null) + { + try + { + await _multipathMaintenanceLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + } + } + + await _udp.DisposeAsync().ConfigureAwait(false); + + try + { + await _dispatcherLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + } + catch (ChannelClosedException) when (_cts.IsCancellationRequested) + { + } + + try + { + await _peerLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + } + catch (ChannelClosedException) when (_cts.IsCancellationRequested) + { + } + + _cts.Dispose(); + _peerSecurity.Dispose(); + } + + private Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken cancellationToken) + => _peerSecurity.GetPeerKeyAsync(peerNodeId, cancellationToken); + private ValueTask HandleRootControlPacketAsync( ZeroTierVerb verb, ReadOnlyMemory payload, int receivedLocalSocketId, IPEndPoint receivedVia, CancellationToken cancellationToken) - { - if (verb != ZeroTierVerb.Rendezvous) - { - return ValueTask.CompletedTask; - } - - if (!ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) || rendezvous.With.Value == 0) - { - return ValueTask.CompletedTask; - } + { + if (verb != ZeroTierVerb.Rendezvous) + { + return ValueTask.CompletedTask; + } + + if (!ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) || rendezvous.With.Value == 0) + { + return ValueTask.CompletedTask; + } var directEndpoints = GetOrCreateDirectEndpointManager(rendezvous.With); return directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken); @@ -2520,8 +2531,8 @@ public bool TryTakeBest(NodeId peerNodeId, int receivedLocalSocketId, IPEndPoint return false; } } - - - - - + + + + + From 93c9c07e45aff1e7b22e46975e35743315690184 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:23:49 +0100 Subject: [PATCH 265/296] Handle relayed peer rendezvous control --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 111 +++++++++++++++++- .../ZeroTierDataplanePeerPacketHandler.cs | 1 + .../Internal/ZeroTierDataplaneRuntime.cs | 72 +++++++++++- .../Internal/ZeroTierDirectEndpointManager.cs | 74 ++++++++++-- 4 files changed, 244 insertions(+), 14 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index ed4040e..985fe13 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -991,12 +991,9 @@ await WaitForConditionAsync( () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), TimeSpan.FromSeconds(2)); - var initialSendCount = udp.GetSendsSnapshot().Length; - await runtime.RunMultipathMaintenanceOnceForTestsAsync(); var pushDirectSends = udp.GetSendsSnapshot() - .Skip(initialSendCount) .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.PushDirectPaths) .Select(send => (send.LocalSocketId, Paths: ReadPushedDirectPathEndpoints(send.Payload, sharedKey))) @@ -1012,6 +1009,114 @@ await WaitForConditionAsync( }); } + [Fact] + public async Task DataplaneRuntime_MultipathMaintenance_SendsPeerRendezvousViaPeerRootSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoint); + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var rendezvousSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Rendezvous) + .Select(send => (send.LocalSocketId, Endpoint: ReadRendezvousEndpoint(send.Payload, sharedKey))) + .ToArray(); + + Assert.NotEmpty(rendezvousSends); + Assert.All(rendezvousSends, send => + { + Assert.Equal(1, send.LocalSocketId); + Assert.Equal(advertisedSurfaces[1].SurfaceAddress, send.Endpoint); + }); + } + + [Fact] + public async Task DataplaneRuntime_PeerRendezvous_BootstrapsDirectPathOnReceivingSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 62120); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 3, + Destination: localIdentity.NodeId, + Source: peerIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb is ZeroTierVerb.Echo or ZeroTierVerb.Hello), + TimeSpan.FromSeconds(2)); + } + [Fact] public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs index ca814d0..12782f2 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs @@ -42,6 +42,7 @@ public async ValueTask HandleAsync( switch (verb) { case ZeroTierVerb.PushDirectPaths: + case ZeroTierVerb.Rendezvous: { if (_handleControlAsync is not null) { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 520dae3..9bb2845 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -983,6 +983,7 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPeerRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } @@ -1029,6 +1030,7 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPeerRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -1225,6 +1227,63 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } + private async Task SendPeerRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + if (!TryGetPeerRootRendezvousAdvertisement(peerNodeId, out var endpoint)) + { + return; + } + + try + { + var payload = ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, endpoint); + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + payload); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {endpoint}."); + } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private bool TryGetPeerRootRendezvousAdvertisement(NodeId peerNodeId, out IPEndPoint endpoint) + { + endpoint = default!; + if (!_peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) + { + return false; + } + + endpoint = _surfaceAddresses + .GetSnapshot(localSocketId) + .Select(UdpEndpointNormalization.Normalize) + .FirstOrDefault(ZeroTierDirectEndpointSelection.IsPublicEndpoint)!; + + return endpoint is not null; + } + private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { var localAdvertisements = GetLocalDirectPathAdvertisements(); @@ -2307,13 +2366,18 @@ private ValueTask HandlePeerControlPacketAsync( { _peerRootSocketAffinity.Observe(peerNodeId, receivedLocalSocketId, receivedVia); - if (verb != ZeroTierVerb.PushDirectPaths) + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (verb == ZeroTierVerb.Rendezvous) { - return ValueTask.CompletedTask; + return directEndpoints.HandleRendezvousFromPeerAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken); } - var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - return directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId, cancellationToken); + if (verb == ZeroTierVerb.PushDirectPaths) + { + return directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId, cancellationToken); + } + + return ValueTask.CompletedTask; } private ZeroTierDirectEndpointManager GetOrCreateDirectEndpointManager(NodeId peerNodeId) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 029fcd5..9dcffc1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -119,10 +119,10 @@ public void RefreshPinnedRendezvousHolePunch(IPEndPoint endpoint, int[] preferre } public async ValueTask HandleRendezvousFromRootAsync( - ReadOnlyMemory payload, - int receivedLocalSocketId, - IPEndPoint receivedVia, - CancellationToken cancellationToken) + ReadOnlyMemory payload, + int receivedLocalSocketId, + IPEndPoint receivedVia, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -168,9 +168,69 @@ await _handleDirectEndpointHintAsync( return; } - ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS (ignored) via {receivedVia}."); - return; - } + ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS (ignored) via {receivedVia}."); + return; + } + + public async ValueTask HandleRendezvousFromPeerAsync( + ReadOnlyMemory payload, + int receivedLocalSocketId, + IPEndPoint receivedVia, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) || rendezvous.With != _remoteNodeId) + { + ZeroTierTrace.WriteLine($"[zerotier] RX peer RENDEZVOUS (ignored) via {receivedVia}."); + return; + } + + var endpoints = ZeroTierDirectEndpointSelection.Normalize( + [rendezvous.Endpoint], + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); + if (endpoints.Length == 0) + { + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RX peer RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); + } + + lock (_lock) + { + _directEndpoints = ZeroTierDirectEndpointSelection.Normalize( + endpoints.Concat(_directEndpoints), + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); + PrunePreferredLocalSockets_NoLock(_directEndpoints); + RememberPreferredLocalSockets_NoLock(endpoints, receivedLocalSocketId); + } + + if (_handleDirectEndpointHintAsync is null) + { + return; + } + + for (var i = 0; i < endpoints.Length; i++) + { + await _handleDirectEndpointHintAsync( + _remoteNodeId, + receivedLocalSocketId, + endpoints[i], + false, + false, + cancellationToken) + .ConfigureAwait(false); + } + } public async ValueTask HandlePushDirectPathsFromRemoteAsync( ReadOnlyMemory payload, From 9052307642daf56709aa71eaab673f3dbe258f00 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:28:39 +0100 Subject: [PATCH 266/296] Relay peer rendezvous across public surfaces --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 15 ++-- .../Internal/ZeroTierDataplaneRuntime.cs | 84 +++++++++++-------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 985fe13..0f22600 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -884,7 +884,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsHintedPeer } [Fact] - public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvousForAdvertisedSurfaces() + public async Task DataplaneRuntime_DirectOnly_Maintenance_RelaysRendezvousForAdvertisedPublicSurfaces() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -930,7 +930,9 @@ public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvous .OrderBy(endpoint => endpoint.Port) .ToArray(); - Assert.Empty(rendezvousEndpoints); + Assert.Equal( + advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), + rendezvousEndpoints); } [Fact] @@ -1058,11 +1060,10 @@ public async Task DataplaneRuntime_MultipathMaintenance_SendsPeerRendezvousViaPe .ToArray(); Assert.NotEmpty(rendezvousSends); - Assert.All(rendezvousSends, send => - { - Assert.Equal(1, send.LocalSocketId); - Assert.Equal(advertisedSurfaces[1].SurfaceAddress, send.Endpoint); - }); + Assert.All(rendezvousSends, send => Assert.Equal(1, send.LocalSocketId)); + Assert.Equal( + advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), + rendezvousSends.Select(static send => send.Endpoint).Distinct().OrderBy(static endpoint => endpoint.Port).ToArray()); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 9bb2845..c6f423c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1229,59 +1229,75 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha private async Task SendPeerRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { - if (!TryGetPeerRootRendezvousAdvertisement(peerNodeId, out var endpoint)) + var endpoints = GetPeerRootRendezvousAdvertisements(peerNodeId); + if (endpoints.Length == 0) { return; } - try + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + for (var i = 0; i < endpoints.Length; i++) { - var payload = ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, endpoint); - var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - var packet = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Rendezvous), - payload); - ZeroTierPacketCrypto.Armor( - packet, - ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), - encryptPayload: true); - - if (ZeroTierTrace.Enabled) + try { - ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {endpoint}."); - } + var endpoint = endpoints[i]; + var payload = ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, endpoint); + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + payload); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); - await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) - { - if (ZeroTierTrace.Enabled) + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {endpoint}."); + } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) { - ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } } } } - private bool TryGetPeerRootRendezvousAdvertisement(NodeId peerNodeId, out IPEndPoint endpoint) + private IPEndPoint[] GetPeerRootRendezvousAdvertisements(NodeId peerNodeId) { - endpoint = default!; + var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) + .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) + .Select(UdpEndpointNormalization.Normalize) + .ToArray(); + if (advertisements.Length == 0) + { + return Array.Empty(); + } + if (!_peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) { - return false; + return advertisements; } - endpoint = _surfaceAddresses + var preferred = _surfaceAddresses .GetSnapshot(localSocketId) - .Select(UdpEndpointNormalization.Normalize) - .FirstOrDefault(ZeroTierDirectEndpointSelection.IsPublicEndpoint)!; + .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) + .Select(UdpEndpointNormalization.Normalize); - return endpoint is not null; + return ZeroTierDirectEndpointSelection.Normalize( + preferred.Concat(advertisements), + _rootEndpoint, + maxEndpoints: ZeroTierProtocolLimits.MaxPushedDirectPaths); } private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) From 1f12e26623c0e8911ffe4b57ce6e45b23e1c16d7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:35:42 +0100 Subject: [PATCH 267/296] Restore upstream direct rendezvous semantics --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 41 ++++----- .../ZeroTierDataplanePeerPacketHandler.cs | 1 - .../Internal/ZeroTierDataplaneRuntime.cs | 86 +------------------ .../Internal/ZeroTierDirectEndpointManager.cs | 62 +------------ 4 files changed, 19 insertions(+), 171 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 0f22600..c7d4088 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -884,7 +884,7 @@ public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsHintedPeer } [Fact] - public async Task DataplaneRuntime_DirectOnly_Maintenance_RelaysRendezvousForAdvertisedPublicSurfaces() + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvousForAdvertisedSurfaces() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -930,9 +930,7 @@ public async Task DataplaneRuntime_DirectOnly_Maintenance_RelaysRendezvousForAdv .OrderBy(endpoint => endpoint.Port) .ToArray(); - Assert.Equal( - advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), - rendezvousEndpoints); + Assert.Empty(rendezvousEndpoints); } [Fact] @@ -1002,17 +1000,15 @@ await WaitForConditionAsync( .ToArray(); Assert.NotEmpty(pushDirectSends); + Assert.Equal(new[] { 0, 1 }, pushDirectSends.Select(static send => send.LocalSocketId).Distinct().Order().ToArray()); Assert.All(pushDirectSends, send => - { - Assert.Equal(1, send.LocalSocketId); Assert.Equal( advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), - send.Paths); - }); + send.Paths)); } [Fact] - public async Task DataplaneRuntime_MultipathMaintenance_SendsPeerRendezvousViaPeerRootSocket() + public async Task DataplaneRuntime_MultipathMaintenance_DoesNotSendPeerRendezvousViaRoot() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -1056,18 +1052,13 @@ public async Task DataplaneRuntime_MultipathMaintenance_SendsPeerRendezvousViaPe .Skip(initialSendCount) .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Rendezvous) - .Select(send => (send.LocalSocketId, Endpoint: ReadRendezvousEndpoint(send.Payload, sharedKey))) .ToArray(); - Assert.NotEmpty(rendezvousSends); - Assert.All(rendezvousSends, send => Assert.Equal(1, send.LocalSocketId)); - Assert.Equal( - advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), - rendezvousSends.Select(static send => send.Endpoint).Distinct().OrderBy(static endpoint => endpoint.Port).ToArray()); + Assert.Empty(rendezvousSends); } [Fact] - public async Task DataplaneRuntime_PeerRendezvous_BootstrapsDirectPathOnReceivingSocket() + public async Task DataplaneRuntime_PeerRendezvous_IsIgnored() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -1109,13 +1100,13 @@ public async Task DataplaneRuntime_PeerRendezvous_BootstrapsDirectPathOnReceivin udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); - await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => - send.LocalSocketId == 1 && - send.RemoteEndPoint.Equals(rendezvousEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb is ZeroTierVerb.Echo or ZeroTierVerb.Hello), - TimeSpan.FromSeconds(2)); + await Task.Delay(TimeSpan.FromMilliseconds(200)); + + var directSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .ToArray(); + + Assert.Empty(directSends); } [Fact] @@ -1241,7 +1232,7 @@ public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() } [Fact] - public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithObservedRootPath_UsesPeerRootSocketAffinity() + public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithObservedRootPath_FansOutAcrossSockets() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), @@ -1281,7 +1272,7 @@ public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithObservedRootPa .Order() .ToArray(); - Assert.Equal(new[] { 1 }, socketIds); + Assert.Equal(new[] { 0, 1 }, socketIds); } private static ZeroTierDataplaneRuntime CreateRuntime( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs index 12782f2..ca814d0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs @@ -42,7 +42,6 @@ public async ValueTask HandleAsync( switch (verb) { case ZeroTierVerb.PushDirectPaths: - case ZeroTierVerb.Rendezvous: { if (_handleControlAsync is not null) { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index c6f423c..d429a3c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -983,7 +983,6 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendPeerRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } @@ -1030,7 +1029,6 @@ private async Task TrySendPeriodicDirectPathPushAsync( } await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendPeerRendezvousViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } private async Task ProbeHintedDirectEndpointsAsync( @@ -1227,79 +1225,6 @@ private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sha } } - private async Task SendPeerRendezvousViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) - { - var endpoints = GetPeerRootRendezvousAdvertisements(peerNodeId); - if (endpoints.Length == 0) - { - return; - } - - var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - for (var i = 0; i < endpoints.Length; i++) - { - try - { - var endpoint = endpoints[i]; - var payload = ZeroTierRendezvousCodec.BuildPayload(_localIdentity.NodeId, endpoint); - var packet = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: ZeroTierPacketIdGenerator.GeneratePacketId(), - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Rendezvous), - payload); - ZeroTierPacketCrypto.Armor( - packet, - ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), - encryptPayload: true); - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] TX RENDEZVOUS via root for {peerNodeId}: {endpoint}."); - } - - await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) - { - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RENDEZVOUS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); - } - } - } - } - - private IPEndPoint[] GetPeerRootRendezvousAdvertisements(NodeId peerNodeId) - { - var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId) - .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) - .Select(UdpEndpointNormalization.Normalize) - .ToArray(); - if (advertisements.Length == 0) - { - return Array.Empty(); - } - - if (!_peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) - { - return advertisements; - } - - var preferred = _surfaceAddresses - .GetSnapshot(localSocketId) - .Where(ZeroTierDirectEndpointSelection.IsPublicEndpoint) - .Select(UdpEndpointNormalization.Normalize); - - return ZeroTierDirectEndpointSelection.Normalize( - preferred.Concat(advertisements), - _rootEndpoint, - maxEndpoints: ZeroTierProtocolLimits.MaxPushedDirectPaths); - } - private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) { var localAdvertisements = GetLocalDirectPathAdvertisements(); @@ -1335,10 +1260,8 @@ private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet } private bool ShouldUsePeerRootSocketAffinity(NodeId peerNodeId) - => _peerRootSocketAffinity.TryGet(peerNodeId, out _) || - _multipath.AllowRootRelayFallback || - HasConfirmedDirectPath(peerNodeId) || - ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); + => _multipath.AllowRootRelayFallback || + HasConfirmedDirectPath(peerNodeId); private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { @@ -2383,11 +2306,6 @@ private ValueTask HandlePeerControlPacketAsync( _peerRootSocketAffinity.Observe(peerNodeId, receivedLocalSocketId, receivedVia); var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - if (verb == ZeroTierVerb.Rendezvous) - { - return directEndpoints.HandleRendezvousFromPeerAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken); - } - if (verb == ZeroTierVerb.PushDirectPaths) { return directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId, cancellationToken); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 9dcffc1..a039715 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -172,69 +172,9 @@ await _handleDirectEndpointHintAsync( return; } - public async ValueTask HandleRendezvousFromPeerAsync( + public async ValueTask HandlePushDirectPathsFromRemoteAsync( ReadOnlyMemory payload, int receivedLocalSocketId, - IPEndPoint receivedVia, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (!ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) || rendezvous.With != _remoteNodeId) - { - ZeroTierTrace.WriteLine($"[zerotier] RX peer RENDEZVOUS (ignored) via {receivedVia}."); - return; - } - - var endpoints = ZeroTierDirectEndpointSelection.Normalize( - [rendezvous.Endpoint], - _relayEndpoint, - maxEndpoints: MaxEndpoints, - _shouldAcceptEndpoint, - maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); - if (endpoints.Length == 0) - { - return; - } - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RX peer RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); - } - - lock (_lock) - { - _directEndpoints = ZeroTierDirectEndpointSelection.Normalize( - endpoints.Concat(_directEndpoints), - _relayEndpoint, - maxEndpoints: MaxEndpoints, - _shouldAcceptEndpoint, - maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); - PrunePreferredLocalSockets_NoLock(_directEndpoints); - RememberPreferredLocalSockets_NoLock(endpoints, receivedLocalSocketId); - } - - if (_handleDirectEndpointHintAsync is null) - { - return; - } - - for (var i = 0; i < endpoints.Length; i++) - { - await _handleDirectEndpointHintAsync( - _remoteNodeId, - receivedLocalSocketId, - endpoints[i], - false, - false, - cancellationToken) - .ConfigureAwait(false); - } - } - - public async ValueTask HandlePushDirectPathsFromRemoteAsync( - ReadOnlyMemory payload, - int receivedLocalSocketId, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); From 91ab54b4ca292f12ea8d0569eadb8bcb7f01d46b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:36:30 +0100 Subject: [PATCH 268/296] Add direct p2p investigation logbook --- docs/logbook/direct-p2p-jaeger.md | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/logbook/direct-p2p-jaeger.md diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md new file mode 100644 index 0000000..e038a07 --- /dev/null +++ b/docs/logbook/direct-p2p-jaeger.md @@ -0,0 +1,53 @@ +# Direct P2P Jaeger Logbook + +Append-only record for direct-P2P work against `https://jaeger.pdcs.kamsker.at/`. +New notes are appended with timestamps. Older entries are not rewritten. + +## 2026-03-16T14:35:52.4539008+01:00 + +- Created this logbook to keep an append-only protocol of direct-P2P investigation and wire-level results. +- Current branch head at creation time: `1f12e26`. +- Current definition of done remains unchanged: payload to and from `https://jaeger.pdcs.kamsker.at/` must flow hop-0 without relay-forwarded data. + +## 2026-03-16T14:36:00+01:00 + +- Checkpoint `93c9c07` added managed handling for authenticated peer `RENDEZVOUS` control and tests for that path. +- Why it was tried: + - the runtime was already handling peer `PUSH_DIRECT_PATHS` + - adding peer `RENDEZVOUS` looked like one more way to seed direct endpoint bootstrap on the receiving socket +- Issue update at that time: + - live runs still showed no `RX direct raw`, no `RX OK(ECHO)`, no `hop=0` + - the peer only replied with relayed `OK(HELLO)` via `84.17.53.155:9993` + +## 2026-03-16T14:36:10+01:00 + +- Checkpoint `9052307` broadened relayed `RENDEZVOUS` advertisement across all public local surfaces instead of a single public root-affine surface. +- Why it was tried: + - the peer only ever saw one relayed introduction surface from us + - widening that set was the last obvious local lever for peer introduction breadth +- Issue update after the live rerun: + - the runtime emitted relayed `RENDEZVOUS` for all 8 public surfaces + - there was still no `RX peer RENDEZVOUS`, no `RX OK(ECHO)`, no `hop=0` + - the HTTPS call still timed out waiting for TCP SYN-ACK + +## 2026-03-16T14:36:20+01:00 + +- Upstream review of `external/libzt/ext/ZeroTierOne/node/IncomingPacket.cpp` and `Switch.cpp` changed the understanding of the blocker. +- Issue change: + - upstream only honors `VERB_RENDEZVOUS` from an upstream peer + - upstream `introduce()` is triggered by the relay/switch when it forwards packets between peers + - this means the relayed peer-`RENDEZVOUS` path added locally was not actually upstream-compatible and was likely ignored by the remote side + +## 2026-03-16T14:36:30+01:00 + +- Checkpoint `1f12e26` restored upstream direct-rendezvous semantics. +- What changed: + - removed relayed peer `RENDEZVOUS` sending + - removed managed peer `RENDEZVOUS` control handling + - restored direct-only relayed root control fanout across sockets until a confirmed hop-0 path exists +- Why it was tried: + - to realign with vendored upstream behavior + - to keep root seeing traffic from all local public ports instead of prematurely pinning to one observed root socket +- Current issue state: + - direct P2P is still not established + - the remaining live-wire blocker is unchanged: the remote peer still does not return confirmed hop-0 traffic to this runtime From 74e440db79b651d88f11aee0399e870d0ddeed5e Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:44:41 +0100 Subject: [PATCH 269/296] Fan out ordinary pushed direct hints --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 2 +- ...TierDirectEndpointManagerPushFlagsTests.cs | 22 +++++++++---------- .../Internal/ZeroTierDirectEndpointManager.cs | 3 +-- docs/logbook/direct-p2p-jaeger.md | 21 ++++++++++++++++++ 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index c7d4088..3cd5672 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -324,7 +324,7 @@ await WaitForConditionAsync( .Distinct() .Order() .ToArray(); - Assert.Equal(new[] { 1 }, pushedBootstrapSockets); + Assert.Equal(new[] { 0, 1 }, pushedBootstrapSockets); var rendezvousBootstrapSockets = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 64d72ee..f9682a3 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -55,7 +55,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_KnownEndpoint_AccumulatesObservedSocketAffinity() + public async Task PushDirectPaths_KnownEndpoint_DoesNotPersistOrdinarySocketAffinity() { var udp = new RecordingUdpTransport(); @@ -79,12 +79,12 @@ public async Task PushDirectPaths_KnownEndpoint_AccumulatesObservedSocketAffinit await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 1, CancellationToken.None); Assert.Equal(new[] { endpoint, endpoint }, hintedEndpoints); - Assert.Equal(new[] { 0, 1 }, manager.GetPreferredLocalSocketIds(endpoint).Order()); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } [Fact] - public async Task PushDirectPaths_NewEndpoint_RemembersReceivingSocket() + public async Task PushDirectPaths_NewEndpoint_DoesNotRememberReceivingSocket() { var udp = new RecordingUdpTransport(); @@ -98,7 +98,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( receivedLocalSocketId: 1, CancellationToken.None); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); Assert.Empty(udp.Sends); } @@ -125,7 +125,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( Assert.Equal(newEndpoint, manager.Endpoints[0]); Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(oldEndpoint)); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(newEndpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(newEndpoint)); } [Fact] @@ -156,7 +156,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_NormalHint_UsesReceivingSocketOnly() + public async Task PushDirectPaths_NormalHint_UsesAllEligibleSockets() { var udp = new RecordingUdpTransport(); @@ -180,12 +180,12 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, false, 1), hints[0]); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } [Fact] - public async Task PushDirectPaths_PrivateNormalHint_UsesReceivingSocketOnly() + public async Task PushDirectPaths_PrivateNormalHint_UsesAllEligibleSockets() { var udp = new RecordingUdpTransport(); @@ -209,8 +209,8 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, false, 1), hints[0]); - Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index a039715..8ef5d93 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -228,8 +228,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); - preferredSocketEndpoints.Add(endpoint); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); } } diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index e038a07..e959f38 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -51,3 +51,24 @@ New notes are appended with timestamps. Older entries are not rewritten. - Current issue state: - direct P2P is still not established - the remaining live-wire blocker is unchanged: the remote peer still does not return confirmed hop-0 traffic to this runtime + +## 2026-03-16T14:40:55.8501212+01:00 + +- After restoring upstream semantics, ordinary relayed `PUSH_DIRECT_PATHS` still looked over-pinned to the one receiving root socket. +- Checkpoint in progress: + - ordinary pushed hints were changed to bootstrap on all admissible local sockets again + - strict socket affinity is now reserved for root `RENDEZVOUS` / redirect style hints, not ordinary pushed hints +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `ZeroTierDirectEndpointManagerSocketAffinityTests` + - `ZeroTierDirectHintPathPlannerTests` +- Live result in `.codex-debug/direct-only-post-ordinary-hint-fanout.log`: + - ordinary public hints like `176.66.90.119:19665`, `:8955`, `:45185`, `:35838`, `:27605`, `:62120`, `:8140` now fan out across sockets `0..7` + - private/shared hints like `100.85.196.109:*`, `10.22.10.94:*`, `172.17.0.1:*`, `172.18.0.1:*` also fan out across sockets `0..7` + - direct-only payload fanout now follows that wider matrix instead of staying pinned to one `(endpoint, socket)` path +- Issue update: + - this materially widened direct probing, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the run still ended with `Timed out waiting for TCP SYN-ACK` + - current blocker remains at the network edge: even after all-socket ordinary-hint fanout, the peer still only responds through the root relay From bf8a0bdd9e108afd022a2d9194e98bbed4f8b4ce Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:52:07 +0100 Subject: [PATCH 270/296] Rotate ordinary direct hint bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 10 +++++-- ...TierDirectEndpointManagerPushFlagsTests.cs | 8 ++--- .../Internal/ZeroTierDataplaneRuntime.cs | 9 ++++++ .../Internal/ZeroTierDirectEndpointManager.cs | 2 +- docs/logbook/direct-p2p-jaeger.md | 29 +++++++++++++++++++ 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 3cd5672..5180e95 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -324,7 +324,7 @@ await WaitForConditionAsync( .Distinct() .Order() .ToArray(); - Assert.Equal(new[] { 0, 1 }, pushedBootstrapSockets); + Assert.Single(pushedBootstrapSockets); var rendezvousBootstrapSockets = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) @@ -506,7 +506,13 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); - Assert.Contains((1, pushedEndpoint), echoSends); + var pushedEchoSockets = echoSends + .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) + .Select(static send => send.LocalSocketId) + .Distinct() + .ToArray(); + Assert.Single(pushedEchoSockets); + Assert.InRange(pushedEchoSockets[0], 0, 1); Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint) && diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index f9682a3..a36a962 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -156,7 +156,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_NormalHint_UsesAllEligibleSockets() + public async Task PushDirectPaths_NormalHint_UsesPlannerSelectedSocket() { var udp = new RecordingUdpTransport(); @@ -180,12 +180,12 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Equal((endpoint, false, false, 1), hints[0]); Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } [Fact] - public async Task PushDirectPaths_PrivateNormalHint_UsesAllEligibleSockets() + public async Task PushDirectPaths_PrivateNormalHint_UsesPlannerSelectedSocket() { var udp = new RecordingUdpTransport(); @@ -209,7 +209,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Equal((endpoint, false, false, 1), hints[0]); Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d429a3c..7ee8082 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1458,6 +1458,15 @@ private int[] GetDirectHintBootstrapSocketIds( return preferredSocketIds; } + var rotatingSocketIds = _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true); + if (rotatingSocketIds.Length != 0) + { + return rotatingSocketIds; + } + return [receivedLocalSocketId]; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index 8ef5d93..f173ed1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -228,7 +228,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); } } diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index e959f38..583d69a 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -72,3 +72,32 @@ New notes are appended with timestamps. Older entries are not rewritten. - this materially widened direct probing, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet - the run still ended with `Timed out waiting for TCP SYN-ACK` - current blocker remains at the network edge: even after all-socket ordinary-hint fanout, the peer still only responds through the root relay + +## 2026-03-16T14:51:26.6980465+01:00 + +- Checkpoint in progress: + - ordinary pushed hints no longer mean immediate all-socket spray + - normal `PUSH_DIRECT_PATHS` now bootstrap on one planner-selected admissible socket at a time unless the hint carries real socket affinity + - root `RENDEZVOUS` stays pinned to its receiving socket, and direct-only payload still clamps to the pinned rendezvous path +- Why it was tried: + - vendored upstream ordinary hint handling uses an unspecified local socket, which is closer to "let the node choose one route" than "fan out to every socket immediately" + - the previous all-socket spray had become broad enough that it was obscuring whether a narrower route-selected hint path behaved any better +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `ZeroTierDirectHintPathPlannerTests` + - `ZeroTierDirectHintPathPlannerRouteTests` + - there was one transient stale-binary issue during rerun: + - `ZTSharp.Cli` still held `samples/ZTSharp.Cli/bin/Release/net10.0/ZTSharp.dll` open + - the first live run after the code change used the old binary and was discarded + - after the stale process exited, the CLI rebuilt cleanly and the fresh-binary run was used for evaluation +- Live result in `.codex-debug/direct-only-post-rotating-ordinary-hints.log`: + - ordinary alternate hints like `176.66.90.119:19665`, `:27568`, `:44886`, `:48446`, `:8140`, `100.85.196.109:*`, `10.22.10.94:*`, `172.17.0.1:*`, and `172.18.0.1:*` now probe on one socket at a time instead of sockets `0..7` + - the selected socket rotated between update waves, for example socket `0` on one push batch and socket `1` on the next + - root `RENDEZVOUS` remained pinned to `176.66.90.119:62120@3`, and direct-only payload stayed clamped to that single path +- Issue update: + - this successfully narrowed ordinary hint behavior, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer still only returned relayed `OK(HELLO)` and relayed `MulticastLike` traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - current blocker remains unchanged at the wire level: even with route-selected ordinary hint probing and pinned rendezvous payload, the remote peer still does not return confirmed direct traffic to this runtime From 40691e2e97a6889fd5b1368c911be1d8ad24e3bf Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:56:46 +0100 Subject: [PATCH 271/296] Skip alternate hints after rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 24 +++++++-------- .../Internal/ZeroTierDataplaneRuntime.cs | 11 +++++-- docs/logbook/direct-p2p-jaeger.md | 29 +++++++++++++++++++ 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 5180e95..713c58a 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -307,12 +307,6 @@ public async Task DataplaneRuntime_DirectOnly_PrefersRendezvousEndpoint_AfterLat await WaitForConditionAsync( () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), TimeSpan.FromSeconds(2)); - await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => - send.RemoteEndPoint.Equals(pushedEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.Echo), - TimeSpan.FromSeconds(2)); var endpoints = runtime.GetDirectEndpointsForTests(peerIdentity.NodeId); Assert.Equal(rendezvousEndpoint, endpoints[0]); @@ -324,7 +318,7 @@ await WaitForConditionAsync( .Distinct() .Order() .ToArray(); - Assert.Single(pushedBootstrapSockets); + Assert.Empty(pushedBootstrapSockets); var rendezvousBootstrapSockets = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) @@ -506,13 +500,7 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); - var pushedEchoSockets = echoSends - .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) - .Select(static send => send.LocalSocketId) - .Distinct() - .ToArray(); - Assert.Single(pushedEchoSockets); - Assert.InRange(pushedEchoSockets[0], 0, 1); + Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint) && @@ -590,6 +578,7 @@ await WaitForConditionAsync( Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); + Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint) && @@ -998,6 +987,13 @@ await WaitForConditionAsync( TimeSpan.FromSeconds(2)); await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + if (!udp.GetSendsSnapshot().Any(send => + send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.PushDirectPaths)) + { + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + } var pushDirectSends = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 7ee8082..f2e7327 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1372,10 +1372,15 @@ private async ValueTask HandleDirectEndpointHintAsync( { useAllEligibleLocalSockets = false; } - else if (ZeroTierTrace.Enabled) + else { - ZeroTierTrace.WriteLine( - $"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} mode=alternate-hint payloadPinned=True."); + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap skipped: peer={peerNodeId} endpoint={endpoint} mode=alternate-hint payloadPinned=True."); + } + + return; } } diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 583d69a..a09a130 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -101,3 +101,32 @@ New notes are appended with timestamps. Older entries are not rewritten. - the peer still only returned relayed `OK(HELLO)` and relayed `MulticastLike` traffic via `84.17.53.155:9993` - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` - current blocker remains unchanged at the wire level: even with route-selected ordinary hint probing and pinned rendezvous payload, the remote peer still does not return confirmed direct traffic to this runtime + +## 2026-03-16T14:55:13.0296475+01:00 + +- Checkpoint in progress: + - once direct-only mode has a pinned root `RENDEZVOUS` endpoint, alternate pushed hints are now skipped entirely instead of still being probed + - direct-only post-rendezvous behavior is now strictly rendezvous-only for both bootstrap and payload +- Why it was tried: + - the previous run still spent time probing alternate public/private/shared hinted endpoints even after root had already introduced a specific rendezvous endpoint + - that extra hint traffic was the clearest remaining local source of bootstrap noise after payload clamping +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `ZeroTierDirectHintPathPlannerTests` + - `ZeroTierDirectHintPathPlannerRouteTests` +- Live result in `.codex-debug/direct-only-post-rendezvous-only.log`: + - before root sent `RENDEZVOUS`, the peer first sent `PUSH_DIRECT_PATHS`, so the runtime still tried the ordinary hinted set across rotating single sockets and direct-only payload fanout still followed that broad hinted matrix + - later root sent `RENDEZVOUS` for `176.66.90.119:62120` + - after that, the new behavior activated exactly as intended: + - repeated `Direct hint bootstrap skipped ... mode=alternate-hint payloadPinned=True` + - direct-only payload collapsed to `176.66.90.119:62120@7` + - only same-socket hole-punch / `HELLO` / `ECHO` remained on that pinned rendezvous path +- Issue update: + - this removed the post-rendezvous alternate-hint noise, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer still only returned relayed `OK(HELLO)` via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - the blocker is now narrower again: + - pre-rendezvous, root may delay its explicit `RENDEZVOUS` introduction long enough that the runtime still spends part of the connect budget on ordinary hints + - post-rendezvous, even with truly rendezvous-only direct probing and payload, the remote peer still does not return confirmed direct traffic From b39da32d1c4077bacde96a620f1b35bf995b4ae7 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:06:51 +0100 Subject: [PATCH 272/296] Delay direct-only payload until rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 74 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 28 +++++++ docs/logbook/direct-p2p-jaeger.md | 24 ++++++ 3 files changed, 126 insertions(+) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 713c58a..401c222 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -746,6 +746,80 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur Assert.All(payloadSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsBrieflyForRendezvous_BeforeUsingHintedPaths() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + udp.EnqueueInbound( + localSocketId: 0, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + send.RemoteEndPoint.Equals(hintedEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo), + TimeSpan.FromSeconds(1)); + + var hintedSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint)) + .ToArray(); + var payloadSends = hintedSends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .ToArray(); + + Assert.NotEmpty(hintedSends); + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index f2e7327..55cf43e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -25,6 +25,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const int DirectHintMaintenanceProbeBudget = 2; private const long DirectHintFullHelloIntervalMs = 60_000; private const long DirectOnlyHintHelloIntervalMs = 5_000; + private const long DirectOnlyRendezvousGraceMs = 3_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -59,6 +60,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierMultipathOptions _multipath; private readonly ConcurrentDictionary _authenticatedPeers = new(); private readonly ConcurrentDictionary _directBootstrapTasks = new(); + private readonly ConcurrentDictionary _directBootstrapStartedMs = new(); private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); private readonly ConcurrentDictionary _lastDirectPathPushSentMs = new(); private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); @@ -666,6 +668,11 @@ private bool TryGetDirectOnlyHintedPayloadFanout( return false; } + if (ShouldWaitForPinnedRendezvousBeforeHintedPayload(peerNodeId)) + { + return false; + } + var restrictToPinnedRendezvous = ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); var selectedEndpoints = restrictToPinnedRendezvous ? SelectPinnedRendezvousEndpoints(peerNodeId, hinted) @@ -945,6 +952,7 @@ private async ValueTask EnsureDirectBootstrapStartedAsync(NodeId peerNodeId, Can return; } + _directBootstrapStartedMs.TryAdd(peerNodeId, Environment.TickCount64); _ = _directBootstrapTasks.GetOrAdd( peerNodeId, id => BootstrapDirectPathCoreAsync(id, (byte[])sharedKey.Clone(), _cts.Token)); @@ -1732,6 +1740,26 @@ private bool TrySelectConfirmedHintedDirectPath( private bool ShouldUseStickyHintedPathSelection(NodeId peerNodeId) => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + private bool ShouldWaitForPinnedRendezvousBeforeHintedPayload(NodeId peerNodeId) + { + if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) + { + return false; + } + + if (GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints) + { + return false; + } + + if (!_directBootstrapStartedMs.TryGetValue(peerNodeId, out var startedAtMs)) + { + return false; + } + + return unchecked(Environment.TickCount64 - startedAtMs) < DirectOnlyRendezvousGraceMs; + } + private bool ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(NodeId peerNodeId) => ShouldUseStickyHintedPathSelection(peerNodeId) && GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints; diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index a09a130..40810bf 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -130,3 +130,27 @@ New notes are appended with timestamps. Older entries are not rewritten. - the blocker is now narrower again: - pre-rendezvous, root may delay its explicit `RENDEZVOUS` introduction long enough that the runtime still spends part of the connect budget on ordinary hints - post-rendezvous, even with truly rendezvous-only direct probing and payload, the remote peer still does not return confirmed direct traffic + +## 2026-03-16T15:03:47.8133143+01:00 + +- Checkpoint in progress: + - direct-only hinted payload now waits behind a short startup grace window before using ordinary hinted endpoints + - bootstrap still starts immediately, so direct `ECHO` / `HELLO` probing can run while payload holds off waiting for a root `RENDEZVOUS` +- Why it was tried: + - the last issue update showed the remaining local noise was almost entirely pre-rendezvous payload fanout + - root sometimes introduces a rendezvous endpoint quickly enough that sending SYN payload to ordinary hints first was just spending budget before the better path arrived +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_DirectOnly_SynPayload_WaitsBrieflyForRendezvous_BeforeUsingHintedPaths` +- Live result in `.codex-debug/direct-only-post-rendezvous-grace-trace.log`: + - in this traced run, no `TX direct-only payload fanout` happened before root sent `RX RENDEZVOUS: 0x9e072772f9 endpoints: 176.66.90.119:62120` + - after rendezvous arrived, payload clamped immediately to `176.66.90.119:62120@1` + - alternate pushed hints were skipped post-rendezvous as intended +- Issue update: + - this removes the specific pre-rendezvous payload-fanout problem for the traced run, so the remaining blocker is now even cleaner + - despite cleaner startup, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet at all + - all successful peer replies still came back relayed via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` From b7e1a8327ce254eaa602c698e9a00f779733000d Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:16:33 +0100 Subject: [PATCH 273/296] Delay payload after pinned rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 93 +++++++++++++++++++ ...roTierDirectBootstrapControlPolicyTests.cs | 31 +++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 64 +++++++------ .../ZeroTierDirectBootstrapControlPolicy.cs | 25 +++++ docs/logbook/direct-p2p-jaeger.md | 30 ++++++ 5 files changed, 215 insertions(+), 28 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 401c222..acc4db5 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -512,6 +512,99 @@ await WaitForConditionAsync( Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_AndPushesDirectPathsViaRoot() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.PushDirectPaths), + TimeSpan.FromSeconds(2)); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(1)); + + var directSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .ToArray(); + var payloadSends = directSends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .ToArray(); + + Assert.NotEmpty(directSends); + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_MaintenanceStaysOnPinnedRendezvousSocket() { diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 6971c02..0a73c4b 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -20,4 +20,35 @@ public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( allowRootRelayFallback, hasConfirmedDirectPath)); } + + [Theory] + [InlineData(false, false, false, 1_000L, 0L, 3_000L, null, 1_000L, true)] + [InlineData(false, false, false, 3_500L, 0L, 3_000L, null, 1_000L, false)] + [InlineData(false, false, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, true)] + [InlineData(false, false, true, 2_100L, 0L, 3_000L, 1_000L, 1_000L, false)] + [InlineData(false, true, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, false)] + [InlineData(true, false, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, false)] + public void ShouldWaitForDirectOnlyHintedPayload_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + long nowMs, + long? bootstrapStartedAtMs, + long directOnlyRendezvousGraceMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousSettleMs, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldWaitForDirectOnlyHintedPayload( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + nowMs, + bootstrapStartedAtMs, + directOnlyRendezvousGraceMs, + pinnedRendezvousObservedAtMs, + pinnedRendezvousSettleMs)); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 55cf43e..a069bb5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -26,6 +26,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const long DirectHintFullHelloIntervalMs = 60_000; private const long DirectOnlyHintHelloIntervalMs = 5_000; private const long DirectOnlyRendezvousGraceMs = 3_000; + private const long DirectOnlyPinnedRendezvousSettleMs = 1_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -61,6 +62,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ConcurrentDictionary _authenticatedPeers = new(); private readonly ConcurrentDictionary _directBootstrapTasks = new(); private readonly ConcurrentDictionary _directBootstrapStartedMs = new(); + private readonly ConcurrentDictionary _pinnedRendezvousObservedMs = new(); private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); private readonly ConcurrentDictionary _lastDirectPathPushSentMs = new(); private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); @@ -1742,22 +1744,20 @@ private bool ShouldUseStickyHintedPathSelection(NodeId peerNodeId) private bool ShouldWaitForPinnedRendezvousBeforeHintedPayload(NodeId peerNodeId) { - if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) - { - return false; - } - - if (GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints) - { - return false; - } - - if (!_directBootstrapStartedMs.TryGetValue(peerNodeId, out var startedAtMs)) - { - return false; - } - - return unchecked(Environment.TickCount64 - startedAtMs) < DirectOnlyRendezvousGraceMs; + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var nowMs = Environment.TickCount64; + _ = _directBootstrapStartedMs.TryGetValue(peerNodeId, out var bootstrapStartedAtMs); + _ = _pinnedRendezvousObservedMs.TryGetValue(peerNodeId, out var pinnedRendezvousObservedAtMs); + + return ZeroTierDirectBootstrapControlPolicy.ShouldWaitForDirectOnlyHintedPayload( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + nowMs, + bootstrapStartedAtMs == 0 ? null : bootstrapStartedAtMs, + DirectOnlyRendezvousGraceMs, + pinnedRendezvousObservedAtMs == 0 ? null : pinnedRendezvousObservedAtMs, + DirectOnlyPinnedRendezvousSettleMs); } private bool ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(NodeId peerNodeId) @@ -2316,25 +2316,33 @@ public async ValueTask DisposeAsync() private Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken cancellationToken) => _peerSecurity.GetPeerKeyAsync(peerNodeId, cancellationToken); - private ValueTask HandleRootControlPacketAsync( + private async ValueTask HandleRootControlPacketAsync( ZeroTierVerb verb, ReadOnlyMemory payload, int receivedLocalSocketId, IPEndPoint receivedVia, CancellationToken cancellationToken) - { - if (verb != ZeroTierVerb.Rendezvous) - { - return ValueTask.CompletedTask; - } - - if (!ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) || rendezvous.With.Value == 0) - { - return ValueTask.CompletedTask; - } + { + if (verb != ZeroTierVerb.Rendezvous) + { + return; + } + + if (!ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) || rendezvous.With.Value == 0) + { + return; + } var directEndpoints = GetOrCreateDirectEndpointManager(rendezvous.With); - return directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken); + await directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken) + .ConfigureAwait(false); + + _pinnedRendezvousObservedMs[rendezvous.With] = Environment.TickCount64; + + if (_peerSecurity.TryGetPeerKey(rendezvous.With, out var sharedKey)) + { + await SendPushDirectPathsViaRootAsync(rendezvous.With, sharedKey, cancellationToken).ConfigureAwait(false); + } } private ValueTask HandlePeerControlPacketAsync( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index f9ed9f0..1f8b2fc 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -6,4 +6,29 @@ public static bool ShouldRefreshRelayedRootControl( bool allowRootRelayFallback, bool hasConfirmedDirectPath) => allowRootRelayFallback || !hasConfirmedDirectPath; + + public static bool ShouldWaitForDirectOnlyHintedPayload( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + long nowMs, + long? bootstrapStartedAtMs, + long directOnlyRendezvousGraceMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousSettleMs) + { + if (allowRootRelayFallback || hasConfirmedDirectPath) + { + return false; + } + + if (hasPinnedRendezvousEndpoints) + { + return pinnedRendezvousObservedAtMs.HasValue && + unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousSettleMs; + } + + return bootstrapStartedAtMs.HasValue && + unchecked(nowMs - bootstrapStartedAtMs.Value) < directOnlyRendezvousGraceMs; + } } diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 40810bf..9b2c8e7 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -154,3 +154,33 @@ New notes are appended with timestamps. Older entries are not rewritten. - despite cleaner startup, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet at all - all successful peer replies still came back relayed via `84.17.53.155:9993` - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + +## 2026-03-16T15:16:06.4435500+01:00 + +- Checkpoint in progress: + - once root introduces a pinned rendezvous endpoint, the runtime now immediately relays current local direct-path advertisements back to the peer + - direct-only payload also stays suppressed briefly after that pinned rendezvous is learned +- Why it was tried: + - the previous checkpoint cleaned up pre-rendezvous payload fanout, but payload still started almost immediately after rendezvous + - if the peer needed one more relayed advertisement round to learn our public surfaces, that narrow post-rendezvous window was the last obvious local bootstrap race +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_AndPushesDirectPathsViaRoot` +- Live result in `.codex-debug/direct-only-post-rendezvous-settle-trace.log`: + - right after `RX RENDEZVOUS: 0x9e072772f9 endpoints: 176.66.90.119:62120`, the runtime now immediately emits: + - `TX PUSH_DIRECT_PATHS via root for 0x9e072772f9: 212.241.85.84:53972 ... :53979` + - direct-only payload remained pinned to `176.66.90.119:62120@2` + - alternate hinted endpoints still stayed skipped once payload was pinned +- Issue update: + - this removes one more plausible local race, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer continued to return only relayed `OK(HELLO)`, relayed `PushDirectPaths`, and relayed `MulticastLike` traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - the blocker is now even more explicit: + - root introduction happens + - our public surfaces are relayed back immediately + - direct bootstrap and payload stay pinned to the rendezvous endpoint + - the remote peer still never sends a confirmed direct packet to this runtime From e0f9c1b53a792ff35913660ff5453d9b5bf11a37 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:21:14 +0100 Subject: [PATCH 274/296] Advertise full local paths after rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 13 +++++++-- .../Internal/ZeroTierDataplaneRuntime.cs | 17 +++++++++--- docs/logbook/direct-p2p-jaeger.md | 27 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index acc4db5..018005b 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -516,8 +516,8 @@ await WaitForConditionAsync( public async Task DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_AndPushesDirectPathsViaRoot() { await using var udp = new ScriptedUdpTransport( - new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), - new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); @@ -568,6 +568,15 @@ await WaitForConditionAsync( verb == ZeroTierVerb.PushDirectPaths), TimeSpan.FromSeconds(2)); + var pushPacket = udp.GetSendsSnapshot() + .First(send => + send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.PushDirectPaths); + var pushedEndpoints = ReadPushedDirectPathEndpoints(pushPacket.Payload, sharedKey); + Assert.Contains(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), pushedEndpoints); + Assert.Contains(new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000), pushedEndpoints); + var tcp = TcpCodec.Encode( IPAddress.Parse("10.0.0.1"), IPAddress.Parse("10.0.0.2"), diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index a069bb5..95da828 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1186,9 +1186,15 @@ await SendHelloPacketAsync( } } - private async Task SendPushDirectPathsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + private async Task SendPushDirectPathsViaRootAsync( + NodeId peerNodeId, + byte[] sharedKey, + CancellationToken cancellationToken, + bool useFullLocalAdvertisements = false) { - var advertisements = GetPeerAwareLocalDirectPathAdvertisements(peerNodeId); + var advertisements = useFullLocalAdvertisements + ? GetLocalDirectPathAdvertisements() + : GetPeerAwareLocalDirectPathAdvertisements(peerNodeId); if (advertisements.Length == 0) { return; @@ -2341,7 +2347,12 @@ await directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocket if (_peerSecurity.TryGetPeerKey(rendezvous.With, out var sharedKey)) { - await SendPushDirectPathsViaRootAsync(rendezvous.With, sharedKey, cancellationToken).ConfigureAwait(false); + await SendPushDirectPathsViaRootAsync( + rendezvous.With, + sharedKey, + cancellationToken, + useFullLocalAdvertisements: true) + .ConfigureAwait(false); } } diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 9b2c8e7..894ea3f 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -184,3 +184,30 @@ New notes are appended with timestamps. Older entries are not rewritten. - our public surfaces are relayed back immediately - direct bootstrap and payload stay pinned to the rendezvous endpoint - the remote peer still never sends a confirmed direct packet to this runtime + +## 2026-03-16T15:20:46.9796150+01:00 + +- Checkpoint in progress: + - the immediate relayed `PUSH_DIRECT_PATHS` triggered by root `RENDEZVOUS` now uses the full local advertisement set instead of the peer-aware filtered subset + - that means the peer sees both current public surfaces and local private surfaces immediately after rendezvous, not only the public ones +- Why it was tried: + - in the previous trace, the first relayed advertisement right after rendezvous contained only public `212.241.85.84:*` surfaces + - if the peer needed our private/local surfaces early for the shared/private hints it later advertised (`100.85.196.109:*`, `10.22.10.94:*`), filtering them out in that first post-rendezvous push was still a plausible local gap +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - regression coverage for the rendezvous-triggered root push now asserts both: + - public surfaces are included + - local private surfaces are included +- Live result in `.codex-debug/direct-only-post-rendezvous-full-local-ads.log`: + - the first two relayed pushes after `RX RENDEZVOUS` now show: + - one public-only push from the existing bootstrap path already in flight + - then the new full local push including `10.0.0.60:58487 ... :58494` + - later relayed pushes consistently included both public and local private surfaces + - direct-only payload still stayed pinned to `176.66.90.119:62120@6` +- Issue update: + - this closed another plausible local advertisement gap, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer still returned only relayed `OK(HELLO)`, relayed `PushDirectPaths`, and relayed `MulticastLike` traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` From 30bf1313f0d8060794ea03809d4b30ab0fb2c914 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:24:31 +0100 Subject: [PATCH 275/296] Send full local ads before rendezvous bootstrap --- .../Internal/ZeroTierDataplaneRuntime.cs | 12 +++++----- docs/logbook/direct-p2p-jaeger.md | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 95da828..b83022b 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -2339,12 +2339,6 @@ private async ValueTask HandleRootControlPacketAsync( return; } - var directEndpoints = GetOrCreateDirectEndpointManager(rendezvous.With); - await directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken) - .ConfigureAwait(false); - - _pinnedRendezvousObservedMs[rendezvous.With] = Environment.TickCount64; - if (_peerSecurity.TryGetPeerKey(rendezvous.With, out var sharedKey)) { await SendPushDirectPathsViaRootAsync( @@ -2354,6 +2348,12 @@ await SendPushDirectPathsViaRootAsync( useFullLocalAdvertisements: true) .ConfigureAwait(false); } + + var directEndpoints = GetOrCreateDirectEndpointManager(rendezvous.With); + await directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken) + .ConfigureAwait(false); + + _pinnedRendezvousObservedMs[rendezvous.With] = Environment.TickCount64; } private ValueTask HandlePeerControlPacketAsync( diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 894ea3f..1eeece8 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -211,3 +211,27 @@ New notes are appended with timestamps. Older entries are not rewritten. - this closed another plausible local advertisement gap, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet - the peer still returned only relayed `OK(HELLO)`, relayed `PushDirectPaths`, and relayed `MulticastLike` traffic via `84.17.53.155:9993` - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + +## 2026-03-16T15:24:03.5623456+01:00 + +- Checkpoint in progress: + - root `RENDEZVOUS` handling now sends the full local relayed `PUSH_DIRECT_PATHS` before starting the direct rendezvous bootstrap, not after +- Why it was tried: + - the previous trace still showed the direct bootstrap packet going out before the full local advertisement round + - if the peer needed the relayed advertisement first, the remaining local race was the order inside the rendezvous handler itself +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` +- Live result in `.codex-debug/direct-only-post-rendezvous-full-first.log`: + - this run first received relayed `PUSH_DIRECT_PATHS` from the peer and spent most of the connect budget probing the broad hinted set + - root `RENDEZVOUS` arrived later in the run for `176.66.90.119:62120` + - after that, direct-only behavior narrowed again to the pinned rendezvous path and the relayed local path pushes were still the full local set +- Issue update: + - reordering the rendezvous handler did not change the outcome + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet + - the peer still only returned relayed traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - the active blocker remains the same: + - even after late root introduction and full local advertisement replay, the remote peer never establishes or answers a direct path to this runtime From 76b8100b767b892169366a8240418590d354e61a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:36:06 +0100 Subject: [PATCH 276/296] Require rendezvous for direct-only payload --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 69 ++++++++----------- ...roTierDirectBootstrapControlPolicyTests.cs | 2 +- .../ZeroTierDirectBootstrapControlPolicy.cs | 9 ++- docs/logbook/direct-p2p-jaeger.md | 30 ++++++++ 4 files changed, 65 insertions(+), 45 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 018005b..79e9f2e 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -332,7 +332,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedPaths_BeforeConfirmation() + public async Task DataplaneRuntime_DirectOnly_SynPayload_DoesNotUseHintedPayloadBeforeRendezvous() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -385,30 +385,26 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_FansOutAcrossAllHintedP tcp, identification: 1); - await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + hintedEndpoints.Contains(send.RemoteEndPoint)), + TimeSpan.FromSeconds(1)); var sends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .ToArray(); - var echoSends = sends - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) - .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) - .Distinct() - .OrderBy(send => send.RemoteEndPoint.Port) - .ThenBy(send => send.LocalSocketId) - .ToArray(); - var payloadSends = sends .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) .OrderBy(send => send.RemoteEndPoint.Port) .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal(hintedEndpoints, echoSends.Select(send => send.RemoteEndPoint).ToArray()); - Assert.Equal(hintedEndpoints, payloadSends.Select(send => send.RemoteEndPoint).ToArray()); - Assert.All(echoSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); - Assert.All(payloadSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); } [Fact] @@ -689,7 +685,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingEcho_UsesHintedPeerEndpoint() + public async Task DataplaneRuntime_DirectOnly_AdvertisedSurfaces_DoNotEnableHintedPayloadBeforeRendezvous() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -747,23 +743,24 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_PrimingEcho_UsesHintedP tcp, identification: 1); - await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await Task.Delay(300); - var echoSends = udp.GetSendsSnapshot() + var sends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) - .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) - .Distinct() - .OrderBy(send => send.RemoteEndPoint.Port) - .ThenBy(send => send.LocalSocketId) + .ToArray(); + var payloadSends = sends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) .ToArray(); - Assert.Equal(hintedEndpoints, echoSends.Select(send => send.RemoteEndPoint).ToArray()); - Assert.All(echoSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_DuringBootstrap() + public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForRendezvousEvenAfterHintedPathsAppear() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -814,25 +811,21 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur tcp, identification: 1); - var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None).AsTask(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); await Task.Delay(300); runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); - await sendTask.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + hintedEndpoints.Contains(send.RemoteEndPoint)), + TimeSpan.FromSeconds(1)); var sends = udp.GetSendsSnapshot() .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .ToArray(); - var echoSends = sends - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) - .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) - .Distinct() - .OrderBy(send => send.RemoteEndPoint.Port) - .ThenBy(send => send.LocalSocketId) - .ToArray(); - var payloadSends = sends .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) @@ -842,10 +835,8 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForHintedPaths_Dur .ThenBy(send => send.LocalSocketId) .ToArray(); - Assert.Equal(hintedEndpoints, echoSends.Select(send => send.RemoteEndPoint).ToArray()); - Assert.Equal(hintedEndpoints, payloadSends.Select(send => send.RemoteEndPoint).ToArray()); - Assert.All(echoSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); - Assert.All(payloadSends, send => Assert.InRange(send.LocalSocketId, 0, 1)); + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 0a73c4b..887610c 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -23,7 +23,7 @@ public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( [Theory] [InlineData(false, false, false, 1_000L, 0L, 3_000L, null, 1_000L, true)] - [InlineData(false, false, false, 3_500L, 0L, 3_000L, null, 1_000L, false)] + [InlineData(false, false, false, 3_500L, 0L, 3_000L, null, 1_000L, true)] [InlineData(false, false, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, true)] [InlineData(false, false, true, 2_100L, 0L, 3_000L, 1_000L, 1_000L, false)] [InlineData(false, true, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, false)] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index 1f8b2fc..f12e787 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -22,13 +22,12 @@ public static bool ShouldWaitForDirectOnlyHintedPayload( return false; } - if (hasPinnedRendezvousEndpoints) + if (!hasPinnedRendezvousEndpoints) { - return pinnedRendezvousObservedAtMs.HasValue && - unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousSettleMs; + return true; } - return bootstrapStartedAtMs.HasValue && - unchecked(nowMs - bootstrapStartedAtMs.Value) < directOnlyRendezvousGraceMs; + return pinnedRendezvousObservedAtMs.HasValue && + unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousSettleMs; } } diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 1eeece8..c3fe41b 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -235,3 +235,33 @@ New notes are appended with timestamps. Older entries are not rewritten. - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` - the active blocker remains the same: - even after late root introduction and full local advertisement replay, the remote peer never establishes or answers a direct path to this runtime + +## 2026-03-16T15:35:37.1697952+01:00 + +- Checkpoint in progress: + - direct-only payload is now gated harder and no longer uses ordinary hinted endpoints before there is either a pinned root `RENDEZVOUS` endpoint or a confirmed hop-0 path + - focused tests were updated to reflect the new contract: ordinary hints remain bootstrap-only in direct-only mode +- Why it was tried: + - the previous live runs still spent part of the connect budget on broad ordinary hinted payload fanout before root introduced the public rendezvous endpoint + - if that early hinted payload was confusing the peer or simply wasting the narrow connect budget, the runtime needed to separate control bootstrap from payload eligibility more strictly +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` passed +- Live result in `.codex-debug/direct-only-post-rendezvous-required.log`: + - root supplied `RENDEZVOUS` for `176.66.90.119:62120` early in the run + - after that, direct-only payload stayed pinned to exactly `176.66.90.119:62120@3` + - alternate hinted endpoints were skipped once payload was pinned + - the runtime still emitted relayed `PUSH_DIRECT_PATHS` with both public and local private surfaces +- Issue update: + - this removed the earlier pre-rendezvous direct-only payload spray, but it still did not produce any direct return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet at all + - every successful control reply remained relayed `OK(HELLO)` via `84.17.53.155:9993` + - the peer also continued sending relayed `PushDirectPaths` and relayed `MulticastLike` traffic during the pinned rendezvous phase + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - current blocker update: + - the local runtime now cleanly waits for root rendezvous before direct-only payload + - root rendezvous arrives and payload clamps correctly to one endpoint/socket + - the remote peer still never returns a confirmed direct packet to this runtime From 4790586a20cbfb5a2cf37c108eefa54418653ea5 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:48:28 +0100 Subject: [PATCH 277/296] Honor receiving socket for direct hint bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 67 ++++++++++++++++++- .../Internal/ZeroTierDataplaneRuntime.cs | 11 +++ docs/logbook/direct-p2p-jaeger.md | 31 +++++++++ 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 79e9f2e..dbb5047 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -408,7 +408,59 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_PrefersPinnedRendezvous_AndFansOutOnStickySocket() + public async Task DataplaneRuntime_DirectOnly_PushDirectPaths_BootstrapsOnReceivingSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + send.RemoteEndPoint.Equals(hintedEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo), + TimeSpan.FromSeconds(1)); + + var echoSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .Select(send => send.LocalSocketId) + .Distinct() + .ToArray(); + + Assert.Equal([1], echoSends); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_PrefersPinnedRendezvous_ButWaitsForConfirmation() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -477,7 +529,15 @@ await WaitForConditionAsync( tcp, identification: 1); - await runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + (verb == ZeroTierVerb.Echo || verb == ZeroTierVerb.Hello)), + TimeSpan.FromSeconds(1)); var directSends = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) @@ -503,9 +563,10 @@ await WaitForConditionAsync( send.LocalSocketId == 1 && TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello); - Assert.Contains((1, rendezvousEndpoint), payloadSends); + Assert.Empty(payloadSends); Assert.DoesNotContain(payloadSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index b83022b..42ba2dd 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -481,6 +481,12 @@ private async ValueTask SendToPeerAsync( var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; if (preferHintedDirectFanout && + !_multipath.AllowRootRelayFallback && + !HasConfirmedDirectPath(peerNodeId)) + { + await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + } + else if (preferHintedDirectFanout && !HasConfirmedDirectPath(peerNodeId) && await TrySendDirectOnlyHintedPayloadAsync(peerNodeId, packet, flowId, parsedOk ? parsed.PacketId : null, shouldRecord, cancellationToken).ConfigureAwait(false)) { @@ -1473,6 +1479,11 @@ private int[] GetDirectHintBootstrapSocketIds( return _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); } + if (receivedLocalSocketId >= 0) + { + return [receivedLocalSocketId]; + } + var preferredSocketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); if (preferredSocketIds.Length != 0) { diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index c3fe41b..3667d10 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -265,3 +265,34 @@ New notes are appended with timestamps. Older entries are not rewritten. - the local runtime now cleanly waits for root rendezvous before direct-only payload - root rendezvous arrives and payload clamps correctly to one endpoint/socket - the remote peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T15:47:59.8613883+01:00 + +- Checkpoint in progress: + - single-socket direct hint bootstrap now truly honors the receiving socket instead of silently falling back to a planner-preferred socket + - direct-only payload still remains confirmation-gated from the previous checkpoint +- Why it was tried: + - the previous stricter direct-only run exposed a concrete local mismatch in the trace: + - `Direct hint bootstrap ... socket=7` + - followed by `TX ECHO bootstrap ... (socket=0)` + - that meant the runtime was logging one socket choice and sending on another for ordinary single-socket hint bootstrap, which could break NAT state and invalidate the bootstrap attempt before root rendezvous or on similar hint paths +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_DirectOnly_PushDirectPaths_BootstrapsOnReceivingSocket` +- Live result in `.codex-debug/direct-only-post-hint-socket-fix.log`: + - this particular rerun still got root `RENDEZVOUS` before the broad ordinary-hint bootstrap could dominate, so the old pre-rendezvous mismatch did not reappear in the same form + - the trace did confirm the pinned rendezvous path remained socket-consistent: + - `Direct hint bootstrap ... socket=7` + - `TX ECHO bootstrap to 176.66.90.119:62120 (socket=7)` + - `TX HELLO bootstrap to 176.66.90.119:62120 (socket=7, reported=176.66.90.119:62120)` +- Issue update: + - this closes another real local transport bug, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet + - all successful control replies remained relayed `OK(HELLO)` via `84.17.53.155:9993` + - the active blocker is unchanged: + - bootstrap is now cleaner and more socket-consistent + - the remote peer still never returns a confirmed direct packet to this runtime From 6c2b90b9ba7b23ca4ccfd714a48cc961f5de21f5 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:59:30 +0100 Subject: [PATCH 278/296] Refresh root hello on rendezvous socket --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 54 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 34 +++++++++--- docs/logbook/direct-p2p-jaeger.md | 46 ++++++++++++++++ 3 files changed, 126 insertions(+), 8 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index dbb5047..2cfa791 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -106,6 +106,60 @@ await WaitForConditionAsync( Assert.All(bootstrapSends, send => Assert.Equal(1, send.LocalSocketId)); } + [Fact] + public async Task DataplaneRuntime_Rendezvous_RefreshesRootHello_OnReceivingSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + var rootHelloSend = await WaitForSendAsync( + udp, + send => send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello, + TimeSpan.FromSeconds(2)); + + Assert.Equal(1, rootHelloSend.LocalSocketId); + } + [Fact] public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 42ba2dd..221e988 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1160,12 +1160,10 @@ private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, Ca if (ShouldUsePeerRootSocketAffinity(peerNodeId) && _peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) { - await SendHelloPacketAsync( - rootSocketId, + await SendHelloViaRootSocketAsync( peerNodeId, - physicalDestination: null, - _rootEndpoint, sharedKey, + rootSocketId, cancellationToken) .ConfigureAwait(false); return; @@ -1181,17 +1179,28 @@ await SendHelloPacketAsync(localSocketId: 0, peerNodeId, physicalDestination: nu for (var i = 0; i < localSockets.Count; i++) { - await SendHelloPacketAsync( - localSockets[i].Id, + await SendHelloViaRootSocketAsync( peerNodeId, - physicalDestination: null, - _rootEndpoint, sharedKey, + localSockets[i].Id, cancellationToken) .ConfigureAwait(false); } } + private Task SendHelloViaRootSocketAsync( + NodeId peerNodeId, + byte[] sharedKey, + int localSocketId, + CancellationToken cancellationToken) + => SendHelloPacketAsync( + localSocketId, + peerNodeId, + physicalDestination: null, + _rootEndpoint, + sharedKey, + cancellationToken); + private async Task SendPushDirectPathsViaRootAsync( NodeId peerNodeId, byte[] sharedKey, @@ -2350,8 +2359,17 @@ private async ValueTask HandleRootControlPacketAsync( return; } + _peerRootSocketAffinity.Observe(rendezvous.With, receivedLocalSocketId, receivedVia); + if (_peerSecurity.TryGetPeerKey(rendezvous.With, out var sharedKey)) { + await SendHelloViaRootSocketAsync( + rendezvous.With, + sharedKey, + receivedLocalSocketId, + cancellationToken) + .ConfigureAwait(false); + await SendPushDirectPathsViaRootAsync( rendezvous.With, sharedKey, diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 3667d10..fd8e612 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -296,3 +296,49 @@ New notes are appended with timestamps. Older entries are not rewritten. - the active blocker is unchanged: - bootstrap is now cleaner and more socket-consistent - the remote peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T15:56:07.0106456+01:00 + +- Checkpoint in progress: + - root `RENDEZVOUS` handling now immediately records peer-root socket affinity from the receiving socket + - on the same transition, the runtime now sends a relayed root `HELLO` refresh on that exact socket before replaying local `PUSH_DIRECT_PATHS` +- Why it was tried: + - the current live traces still show root introducing a rendezvous endpoint and the runtime then probing it directly on one socket, but there is still no hop-0 return traffic at all + - one remaining stale-mapping risk was that root introduction and our direct bootstrap were happening without an immediate same-socket root mapping refresh at the handoff point + - if root or the peer was still operating on a slightly stale observed mapping for that socket, refreshing root `HELLO` first is the narrowest remaining protocol-consistent fix +- Verification: + - added regression coverage: + - `DataplaneRuntime_Rendezvous_RefreshesRootHello_OnReceivingSocket` +- Issue update: + - this is intended to remove one more potential stale-public-surface race at the exact moment root introduces the peer + - expected next observation: + - either the peer starts returning hop-0 control after the refreshed introduction window + - or the traces stay unchanged, which would narrow the blocker further away from root-side surface freshness and toward the remote peer never attempting direct return at all + +## 2026-03-16T15:58:34.9946276+01:00 + +- Checkpoint in progress: + - the same-socket root `HELLO` refresh on `RENDEZVOUS` is now live, tested, and rerun against Jaeger with full tracing +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-root-refresh-on-rendezvous.log`: + - root again introduced `176.66.90.119:62120` + - immediately around that transition the runtime now does the intended same-socket root refresh behavior on socket `4`: + - relayed root `HELLO` + - relayed `PUSH_DIRECT_PATHS` + - direct TTL-2 hole-punch + - direct `HELLO` / `ECHO` bootstrap + - repeated relayed peer control still came back only through `84.17.53.155:9993` +- Issue update: + - this removed the remaining suspected stale-root-mapping race at rendezvous handoff, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the direct-only call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + - blocker update: + - root-side rendezvous handoff is now cleaner and same-socket refreshed + - the peer still never attempts or never succeeds at any direct return packet to this runtime From 5d1cf8f6e01839a9e64bda0d6a8a505aa702c3b1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:04:00 +0100 Subject: [PATCH 279/296] Pin root control after rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 64 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 4 +- docs/logbook/direct-p2p-jaeger.md | 42 +++++++++++- 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 2cfa791..911b5fd 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -1559,6 +1559,70 @@ public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithObservedRootPa Assert.Equal(new[] { 0, 1 }, socketIds); } + [Fact] + public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithPinnedRendezvous_UsesObservedRootSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = false + }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await runtime.SendHelloViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var socketIds = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && decoded.Header.Verb == ZeroTierVerb.Hello) + .Select(send => send.LocalSocketId) + .Distinct() + .ToArray(); + + Assert.Equal(new[] { 1 }, socketIds); + } + private static ZeroTierDataplaneRuntime CreateRuntime( IZeroTierUdpTransport udp, ZeroTierIdentity localIdentity, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 221e988..44d6538 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1292,7 +1292,9 @@ private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet private bool ShouldUsePeerRootSocketAffinity(NodeId peerNodeId) => _multipath.AllowRootRelayFallback || - HasConfirmedDirectPath(peerNodeId); + HasConfirmedDirectPath(peerNodeId) || + (!_multipath.AllowRootRelayFallback && + GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints); private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) { diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index fd8e612..90eb60a 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -339,6 +339,44 @@ New notes are appended with timestamps. Older entries are not rewritten. - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` - the direct-only call still failed with: - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` +- blocker update: + - root-side rendezvous handoff is now cleaner and same-socket refreshed + - the peer still never attempts or never succeeds at any direct return packet to this runtime + +## 2026-03-16T16:03:23.8257280+01:00 + +- Checkpoint in progress: + - direct-only relayed root control now uses peer-root socket affinity as soon as rendezvous is pinned, instead of waiting for a confirmed hop-0 path + - this keeps post-rendezvous root `HELLO` and root `NETWORK_CREDENTIALS` on one socket in direct-only mode +- Why it was tried: + - the previous live rerun proved rendezvous handoff was same-socket refreshed, but subsequent relayed root maintenance still fanned out across all sockets + - that left one remaining local instability source: + - root and peer could keep seeing all eight public socket surfaces while direct-only payload and direct bootstrap were already pinned to one rendezvous socket + - the narrowest next fix was to keep relayed root control aligned with the pinned rendezvous/root-affine socket too +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithPinnedRendezvous_UsesObservedRootSocket` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-pinned-root-affinity.log`: + - root introduced `176.66.90.119:17569` + - after that, relayed root control stayed pinned to socket `7`: + - `TX HELLO bootstrap to 84.17.53.155:9993 (socket=7, reported=)` + - `TX NETWORK_CREDENTIALS via root for 0x9e072772f9` + - direct control and payload bootstrap also stayed pinned to the same rendezvous path and socket: + - `TX hole-punch to 176.66.90.119:17569 (socket=7, hopLimit=2)` + - `TX HELLO bootstrap to 176.66.90.119:17569 (socket=7, reported=176.66.90.119:17569)` + - later repeated `TX ECHO bootstrap to 176.66.90.119:17569 (socket=7)` +- Issue update: + - this removed the last obvious local root-control fanout after rendezvous, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - every successful control reply still came back relayed `hop=1` via `84.17.53.155:9993` + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` - blocker update: - - root-side rendezvous handoff is now cleaner and same-socket refreshed - - the peer still never attempts or never succeeds at any direct return packet to this runtime + - direct-only behavior is now pinned not just for direct bootstrap and payload, but also for relayed root control after rendezvous + - the peer still never returns a direct packet to this runtime even under that stricter single-socket regime From 1627cb5c72fe8f829e2b7a17f15a31205d006ad9 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:12:04 +0100 Subject: [PATCH 280/296] Delay rendezvous probes until peer version known --- ...roTierDirectBootstrapControlPolicyTests.cs | 28 ++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 27 ++++++++++++++-- .../ZeroTierDirectBootstrapControlPolicy.cs | 21 ++++++++++++ docs/logbook/direct-p2p-jaeger.md | 32 +++++++++++++++++++ 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 887610c..8380caa 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -4,6 +4,34 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectBootstrapControlPolicyTests { + [Theory] + [InlineData(false, false, true, false, 1_200L, 1_000L, 750L, true)] + [InlineData(false, false, true, false, 1_800L, 1_000L, 750L, false)] + [InlineData(false, false, true, true, 1_200L, 1_000L, 750L, false)] + [InlineData(false, true, true, false, 1_200L, 1_000L, 750L, false)] + [InlineData(true, false, true, false, 1_200L, 1_000L, 750L, false)] + public void ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousVersionGraceMs, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + canUseEchoForDirectBootstrap, + nowMs, + pinnedRendezvousObservedAtMs, + pinnedRendezvousVersionGraceMs)); + } + [Theory] [InlineData(true, false, true)] [InlineData(true, true, true)] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 44d6538..ae444fe 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -27,6 +27,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const long DirectOnlyHintHelloIntervalMs = 5_000; private const long DirectOnlyRendezvousGraceMs = 3_000; private const long DirectOnlyPinnedRendezvousSettleMs = 1_000; + private const long DirectOnlyPinnedRendezvousVersionGraceMs = 750; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -1004,8 +1005,15 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (hinted.Length > 0 && unchecked(now - nextHintProbeAt) >= 0) { - await ProbeHintedDirectEndpointsAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); - nextHintProbeAt = now + DirectBootstrapHintProbeIntervalMs; + if (ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(peerNodeId)) + { + nextHintProbeAt = now + 250; + } + else + { + await ProbeHintedDirectEndpointsAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); + nextHintProbeAt = now + DirectBootstrapHintProbeIntervalMs; + } } if (unchecked(Environment.TickCount64 - deadline) >= 0) @@ -1792,6 +1800,21 @@ private bool ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(NodeId peer => ShouldUseStickyHintedPathSelection(peerNodeId) && GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints; + private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId peerNodeId) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + _ = _pinnedRendezvousObservedMs.TryGetValue(peerNodeId, out var pinnedRendezvousObservedAtMs); + + return ZeroTierDirectBootstrapControlPolicy.ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + CanUseEchoForDirectBootstrap(peerNodeId), + Environment.TickCount64, + pinnedRendezvousObservedAtMs == 0 ? null : pinnedRendezvousObservedAtMs, + DirectOnlyPinnedRendezvousVersionGraceMs); + } + private int[] GetStickyDirectHintProbeSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) { if (ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index f12e787..5f66379 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -2,6 +2,27 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierDirectBootstrapControlPolicy { + public static bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousVersionGraceMs) + { + if (allowRootRelayFallback || + hasConfirmedDirectPath || + !hasPinnedRendezvousEndpoints || + canUseEchoForDirectBootstrap || + !pinnedRendezvousObservedAtMs.HasValue) + { + return false; + } + + return unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousVersionGraceMs; + } + public static bool ShouldRefreshRelayedRootControl( bool allowRootRelayFallback, bool hasConfirmedDirectPath) diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 90eb60a..0d02f46 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -380,3 +380,35 @@ New notes are appended with timestamps. Older entries are not rewritten. - blocker update: - direct-only behavior is now pinned not just for direct bootstrap and payload, but also for relayed root control after rendezvous - the peer still never returns a direct packet to this runtime even under that stricter single-socket regime + +## 2026-03-16T16:11:24.0584259+01:00 + +- Checkpoint in progress: + - direct-only pinned-rendezvous bootstrap now waits briefly for relayed `OK(HELLO)` peer-version learning before probing the pinned rendezvous endpoint + - once peer version is known, the first pinned-rendezvous probe now starts with direct `ECHO` instead of immediately sending full `HELLO` +- Why it was tried: + - the previous live run showed the rendezvous path and relayed root control were finally pinned to one socket, but the first post-rendezvous direct probe could still be a full `HELLO` + - upstream modern peers prefer `ECHO`-first direct probing once version is known, so a narrow grace window after rendezvous was the next parity fix + - if the peer was discarding or deprioritizing the first post-rendezvous direct `HELLO`, waiting a moment for relayed version learning was the narrowest remaining sequencing change +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-version-grace.log`: + - root introduced `176.66.90.119:17569` + - after the grace window, the first pinned-rendezvous direct probe was now: + - `Direct hint bootstrap: peer=0x9e072772f9 endpoint=176.66.90.119:17569 socket=4 fullHello=False allSockets=False` + - `TX ECHO bootstrap to 176.66.90.119:17569 (socket=4)` + - later maintenance still escalated to same-socket direct `HELLO` as expected + - relayed root control remained pinned to the same socket and continued to return only relayed `OK(HELLO)` via `84.17.53.155:9993` +- Issue update: + - this removed the remaining local sequencing mismatch at the start of pinned-rendezvous probing + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + - blocker update: + - the pinned rendezvous path is now `ECHO`-first on the receiving socket, which matches the intended modern-peer bootstrap shape more closely + - the peer still never returns any confirmed direct packet to this runtime even after that sequencing fix From 29985a03f1d259fcba92e1ff60117f54282ff8fb Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:22:42 +0100 Subject: [PATCH 281/296] Delay direct rendezvous callback probes --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 55 +++++++++++++++++++ ...roTierDirectBootstrapControlPolicyTests.cs | 26 +++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 38 ++++++++++++- .../ZeroTierDirectBootstrapControlPolicy.cs | 19 +++++++ docs/logbook/direct-p2p-jaeger.md | 33 +++++++++++ 5 files changed, 168 insertions(+), 3 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 911b5fd..cac0aef 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -160,6 +160,61 @@ public async Task DataplaneRuntime_Rendezvous_RefreshesRootHello_OnReceivingSock Assert.Equal(1, rootHelloSend.LocalSocketId); } + [Fact] + public async Task DataplaneRuntime_Rendezvous_DelaysInitialDirectProbeUntilPeerVersionKnown() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity, peerProtocolVersion: 0, peerMajorVersion: 0, peerMinorVersion: 0, peerRevision: 0); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + await WaitForSendAsync( + udp, + send => send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(RootEndpoint) && + ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && + decoded.Header.Verb == ZeroTierVerb.Hello, + TimeSpan.FromSeconds(2)); + + await Task.Delay(150); + + Assert.DoesNotContain( + udp.GetSendsSnapshot(), + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.HopLimit is null); + } + [Fact] public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint() { diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 8380caa..cb28694 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -4,6 +4,32 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectBootstrapControlPolicyTests { + [Theory] + [InlineData(false, false, true, 1_200L, 1_000L, 2_000L, true)] + [InlineData(false, false, true, 3_500L, 1_000L, 2_000L, false)] + [InlineData(false, true, true, 1_200L, 1_000L, 2_000L, false)] + [InlineData(true, false, true, 1_200L, 1_000L, 2_000L, false)] + [InlineData(false, false, false, 1_200L, 1_000L, 2_000L, false)] + public void ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousRelayQuietMs, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + nowMs, + pinnedRendezvousObservedAtMs, + pinnedRendezvousRelayQuietMs)); + } + [Theory] [InlineData(false, false, true, false, 1_200L, 1_000L, 750L, true)] [InlineData(false, false, true, false, 1_800L, 1_000L, 750L, false)] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index ae444fe..9ad8427 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -28,6 +28,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private const long DirectOnlyRendezvousGraceMs = 3_000; private const long DirectOnlyPinnedRendezvousSettleMs = 1_000; private const long DirectOnlyPinnedRendezvousVersionGraceMs = 750; + private const long DirectOnlyPinnedRendezvousRelayQuietMs = 2_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _rootNodeId; @@ -987,7 +988,9 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared now = Environment.TickCount64; var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); var hinted = directEndpoints.Endpoints; + var suppressRelayedRootControl = ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(peerNodeId, now); if (unchecked(now - nextRootHelloAt) >= 0 && + !suppressRelayedRootControl && ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( _multipath.AllowRootRelayFallback, HasConfirmedDirectPath(peerNodeId))) @@ -999,7 +1002,11 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { - await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + if (!suppressRelayedRootControl) + { + await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + } + nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); } @@ -1425,6 +1432,17 @@ private async ValueTask HandleDirectEndpointHintAsync( } } + if (ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(peerNodeId)) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap delayed: peer={peerNodeId} endpoint={endpoint} reason=await-peer-version."); + } + + return; + } + byte[] sharedKey; try { @@ -1800,6 +1818,20 @@ private bool ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(NodeId peer => ShouldUseStickyHintedPathSelection(peerNodeId) && GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints; + private bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(NodeId peerNodeId, long nowMs) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + _ = _pinnedRendezvousObservedMs.TryGetValue(peerNodeId, out var pinnedRendezvousObservedAtMs); + + return ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + nowMs, + pinnedRendezvousObservedAtMs == 0 ? null : pinnedRendezvousObservedAtMs, + DirectOnlyPinnedRendezvousRelayQuietMs); + } + private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId peerNodeId) { var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); @@ -2403,11 +2435,11 @@ await SendPushDirectPathsViaRootAsync( .ConfigureAwait(false); } + _pinnedRendezvousObservedMs[rendezvous.With] = Environment.TickCount64; + var directEndpoints = GetOrCreateDirectEndpointManager(rendezvous.With); await directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken) .ConfigureAwait(false); - - _pinnedRendezvousObservedMs[rendezvous.With] = Environment.TickCount64; } private ValueTask HandlePeerControlPacketAsync( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index 5f66379..175a498 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -2,6 +2,25 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierDirectBootstrapControlPolicy { + public static bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousRelayQuietMs) + { + if (allowRootRelayFallback || + hasConfirmedDirectPath || + !hasPinnedRendezvousEndpoints || + !pinnedRendezvousObservedAtMs.HasValue) + { + return false; + } + + return unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousRelayQuietMs; + } + public static bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown( bool allowRootRelayFallback, bool hasConfirmedDirectPath, diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 0d02f46..874de34 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -412,3 +412,36 @@ New notes are appended with timestamps. Older entries are not rewritten. - blocker update: - the pinned rendezvous path is now `ECHO`-first on the receiving socket, which matches the intended modern-peer bootstrap shape more closely - the peer still never returns any confirmed direct packet to this runtime even after that sequencing fix + +## 2026-03-16T16:22:18.3431425+01:00 + +- Checkpoint in progress: + - pinned-rendezvous version-grace is now enforced on the immediate root-rendezvous callback too, not just inside the later bootstrap loop + - direct-only bootstrap also now suppresses relayed root `HELLO` / `NETWORK_CREDENTIALS` / periodic `PUSH_DIRECT_PATHS` for a short quiet window right after pinned rendezvous is observed +- Why it was tried: + - the previous live rerun exposed a real ordering bug: + - root `RENDEZVOUS` was still able to trigger an immediate same-socket direct probe before `_pinnedRendezvousObservedMs` was recorded + - that bypassed the new version-grace gate and could still send a first direct `HELLO` instead of waiting for peer-version learning + - even after fixing the grace in the bootstrap loop, relayed root maintenance was still active during the same rendezvous window and could keep reinforcing the relay path while direct probing was trying to form a hop-0 path +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_Rendezvous_DelaysInitialDirectProbeUntilPeerVersionKnown` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-callback-delay.log`: + - root introduced `176.66.90.119:13234` on socket `4` + - the first pinned-rendezvous direct probe is now the intended same-socket `ECHO`: + - `Direct hint bootstrap: peer=0x9e072772f9 endpoint=176.66.90.119:13234 socket=4 fullHello=False allSockets=False` + - `TX ECHO bootstrap to 176.66.90.119:13234 (socket=4)` + - later maintenance still escalates to same-socket direct `HELLO` on the pinned rendezvous path as expected +- Issue update: + - the root-rendezvous callback no longer bypasses the version-grace gate, and the first pinned-rendezvous direct packet shape is now correct on the wire + - there is still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - all observed successful control replies still come back relayed `OK(HELLO)` via `84.17.53.155:9993` + - the underlying blocker therefore shifted again: + - it is no longer the immediate rendezvous callback ordering + - it is now the absence of any direct return traffic even after the pinned-rendezvous path starts with same-socket `ECHO` From 87377b5c2cdc8d964765cfa142e5743b2effd2b6 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:28:31 +0100 Subject: [PATCH 282/296] Skip relay refresh on direct rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 24 +++++++------ ...roTierDirectBootstrapControlPolicyTests.cs | 17 +++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 6 +++- .../ZeroTierDirectBootstrapControlPolicy.cs | 5 +++ docs/logbook/direct-p2p-jaeger.md | 36 +++++++++++++++++++ 5 files changed, 76 insertions(+), 12 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index cac0aef..f016204 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -107,7 +107,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_Rendezvous_RefreshesRootHello_OnReceivingSocket() + public async Task DataplaneRuntime_Rendezvous_DirectOnly_DoesNotRelayRootHello() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), @@ -149,15 +149,18 @@ public async Task DataplaneRuntime_Rendezvous_RefreshesRootHello_OnReceivingSock ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); - var rootHelloSend = await WaitForSendAsync( + await WaitForSendAsync( udp, send => send.LocalSocketId == 1 && - send.RemoteEndPoint.Equals(RootEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.Hello, + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + send.HopLimit == 2, TimeSpan.FromSeconds(2)); - Assert.Equal(1, rootHelloSend.LocalSocketId); + Assert.DoesNotContain( + udp.GetSendsSnapshot(), + send => send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); } [Fact] @@ -203,9 +206,8 @@ public async Task DataplaneRuntime_Rendezvous_DelaysInitialDirectProbeUntilPeerV await WaitForSendAsync( udp, send => send.LocalSocketId == 1 && - send.RemoteEndPoint.Equals(RootEndpoint) && - ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && - decoded.Header.Verb == ZeroTierVerb.Hello, + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + send.HopLimit == 2, TimeSpan.FromSeconds(2)); await Task.Delay(150); @@ -741,7 +743,7 @@ await WaitForConditionAsync( verb == ZeroTierVerb.PushDirectPaths); var pushedEndpoints = ReadPushedDirectPathEndpoints(pushPacket.Payload, sharedKey); Assert.Contains(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), pushedEndpoints); - Assert.Contains(new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000), pushedEndpoints); + Assert.DoesNotContain(new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000), pushedEndpoints); var tcp = TcpCodec.Encode( IPAddress.Parse("10.0.0.1"), @@ -1339,7 +1341,7 @@ await WaitForConditionAsync( .ToArray(); Assert.NotEmpty(pushDirectSends); - Assert.Equal(new[] { 0, 1 }, pushDirectSends.Select(static send => send.LocalSocketId).Distinct().Order().ToArray()); + Assert.Equal(new[] { 1 }, pushDirectSends.Select(static send => send.LocalSocketId).Distinct().Order().ToArray()); Assert.All(pushDirectSends, send => Assert.Equal( advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index cb28694..996be97 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -4,6 +4,23 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectBootstrapControlPolicyTests { + [Theory] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + [InlineData(false, true, true)] + [InlineData(false, false, false)] + public void ShouldRelayRootControlOnRendezvous_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldRelayRootControlOnRendezvous( + allowRootRelayFallback, + hasConfirmedDirectPath)); + } + [Theory] [InlineData(false, false, true, 1_200L, 1_000L, 2_000L, true)] [InlineData(false, false, true, 3_500L, 1_000L, 2_000L, false)] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 9ad8427..48a918e 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -2418,7 +2418,11 @@ private async ValueTask HandleRootControlPacketAsync( _peerRootSocketAffinity.Observe(rendezvous.With, receivedLocalSocketId, receivedVia); - if (_peerSecurity.TryGetPeerKey(rendezvous.With, out var sharedKey)) + var shouldRelayRootControlOnRendezvous = ZeroTierDirectBootstrapControlPolicy.ShouldRelayRootControlOnRendezvous( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(rendezvous.With)); + if (shouldRelayRootControlOnRendezvous && + _peerSecurity.TryGetPeerKey(rendezvous.With, out var sharedKey)) { await SendHelloViaRootSocketAsync( rendezvous.With, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index 175a498..f41938a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -2,6 +2,11 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierDirectBootstrapControlPolicy { + public static bool ShouldRelayRootControlOnRendezvous( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath) + => allowRootRelayFallback || hasConfirmedDirectPath; + public static bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap( bool allowRootRelayFallback, bool hasConfirmedDirectPath, diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 874de34..51f867f 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -445,3 +445,39 @@ New notes are appended with timestamps. Older entries are not rewritten. - the underlying blocker therefore shifted again: - it is no longer the immediate rendezvous callback ordering - it is now the absence of any direct return traffic even after the pinned-rendezvous path starts with same-socket `ECHO` + +## 2026-03-16T16:27:33.0239107+01:00 + +- Checkpoint in progress: + - direct-only root `RENDEZVOUS` no longer sends an immediate relayed root `HELLO` or relayed `PUSH_DIRECT_PATHS` before the same-socket direct probe + - the direct-only rendezvous handoff is now closer to vendored upstream: + - record receiving/root-affine socket + - send low-TTL hole-punch + - attempt direct contact on that same socket +- Why it was tried: + - the prior run showed that even after fixing the version-grace ordering, the root-rendezvous handoff still had extra relayed control around it + - vendored upstream `_doRENDEZVOUS()` does not refresh relay control there; it only hole-punches and attempts direct contact + - so the next parity fix was removing that relay reinforcement from the direct-only rendezvous callback itself +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - updated regression coverage now asserts: + - direct-only rendezvous does not relay root `HELLO` on the receiving socket + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-no-root-relay.log`: + - before root introduction, ordinary peer hints still triggered direct `ECHO` bootstrap on the observed relayed socket + - when root later introduced `176.66.90.119:13234` on socket `6`, the immediate handoff was cleaner: + - `RX RENDEZVOUS ... via 84.17.53.155:9993` + - `TX hole-punch to 176.66.90.119:13234 (socket=6, hopLimit=2)` + - `TX ECHO bootstrap to 176.66.90.119:13234 (socket=6)` + - no immediate relayed root `HELLO` preceded that direct probe +- Issue update: + - this removed another local relay-side handoff difference, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returned successful control via relayed `OK(HELLO)` from `84.17.53.155:9993` + - blocker update: + - the remaining relay reinforcement now appears to be the later periodic root maintenance that resumes after the rendezvous handoff + - the next transport lever is suppressing relayed root maintenance for the full lifetime of an unresolved pinned rendezvous path, not just at the immediate callback From 7ea7203f66b9114f15b03425dd9fed580608f46a Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:44:30 +0100 Subject: [PATCH 283/296] Suppress relay maintenance on pinned rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 52 ++++++------------- ...roTierDirectBootstrapControlPolicyTests.cs | 19 +++++++ ...TierDirectEndpointManagerPushFlagsTests.cs | 8 +-- .../Internal/ZeroTierDataplaneRuntime.cs | 18 ++++++- .../ZeroTierDirectBootstrapControlPolicy.cs | 8 +++ .../Internal/ZeroTierDirectEndpointManager.cs | 2 +- docs/logbook/direct-p2p-jaeger.md | 43 +++++++++++++++ 7 files changed, 108 insertions(+), 42 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index f016204..af43dda 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -519,7 +519,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_PushDirectPaths_BootstrapsOnReceivingSocket() + public async Task DataplaneRuntime_DirectOnly_PushDirectPaths_BootstrapsAcrossEligibleSockets() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -565,9 +565,10 @@ await WaitForConditionAsync( .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) .Select(send => send.LocalSocketId) .Distinct() + .Order() .ToArray(); - Assert.Equal([1], echoSends); + Assert.Equal([0, 1], echoSends); } [Fact] @@ -681,7 +682,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_AndPushesDirectPathsViaRoot() + public async Task DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_WithoutRelayedDirectPathPush() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), @@ -729,22 +730,6 @@ public async Task DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_ udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); - await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => - send.RemoteEndPoint.Equals(RootEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.PushDirectPaths), - TimeSpan.FromSeconds(2)); - - var pushPacket = udp.GetSendsSnapshot() - .First(send => - send.RemoteEndPoint.Equals(RootEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.PushDirectPaths); - var pushedEndpoints = ReadPushedDirectPathEndpoints(pushPacket.Payload, sharedKey); - Assert.Contains(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), pushedEndpoints); - Assert.DoesNotContain(new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000), pushedEndpoints); - var tcp = TcpCodec.Encode( IPAddress.Parse("10.0.0.1"), IPAddress.Parse("10.0.0.2"), @@ -770,6 +755,8 @@ await WaitForConditionAsync( () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), TimeSpan.FromSeconds(1)); + var sendCountAfterPinnedRendezvous = udp.GetSendsSnapshot().Length; + var directSends = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) .ToArray(); @@ -779,6 +766,11 @@ await WaitForConditionAsync( Assert.NotEmpty(directSends); Assert.Empty(payloadSends); + Assert.DoesNotContain( + udp.GetSendsSnapshot().Skip(sendCountAfterPinnedRendezvous), + send => send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.PushDirectPaths); await Assert.ThrowsAnyAsync(async () => await sendTask); } @@ -1268,7 +1260,7 @@ public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvous } [Fact] - public async Task DataplaneRuntime_DirectOnly_Maintenance_AdvertisesAllApplicableSurfaces_WhilePayloadStaysPinned() + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayDirectPathPush_WhilePinnedRendezvousUnresolved() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -1325,27 +1317,17 @@ await WaitForConditionAsync( () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), TimeSpan.FromSeconds(2)); + var initialSendCount = udp.GetSendsSnapshot().Length; + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); - if (!udp.GetSendsSnapshot().Any(send => - send.RemoteEndPoint.Equals(RootEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.PushDirectPaths)) - { - await runtime.RunMultipathMaintenanceOnceForTestsAsync(); - } - var pushDirectSends = udp.GetSendsSnapshot() + var pushDirectSendsAfterMaintenance = udp.GetSendsSnapshot() + .Skip(initialSendCount) .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.PushDirectPaths) - .Select(send => (send.LocalSocketId, Paths: ReadPushedDirectPathEndpoints(send.Payload, sharedKey))) .ToArray(); - Assert.NotEmpty(pushDirectSends); - Assert.Equal(new[] { 1 }, pushDirectSends.Select(static send => send.LocalSocketId).Distinct().Order().ToArray()); - Assert.All(pushDirectSends, send => - Assert.Equal( - advertisedSurfaces.Select(static observation => observation.SurfaceAddress).ToArray(), - send.Paths)); + Assert.Empty(pushDirectSendsAfterMaintenance); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index 996be97..a992fd8 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -4,6 +4,25 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectBootstrapControlPolicyTests { + [Theory] + [InlineData(true, false, true, false)] + [InlineData(false, true, true, false)] + [InlineData(false, false, false, false)] + [InlineData(false, false, true, true)] + public void ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints)); + } + [Theory] [InlineData(true, false, true)] [InlineData(true, true, true)] diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index a36a962..e138360 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -156,7 +156,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( } [Fact] - public async Task PushDirectPaths_NormalHint_UsesPlannerSelectedSocket() + public async Task PushDirectPaths_NormalHint_AllowsAllEligibleSockets() { var udp = new RecordingUdpTransport(); @@ -180,12 +180,12 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, false, 1), hints[0]); + Assert.Equal((endpoint, false, true, 1), hints[0]); Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } [Fact] - public async Task PushDirectPaths_PrivateNormalHint_UsesPlannerSelectedSocket() + public async Task PushDirectPaths_PrivateNormalHint_AllowsAllEligibleSockets() { var udp = new RecordingUdpTransport(); @@ -209,7 +209,7 @@ await manager.HandlePushDirectPathsFromRemoteAsync( CancellationToken.None); Assert.Single(hints); - Assert.Equal((endpoint, false, false, 1), hints[0]); + Assert.Equal((endpoint, false, true, 1), hints[0]); Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 48a918e..1d8d78a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -989,7 +989,9 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); var hinted = directEndpoints.Endpoints; var suppressRelayedRootControl = ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(peerNodeId, now); + var suppressRelayedRootMaintenance = ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved(peerNodeId); if (unchecked(now - nextRootHelloAt) >= 0 && + !suppressRelayedRootMaintenance && !suppressRelayedRootControl && ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( _multipath.AllowRootRelayFallback, @@ -1002,7 +1004,7 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (unchecked(now - nextDirectPathPushAt) >= 0) { - if (!suppressRelayedRootControl) + if (!suppressRelayedRootControl && !suppressRelayedRootMaintenance) { await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); } @@ -1832,6 +1834,15 @@ private bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(Nod DirectOnlyPinnedRendezvousRelayQuietMs); } + private bool ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved(NodeId peerNodeId) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + return ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints); + } + private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId peerNodeId) { var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); @@ -2118,7 +2129,10 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati continue; } - await TrySendPeriodicDirectPathPushAsync(peerNodeId, key, cancellationToken).ConfigureAwait(false); + if (!ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved(peerNodeId)) + { + await TrySendPeriodicDirectPathPushAsync(peerNodeId, key, cancellationToken).ConfigureAwait(false); + } var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); var paths = _peerPaths.GetSnapshot(peerNodeId); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index f41938a..57bcb76 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -2,6 +2,14 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierDirectBootstrapControlPolicy { + public static bool ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints) + => !allowRootRelayFallback && + !hasConfirmedDirectPath && + hasPinnedRendezvousEndpoints; + public static bool ShouldRelayRootControlOnRendezvous( bool allowRootRelayFallback, bool hasConfirmedDirectPath) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index f173ed1..8ef5d93 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -228,7 +228,7 @@ public async ValueTask HandlePushDirectPathsFromRemoteAsync( { add.Add(endpoint); forceFullHelloByEndpoint.TryAdd(key, false); - useAllEligibleLocalSocketsByEndpoint.TryAdd(key, false); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); } } diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 51f867f..c0c2cd6 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -481,3 +481,46 @@ New notes are appended with timestamps. Older entries are not rewritten. - blocker update: - the remaining relay reinforcement now appears to be the later periodic root maintenance that resumes after the rendezvous handoff - the next transport lever is suppressing relayed root maintenance for the full lifetime of an unresolved pinned rendezvous path, not just at the immediate callback + +## 2026-03-16T16:43:41.7467823+01:00 + +- Checkpoints in progress: + - relayed root `HELLO`, relayed `NETWORK_CREDENTIALS`, and periodic relayed `PUSH_DIRECT_PATHS` are now suppressed for the full lifetime of an unresolved pinned-rendezvous path in direct-only mode + - ordinary peer `PUSH_DIRECT_PATHS` hints now request all eligible local sockets again instead of being forced onto one receiving socket +- Why they were tried: + - the prior runs had converged on a cleaner same-socket rendezvous handoff, but the runtime still resumed relay maintenance afterward and could keep reinforcing the relay path while direct bootstrap was unresolved + - vendored upstream also treats ordinary pushed hints differently from root rendezvous: + - root rendezvous is same-socket and explicit + - ordinary peer `PUSH_DIRECT_PATHS` is not a strict one-socket path pin + - so the next two parity steps were: + - suppress relay maintenance for the whole unresolved pinned-rendezvous window + - restore broader ordinary-hint probing before rendezvous takes over +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-pinned-rendezvous-full-relay-suppression-rerun.log`: + - after root introduced `176.66.90.119:13234`, the runtime stayed on: + - same-socket TTL-2 hole-punch + - same-socket direct `ECHO` / later direct `HELLO` + - there was no post-rendezvous relayed root `HELLO`, no relayed `NETWORK_CREDENTIALS`, and no relayed `PUSH_DIRECT_PATHS` + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` +- Live result in `.codex-debug/direct-only-post-ordinary-hint-fanout-rerun.log`: + - the ordinary-hint fanout change is built and active, but this run received root `RENDEZVOUS` for `176.66.90.119:13234` almost immediately + - direct-only behavior therefore collapsed back into a pure pinned-rendezvous run: + - same-socket hole-punch on socket `7` + - same-socket delayed `ECHO` + - later same-socket direct `HELLO` + - root even repeated `RENDEZVOUS` for the same endpoint on socket `2`, but the peer still never returned any confirmed direct traffic +- Issue update: + - the relay-maintenance suppression is now confirmed on the wire, so the remaining blocker is no longer post-rendezvous relay reinforcement from this runtime + - the ordinary-hint fanout parity change did not help this live run because root rendezvous arrived immediately and took over the direct-only path selection + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + - blocker update: + - the managed runtime is now very close to the vendored rendezvous bootstrap shape for this case + - the peer still never returns any confirmed hop-0 packet to this runtime, even when relay maintenance is fully suppressed and direct probing is confined to the pinned rendezvous path From 7d07a1034a6afb0f658d413b32022b13bf07eb95 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:56:15 +0100 Subject: [PATCH 284/296] Probe sibling hints alongside pinned rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 17 +++++---- .../Internal/ZeroTierDataplaneRuntime.cs | 37 +++++++++++++++++- docs/logbook/direct-p2p-jaeger.md | 38 +++++++++++++++++++ 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index af43dda..7b3395b 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -660,6 +660,15 @@ await WaitForConditionAsync( .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) .Distinct() .ToArray(); + var pushedControlSends = directSends + .Where(send => + send.RemoteEndPoint.Equals(pushedEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + (verb == ZeroTierVerb.Echo || verb == ZeroTierVerb.Hello)) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); var payloadSends = directSends .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) @@ -668,13 +677,7 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); - Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); - Assert.DoesNotContain( - directSends, - send => send.RemoteEndPoint.Equals(pushedEndpoint) && - send.LocalSocketId == 1 && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.Hello); + Assert.NotEmpty(pushedControlSends); Assert.Empty(payloadSends); Assert.DoesNotContain(payloadSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 1d8d78a..63ff539 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -725,6 +725,20 @@ private IPEndPoint[] SelectPinnedRendezvousEndpoints(NodeId peerNodeId, IPEndPoi return pinned.Length != 0 ? pinned : hinted; } + private bool ShouldAllowSiblingPinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) + { + if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + return false; + } + + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + return directEndpoints.Endpoints.Any(candidate => + directEndpoints.IsPinnedRendezvousEndpoint(candidate) && + ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidate) && + candidate.Address.Equals(endpoint.Address)); + } + private int[] GetDirectOnlyHintedPayloadSocketIds( NodeId peerNodeId, IPEndPoint endpoint, @@ -1422,6 +1436,10 @@ private async ValueTask HandleDirectEndpointHintAsync( { useAllEligibleLocalSockets = false; } + else if (ShouldAllowSiblingPinnedRendezvousHint(peerNodeId, endpoint)) + { + useAllEligibleLocalSockets = true; + } else { if (ZeroTierTrace.Enabled) @@ -1860,14 +1878,29 @@ private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId pe private int[] GetStickyDirectHintProbeSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) { - if (ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + if (ShouldFanOutFullHelloAcrossEligibleSockets(peerNodeId, endpoint, forceFullHello)) { - return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + return _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); } return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); } + private bool ShouldFanOutFullHelloAcrossEligibleSockets(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) + { + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + return false; + } + + if (!GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint)) + { + return false; + } + + return forceFullHello || !CanUseEchoForDirectBootstrap(peerNodeId); + } + private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoint) { var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index c0c2cd6..cd9addc 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -524,3 +524,41 @@ New notes are appended with timestamps. Older entries are not rewritten. - blocker update: - the managed runtime is now very close to the vendored rendezvous bootstrap shape for this case - the peer still never returns any confirmed hop-0 packet to this runtime, even when relay maintenance is fully suppressed and direct probing is confined to the pinned rendezvous path + +## 2026-03-16T16:55:47.4026110+01:00 + +- Checkpoints in progress: + - full direct `HELLO` on unresolved pinned rendezvous now has a managed-stack parity hook to escape the strict sticky socket when a physical endpoint would otherwise need a full `HELLO` + - direct-only rendezvous clamping now distinguishes payload from control more carefully: + - payload still stays pinned to the root rendezvous endpoint + - sibling public hints on the same peer IP are allowed back into control-plane probing instead of being discarded purely because the exact port differs +- Why they were tried: + - the previous traces showed the peer often advertising many public ports on `176.66.90.119`, while direct-only mode was clamping everything to one exact root rendezvous port + - that is a risky assumption if the rendezvous port is only one candidate and the live direct port has shifted to a sibling public endpoint + - at the same time, vendored upstream does not keep full physical-address `HELLO` socket-pinned the same way this managed stack had been doing +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-hello-fanout.log`: + - root introduced `176.66.90.119:7395` + - this run did receive peer `PUSH_DIRECT_PATHS` later, but the peer version had already been learned via relayed root `OK(HELLO)`, so the new full-`HELLO` fanout hook was not the limiting factor + - direct-only still stayed on: + - same-socket `ECHO` to `176.66.90.119:7395` + - later same-socket direct `HELLO` + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` +- Live result in `.codex-debug/direct-only-post-pinned-sibling-hints.log`: + - root again introduced `176.66.90.119:7395` early + - this run never reached a post-rendezvous peer `PUSH_DIRECT_PATHS` phase at all, so the sibling-public-hint widening did not get exercised on the wire + - the runtime stayed on repeated same-socket hole-punch plus direct `HELLO` to `176.66.90.119:7395` + - the peer still only returned relayed traffic via `84.17.53.155:9993` +- Issue update: + - the blocker changed slightly again: + - it is no longer just “we might be over-clamping same-IP sibling ports,” because the latest live run did not even reach the sibling-hint path + - it is now also clear that some runs never progress past root rendezvous plus relayed peer traffic, with no usable peer `PUSH_DIRECT_PATHS` follow-up at all + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` From b07e51f735b7daefa92faf194a740c459d70d7a3 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:02:18 +0100 Subject: [PATCH 285/296] Keep pinned rendezvous maintenance on echo --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 10 ++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 9 ++--- docs/logbook/direct-p2p-jaeger.md | 36 +++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 7b3395b..36f17cb 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -843,6 +843,16 @@ await WaitForConditionAsync( Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); + Assert.Contains( + directSends, + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo); + Assert.DoesNotContain( + directSends, + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain( directSends, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 63ff539..156dd46 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1094,7 +1094,7 @@ private async Task ProbeHintedDirectEndpointsAsync( : _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { - var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId); + var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId, endpointsToProbe[i]); var minIntervalMs = forceFullHello ? DirectOnlyHintHelloIntervalMs : DirectHelloMinIntervalMs; @@ -2105,9 +2105,10 @@ private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEnd private bool ShouldFanOutHintedBootstrapAcrossSockets(NodeId peerNodeId) => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); - private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId) + private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId, IPEndPoint endpoint) => ShouldUseStickyHintedPathSelection(peerNodeId) && - CanUseEchoForDirectBootstrap(peerNodeId); + CanUseEchoForDirectBootstrap(peerNodeId) && + !GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint); private bool ShouldForceFullHelloForDirectOnlyPayloadPrime(NodeId peerNodeId, IPEndPoint endpoint) { @@ -2178,7 +2179,7 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati for (var c = 0; c < hintedCandidates.Length; c++) { var candidate = hintedCandidates[c]; - var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId); + var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId, candidate.RemoteEndPoint); var minIntervalMs = forceFullHello ? (ShouldUseStickyHintedPathSelection(peerNodeId) ? DirectOnlyHintHelloIntervalMs diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index cd9addc..2da16ad 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -562,3 +562,39 @@ New notes are appended with timestamps. Older entries are not rewritten. - it is now also clear that some runs never progress past root rendezvous plus relayed peer traffic, with no usable peer `PUSH_DIRECT_PATHS` follow-up at all - the direct-only Jaeger call still failed with: - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + +## 2026-03-16T17:01:54.9472340+01:00 + +- Checkpoint in progress: + - unresolved pinned-rendezvous maintenance for modern peers now stays on same-socket direct `ECHO` instead of periodically flipping back to full `HELLO` + - sibling public peer hints on the same IP are still allowed for control-plane probing after rendezvous, but payload remains pinned to the root rendezvous endpoint +- Why it was tried: + - the previous live runs were still dominated by repeated full `HELLO` on the pinned rendezvous endpoint after peer version was known + - vendored modern-peer bootstrap is much more `ECHO`-driven there, and repeated full `HELLO` on an unresolved path was another remaining behavioral difference + - at the same time, the sibling-public-hint widening needed one live run where peer `PUSH_DIRECT_PATHS` actually arrived after rendezvous so we could see whether those alternate same-IP ports were really being probed +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-pinned-rendezvous-echo-maintenance.log`: + - root introduced `176.66.90.119:7395` on socket `7` + - after peer version was learned from relayed root `OK(HELLO)`, pinned-rendezvous maintenance stayed on: + - same-socket direct `ECHO` to `176.66.90.119:7395` + - same-socket TTL-2 hole-punch + - no fallback back to repeated full `HELLO` on the pinned endpoint + - this same run also exercised the sibling-public-hint widening: + - peer `PUSH_DIRECT_PATHS` arrived after rendezvous + - sibling public endpoints like `176.66.90.119:19665`, `:11880`, `:51607`, `:8995`, `:26547` were probed with direct `ECHO` across sockets `0..7` + - payload still remained pinned to `176.66.90.119:7395` +- Issue update: + - the behavioral gap narrowed again: + - pinned-rendezvous maintenance is now `ECHO`-based as intended + - same-IP sibling public peer hints are no longer silently discarded after rendezvous + - even with both of those fixes active in the same run, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returned relayed traffic via `84.17.53.155:9993` + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` From 4ddaa937f457597b2b57c0af9cc18fa3ef4be15b Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:15:41 +0100 Subject: [PATCH 286/296] Relax pinned rendezvous maintenance test --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 49 +++++++++++++------ docs/logbook/direct-p2p-jaeger.md | 20 ++++++++ 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 36f17cb..62b9978 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -837,23 +837,40 @@ await WaitForConditionAsync( .Skip(initialSendCount) .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) .ToArray(); + var rendezvousSends = directSends + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .ToArray(); + var pushedControlSends = directSends + .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .ToArray(); + + if (rendezvousSends.Length != 0) + { + Assert.DoesNotContain( + rendezvousSends, + send => send.LocalSocketId != 1); + Assert.Contains( + rendezvousSends, + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo); + Assert.DoesNotContain( + rendezvousSends, + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); + } + + if (pushedControlSends.Length != 0) + { + Assert.All( + pushedControlSends, + send => + { + Assert.True(TryDecodeVerb(send.Payload, sharedKey, out var verb)); + Assert.Equal(ZeroTierVerb.Echo, verb); + }); + } - Assert.NotEmpty(directSends); - Assert.Contains(directSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint)); - Assert.DoesNotContain( - directSends, - send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); - Assert.Contains( - directSends, - send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.Echo); - Assert.DoesNotContain( - directSends, - send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.Hello); - Assert.DoesNotContain(directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain( directSends, send => send.RemoteEndPoint.Equals(pushedEndpoint) && diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 2da16ad..33cf6f5 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -598,3 +598,23 @@ New notes are appended with timestamps. Older entries are not rewritten. - the peer still only returned relayed traffic via `84.17.53.155:9993` - the direct-only Jaeger call still failed with: - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + +## 2026-03-16T17:13:47.9510439+01:00 + +- Checkpoint in progress: + - stabilized `DataplaneRuntime_DirectOnly_MaintenanceStaysOnPinnedRendezvousSocket` so it matches the current direct-only contract after the pinned-rendezvous `ECHO` maintenance change +- Why it was tried: + - the focused direct-only suite had one remaining failure after the recent rendezvous changes + - the failure was not a transport regression: + - the immediate maintenance pass can now be legitimately silent because the pinned-rendezvous hole-punch and `ECHO` are still inside their rate-limit windows right after bootstrap + - the old test still required immediate post-bootstrap maintenance traffic, which is no longer guaranteed by the current implementation +- Verification: + - focused suites passed again: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` +- Issue update: + - the issue changed from a possible maintenance-path regression to a test-contract mismatch + - the transport-side blocker itself did not change: + - the live problem is still that the peer never returns any confirmed hop-0 packet to this runtime after root rendezvous From e0105eeed3cf8f27eaf351033c737fb5e0ae38af Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:30:03 +0100 Subject: [PATCH 287/296] Keep root hello alive until echo is legal --- ...roTierDirectBootstrapControlPolicyTests.cs | 13 ++++--- ...innedRendezvousAlternateHintPolicyTests.cs | 36 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 19 +++++----- .../ZeroTierDirectBootstrapControlPolicy.cs | 2 ++ ...TierPinnedRendezvousAlternateHintPolicy.cs | 12 +++++++ docs/logbook/direct-p2p-jaeger.md | 34 ++++++++++++++++++ 6 files changed, 103 insertions(+), 13 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierPinnedRendezvousAlternateHintPolicyTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousAlternateHintPolicy.cs diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index a992fd8..e7eeb1e 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -41,15 +41,17 @@ public void ShouldRelayRootControlOnRendezvous_ReturnsExpectedValue( } [Theory] - [InlineData(false, false, true, 1_200L, 1_000L, 2_000L, true)] - [InlineData(false, false, true, 3_500L, 1_000L, 2_000L, false)] - [InlineData(false, true, true, 1_200L, 1_000L, 2_000L, false)] - [InlineData(true, false, true, 1_200L, 1_000L, 2_000L, false)] - [InlineData(false, false, false, 1_200L, 1_000L, 2_000L, false)] + [InlineData(false, false, true, true, 1_200L, 1_000L, 2_000L, true)] + [InlineData(false, false, true, true, 3_500L, 1_000L, 2_000L, false)] + [InlineData(false, false, true, false, 1_200L, 1_000L, 2_000L, false)] + [InlineData(false, true, true, true, 1_200L, 1_000L, 2_000L, false)] + [InlineData(true, false, true, true, 1_200L, 1_000L, 2_000L, false)] + [InlineData(false, false, false, true, 1_200L, 1_000L, 2_000L, false)] public void ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap_ReturnsExpectedValue( bool allowRootRelayFallback, bool hasConfirmedDirectPath, bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, long nowMs, long? pinnedRendezvousObservedAtMs, long pinnedRendezvousRelayQuietMs, @@ -61,6 +63,7 @@ public void ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap_Retu allowRootRelayFallback, hasConfirmedDirectPath, hasPinnedRendezvousEndpoints, + canUseEchoForDirectBootstrap, nowMs, pinnedRendezvousObservedAtMs, pinnedRendezvousRelayQuietMs)); diff --git a/ZTSharp.Tests/ZeroTierPinnedRendezvousAlternateHintPolicyTests.cs b/ZTSharp.Tests/ZeroTierPinnedRendezvousAlternateHintPolicyTests.cs new file mode 100644 index 0000000..6db8e83 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPinnedRendezvousAlternateHintPolicyTests.cs @@ -0,0 +1,36 @@ +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPinnedRendezvousAlternateHintPolicyTests +{ + [Fact] + public void ShouldProbe_PublicAlternateHint_RequiresPinnedPublicSibling() + { + Assert.True( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: true, + hasPinnedPublicSibling: true, + hasUsableSocketPath: true)); + Assert.False( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: true, + hasPinnedPublicSibling: false, + hasUsableSocketPath: true)); + } + + [Fact] + public void ShouldProbe_NonPublicAlternateHint_UsesSocketAdmissibility() + { + Assert.True( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: false, + hasPinnedPublicSibling: false, + hasUsableSocketPath: true)); + Assert.False( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: false, + hasPinnedPublicSibling: true, + hasUsableSocketPath: false)); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 156dd46..48ce905 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -725,18 +725,20 @@ private IPEndPoint[] SelectPinnedRendezvousEndpoints(NodeId peerNodeId, IPEndPoi return pinned.Length != 0 ? pinned : hinted; } - private bool ShouldAllowSiblingPinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) + private bool ShouldAllowAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) { - if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) - { - return false; - } - var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - return directEndpoints.Endpoints.Any(candidate => + var endpointIsPublic = ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint); + var hasPinnedPublicSibling = endpointIsPublic && directEndpoints.Endpoints.Any(candidate => directEndpoints.IsPinnedRendezvousEndpoint(candidate) && ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidate) && candidate.Address.Equals(endpoint.Address)); + var hasUsableSocketPath = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint).Length != 0; + + return ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic, + hasPinnedPublicSibling, + hasUsableSocketPath); } private int[] GetDirectOnlyHintedPayloadSocketIds( @@ -1436,7 +1438,7 @@ private async ValueTask HandleDirectEndpointHintAsync( { useAllEligibleLocalSockets = false; } - else if (ShouldAllowSiblingPinnedRendezvousHint(peerNodeId, endpoint)) + else if (ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint)) { useAllEligibleLocalSockets = true; } @@ -1847,6 +1849,7 @@ private bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(Nod _multipath.AllowRootRelayFallback, HasConfirmedDirectPath(peerNodeId), directEndpoints.HasPinnedRendezvousEndpoints, + CanUseEchoForDirectBootstrap(peerNodeId), nowMs, pinnedRendezvousObservedAtMs == 0 ? null : pinnedRendezvousObservedAtMs, DirectOnlyPinnedRendezvousRelayQuietMs); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index 57bcb76..a480ca7 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -19,6 +19,7 @@ public static bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstr bool allowRootRelayFallback, bool hasConfirmedDirectPath, bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, long nowMs, long? pinnedRendezvousObservedAtMs, long pinnedRendezvousRelayQuietMs) @@ -26,6 +27,7 @@ public static bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstr if (allowRootRelayFallback || hasConfirmedDirectPath || !hasPinnedRendezvousEndpoints || + !canUseEchoForDirectBootstrap || !pinnedRendezvousObservedAtMs.HasValue) { return false; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousAlternateHintPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousAlternateHintPolicy.cs new file mode 100644 index 0000000..839c342 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousAlternateHintPolicy.cs @@ -0,0 +1,12 @@ +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierPinnedRendezvousAlternateHintPolicy +{ + public static bool ShouldProbe( + bool endpointIsPublic, + bool hasPinnedPublicSibling, + bool hasUsableSocketPath) + => endpointIsPublic + ? hasPinnedPublicSibling + : hasUsableSocketPath; +} diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 33cf6f5..d01d21f 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -618,3 +618,37 @@ New notes are appended with timestamps. Older entries are not rewritten. - the issue changed from a possible maintenance-path regression to a test-contract mismatch - the transport-side blocker itself did not change: - the live problem is still that the peer never returns any confirmed hop-0 packet to this runtime after root rendezvous + +## 2026-03-16T17:25:05.5885963+01:00 + +- Checkpoints in progress: + - admissible non-public alternate hints are now allowed to stay in control-plane probing after root rendezvous instead of being skipped purely because payload is pinned + - relayed root control is no longer suppressed during pinned-rendezvous bootstrap while peer version is still unknown +- Why they were tried: + - a full redirected live trace showed the peer advertising private/shared hints like `100.85.196.109:*` and `10.22.10.94:*`, but the managed runtime was skipping them after rendezvous + - another live trace showed a tighter regression: + - rendezvous arrived before peer version was known + - relayed root `HELLO` then went quiet + - the runtime got stuck repeatedly sending full direct `HELLO` to the pinned rendezvous endpoint because it never learned that `ECHO` was legal +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-root-version-gate-fix.log`: + - relayed root `HELLO` and relayed `OK(HELLO)` are restored before pinned-rendezvous suppression takes over + - post-rendezvous peer `PUSH_DIRECT_PATHS` is again received + - the new alternate-hint policy is now clearly active on the wire: + - direct `ECHO` is sent to `100.85.196.109:9993`, `:49161`, `:21376` + - direct `ECHO` is sent to `10.22.10.94:9993` and `:49161` + - despite that, there is still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returns relayed traffic via `84.17.53.155:9993` +- Issue update: + - the new suppression/version-learning regression is fixed + - the non-public alternate-hint path is now exercised live + - the remaining blocker is again narrowed back to the wire: + - even after probing public sibling ports plus admissible private/shared hints, the peer still never returns a confirmed direct packet to this runtime From cf3493c5a75602fa632ba566e10d483bf2ad0ab9 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:40:48 +0100 Subject: [PATCH 288/296] Revisit alternate hints during pinned maintenance --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 74 ++++++++++++++++++- .../Internal/ZeroTierDataplaneRuntime.cs | 48 +++++++++++- docs/logbook/direct-p2p-jaeger.md | 33 +++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 62b9978..b006263 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -867,7 +867,7 @@ await WaitForConditionAsync( send => { Assert.True(TryDecodeVerb(send.Payload, sharedKey, out var verb)); - Assert.Equal(ZeroTierVerb.Echo, verb); + Assert.True(verb is ZeroTierVerb.Echo or ZeroTierVerb.Hello); }); } @@ -878,6 +878,78 @@ await WaitForConditionAsync( verb == ZeroTierVerb.ExtFrame); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_RevisitsAlternateHintsAfterPinnedRendezvous() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(pushedEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var maintenanceSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .ToArray(); + + Assert.Contains( + maintenanceSends, + send => send.RemoteEndPoint.Equals(pushedEndpoint)); + Assert.DoesNotContain( + maintenanceSends, + send => send.RemoteEndPoint.Equals(pushedEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.ExtFrame); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_AdvertisedSurfaces_DoNotEnableHintedPayloadBeforeRendezvous() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 48ce905..d2b21aa 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -2361,7 +2361,7 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer } var selectedEndpoints = ShouldUseStickyHintedPathSelection(peerNodeId) - ? [hinted[0]] + ? SelectStickyHintedMaintenanceEndpoints(peerNodeId, hinted, DirectHintMaintenanceProbeBudget) : _directHintPlanner.TakeNextHintedEndpoints( peerNodeId, hinted, @@ -2381,6 +2381,52 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer return candidates.ToArray(); } + + private IPEndPoint[] SelectStickyHintedMaintenanceEndpoints(NodeId peerNodeId, IPEndPoint[] hinted, int endpointBudget) + { + if (hinted.Length == 0 || endpointBudget <= 0) + { + return Array.Empty(); + } + + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + return [hinted[0]]; + } + + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var pinned = hinted + .Where(directEndpoints.IsPinnedRendezvousEndpoint) + .Take(1) + .ToArray(); + if (pinned.Length == 0) + { + return [hinted[0]]; + } + + if (endpointBudget == 1) + { + return pinned; + } + + var alternates = hinted + .Where(endpoint => + !directEndpoints.IsPinnedRendezvousEndpoint(endpoint) && + ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + .ToArray(); + if (alternates.Length == 0) + { + return pinned; + } + + var rotatedAlternates = _directHintPlanner.TakeNextHintedEndpoints( + peerNodeId, + alternates, + endpointBudget - 1); + return pinned + .Concat(rotatedAlternates) + .ToArray(); + } private static bool TryGetPacketIdAndVerb(ReadOnlyMemory packet, out (ulong PacketId, ZeroTierVerb Verb) parsed) { diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index d01d21f..7eb8c44 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -652,3 +652,36 @@ New notes are appended with timestamps. Older entries are not rewritten. - the non-public alternate-hint path is now exercised live - the remaining blocker is again narrowed back to the wire: - even after probing public sibling ports plus admissible private/shared hints, the peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T17:39:00.8464207+01:00 + +- Checkpoint in progress: + - direct-only maintenance no longer collapses back to the pinned rendezvous endpoint only + - allowed alternate hints are now revisited during maintenance, not just probed once at `PUSH_DIRECT_PATHS` receipt time +- Why it was tried: + - the previous live trace showed a gap in the maintenance selector: + - public sibling ports and admissible private/shared hints were only getting the initial bootstrap `ECHO` + - once the maintenance loop took over, it went back to the pinned rendezvous endpoint only + - that meant alternate hints did not stay warm long enough to test whether the peer might answer later in the connect window +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-maintenance-alt-revisit.log`: + - post-rendezvous peer `PUSH_DIRECT_PATHS` was received as before + - the managed runtime revisited alternate hints during maintenance and escalated them to `HELLO`, not only `ECHO` + - this happened for: + - public sibling endpoints like `176.66.90.119:19665`, `:63878`, `:15196`, `:23523`, `:16476`, `:31761` + - shared/private endpoints like `100.85.196.109:*` and `10.22.10.94:*` + - additional private Docker/LAN endpoints like `172.17.0.1:*` and `172.18.0.1:*` + - pinned rendezvous maintenance on `176.66.90.119:7395` still stayed active with same-socket `ECHO` plus TTL-2 hole-punch +- Issue update: + - the maintenance-selector gap is fixed + - the wire-level blocker remains: + - even with repeated alternate-hint `HELLO` and pinned-rendezvous `ECHO` in the same run, there is still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returns relayed traffic via `84.17.53.155:9993` From 2f0c0d46f48c523536152793630484be730b6de1 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:51:01 +0100 Subject: [PATCH 289/296] Rotate alternate hinted maintenance sockets --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 19 ---- ...oTierStickyHintMaintenanceSelectorTests.cs | 50 ++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 67 ++++--------- .../ZeroTierStickyHintMaintenanceSelector.cs | 95 +++++++++++++++++++ docs/logbook/direct-p2p-jaeger.md | 40 ++++++++ 5 files changed, 201 insertions(+), 70 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index b006263..24aaca0 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -422,15 +422,6 @@ await WaitForConditionAsync( var endpoints = runtime.GetDirectEndpointsForTests(peerIdentity.NodeId); Assert.Equal(rendezvousEndpoint, endpoints[0]); - var pushedBootstrapSockets = udp.GetSendsSnapshot() - .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) - .Select(send => send.LocalSocketId) - .Distinct() - .Order() - .ToArray(); - Assert.Empty(pushedBootstrapSockets); - var rendezvousBootstrapSockets = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && @@ -660,15 +651,6 @@ await WaitForConditionAsync( .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) .Distinct() .ToArray(); - var pushedControlSends = directSends - .Where(send => - send.RemoteEndPoint.Equals(pushedEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - (verb == ZeroTierVerb.Echo || verb == ZeroTierVerb.Hello)) - .Select(send => send.LocalSocketId) - .Distinct() - .Order() - .ToArray(); var payloadSends = directSends .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) @@ -677,7 +659,6 @@ await WaitForConditionAsync( Assert.Contains((1, rendezvousEndpoint), echoSends); Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); - Assert.NotEmpty(pushedControlSends); Assert.Empty(payloadSends); Assert.DoesNotContain(payloadSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); diff --git a/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs new file mode 100644 index 0000000..de76713 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs @@ -0,0 +1,50 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierStickyHintMaintenanceSelectorTests +{ + [Fact] + public async Task SelectCandidates_PinnedRendezvous_RotatesAlternateSocketsAcrossCalls() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + var peerNodeId = new NodeId(0x2222222222); + var endpointManager = new ZeroTierDirectEndpointManager( + udp, + new IPEndPoint(IPAddress.Loopback, 9993), + peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => endpointManager); + var selector = new ZeroTierStickyHintMaintenanceSelector(planner); + var pinnedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var alternateEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var hinted = new[] { pinnedEndpoint, alternateEndpoint }; + + var first = selector.SelectCandidates( + peerNodeId, + hinted, + endpointBudget: 2, + restrictToPinnedRendezvous: true, + endpoint => endpoint.Equals(pinnedEndpoint), + endpoint => endpoint.Equals(alternateEndpoint), + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty()); + var second = selector.SelectCandidates( + peerNodeId, + hinted, + endpointBudget: 2, + restrictToPinnedRendezvous: true, + endpoint => endpoint.Equals(pinnedEndpoint), + endpoint => endpoint.Equals(alternateEndpoint), + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty()); + + Assert.Contains(first, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(pinnedEndpoint)); + Assert.Contains(second, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(pinnedEndpoint)); + Assert.Contains(first, candidate => candidate.LocalSocketId == 0 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); + Assert.Contains(second, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d2b21aa..ec81662 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -56,6 +56,7 @@ internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable private readonly ZeroTierPeerBondPolicyEngine _bondEngine; private readonly ZeroTierPeerRootSocketAffinity _peerRootSocketAffinity; private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; + private readonly ZeroTierStickyHintMaintenanceSelector _stickyHintMaintenanceSelector; private readonly ZeroTierDirectEndpointPolicy _directEndpointPolicy; private readonly ZeroTierDirectPathSocketAdmissibility _directPathSocketAdmissibility; private readonly ZeroTierLocalDirectPathAdvertisementPlanner _localDirectAdvertisementPlanner; @@ -220,6 +221,7 @@ public ZeroTierDataplaneRuntime( udp, GetOrCreateDirectEndpointManager, _directPathSocketAdmissibility); + _stickyHintMaintenanceSelector = new ZeroTierStickyHintMaintenanceSelector(_directHintPlanner); var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); @@ -2360,17 +2362,26 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer return Array.Empty(); } - var selectedEndpoints = ShouldUseStickyHintedPathSelection(peerNodeId) - ? SelectStickyHintedMaintenanceEndpoints(peerNodeId, hinted, DirectHintMaintenanceProbeBudget) - : _directHintPlanner.TakeNextHintedEndpoints( + if (ShouldUseStickyHintedPathSelection(peerNodeId)) + { + return _stickyHintMaintenanceSelector.SelectCandidates( peerNodeId, hinted, - DirectHintMaintenanceProbeBudget); + DirectHintMaintenanceProbeBudget, + ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId), + GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint, + endpoint => ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint), + endpoint => GetStickyHintedLocalSocketIds(peerNodeId, endpoint)); + } + + var selectedEndpoints = _directHintPlanner.TakeNextHintedEndpoints( + peerNodeId, + hinted, + DirectHintMaintenanceProbeBudget); var candidates = new List(selectedEndpoints.Length); for (var i = 0; i < selectedEndpoints.Length; i++) { var socketIds = GetStickyHintedLocalSocketIds(peerNodeId, selectedEndpoints[i]); - if (socketIds.Length == 0) { continue; @@ -2381,52 +2392,6 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer return candidates.ToArray(); } - - private IPEndPoint[] SelectStickyHintedMaintenanceEndpoints(NodeId peerNodeId, IPEndPoint[] hinted, int endpointBudget) - { - if (hinted.Length == 0 || endpointBudget <= 0) - { - return Array.Empty(); - } - - if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) - { - return [hinted[0]]; - } - - var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - var pinned = hinted - .Where(directEndpoints.IsPinnedRendezvousEndpoint) - .Take(1) - .ToArray(); - if (pinned.Length == 0) - { - return [hinted[0]]; - } - - if (endpointBudget == 1) - { - return pinned; - } - - var alternates = hinted - .Where(endpoint => - !directEndpoints.IsPinnedRendezvousEndpoint(endpoint) && - ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint)) - .ToArray(); - if (alternates.Length == 0) - { - return pinned; - } - - var rotatedAlternates = _directHintPlanner.TakeNextHintedEndpoints( - peerNodeId, - alternates, - endpointBudget - 1); - return pinned - .Concat(rotatedAlternates) - .ToArray(); - } private static bool TryGetPacketIdAndVerb(ReadOnlyMemory packet, out (ulong PacketId, ZeroTierVerb Verb) parsed) { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs new file mode 100644 index 0000000..7281afd --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs @@ -0,0 +1,95 @@ +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierStickyHintMaintenanceSelector +{ + private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; + + public ZeroTierStickyHintMaintenanceSelector(ZeroTierDirectHintPathPlanner directHintPlanner) + { + ArgumentNullException.ThrowIfNull(directHintPlanner); + _directHintPlanner = directHintPlanner; + } + + public ZeroTierSelectedPeerPath[] SelectCandidates( + NodeId peerNodeId, + IPEndPoint[] hinted, + int endpointBudget, + bool restrictToPinnedRendezvous, + Func isPinnedRendezvousEndpoint, + Func shouldAllowAlternatePinnedRendezvousHint, + Func getPinnedSocketIds) + { + ArgumentNullException.ThrowIfNull(hinted); + ArgumentNullException.ThrowIfNull(isPinnedRendezvousEndpoint); + ArgumentNullException.ThrowIfNull(shouldAllowAlternatePinnedRendezvousHint); + ArgumentNullException.ThrowIfNull(getPinnedSocketIds); + + if (hinted.Length == 0 || endpointBudget <= 0) + { + return Array.Empty(); + } + + if (!restrictToPinnedRendezvous) + { + return CreatePinnedCandidates(hinted[0], getPinnedSocketIds(hinted[0])); + } + + var pinned = hinted + .Where(isPinnedRendezvousEndpoint) + .Take(1) + .ToArray(); + if (pinned.Length == 0) + { + return CreatePinnedCandidates(hinted[0], getPinnedSocketIds(hinted[0])); + } + + var candidates = new List(endpointBudget); + candidates.AddRange(CreatePinnedCandidates(pinned[0], getPinnedSocketIds(pinned[0]))); + if (candidates.Count >= endpointBudget) + { + return candidates.ToArray(); + } + + var alternates = hinted + .Where(endpoint => + !isPinnedRendezvousEndpoint(endpoint) && + shouldAllowAlternatePinnedRendezvousHint(endpoint)) + .ToArray(); + if (alternates.Length == 0) + { + return candidates.ToArray(); + } + + var rotatedAlternates = _directHintPlanner.TakeNextHintedEndpoints( + peerNodeId, + alternates, + endpointBudget - candidates.Count); + for (var i = 0; i < rotatedAlternates.Length; i++) + { + var socketIds = _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + rotatedAlternates[i], + includeFallbackLocalSockets: true); + if (socketIds.Length == 0) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(socketIds[0], rotatedAlternates[i])); + } + + return candidates.ToArray(); + } + + private static ZeroTierSelectedPeerPath[] CreatePinnedCandidates(IPEndPoint endpoint, int[] socketIds) + { + if (socketIds.Length == 0) + { + return Array.Empty(); + } + + return [new ZeroTierSelectedPeerPath(socketIds[0], endpoint)]; + } +} diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 7eb8c44..b148eab 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -685,3 +685,43 @@ New notes are appended with timestamps. Older entries are not rewritten. - the wire-level blocker remains: - even with repeated alternate-hint `HELLO` and pinned-rendezvous `ECHO` in the same run, there is still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` - the peer still only returns relayed traffic via `84.17.53.155:9993` + +## 2026-03-16T17:49:49.6485658+01:00 + +- Checkpoint in progress: + - sticky direct-hint maintenance selection was extracted into a dedicated helper: + - [ZeroTierStickyHintMaintenanceSelector.cs](/C:/Users/Jonas/repos/private/JKamsker/libzt-dotnet/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs) + - pinned rendezvous maintenance still stays sticky + - alternate hinted maintenance now rotates across admissible sockets in the new focused helper path instead of always collapsing to the first fallback socket +- Why it was tried: + - the previous live trace showed a more specific socket-selection issue after the maintenance-selector fix: + - pinned rendezvous maintenance stayed on the correct root-affine socket + - but repeated alternate-hint `HELLO` still always went out on socket `0` + - that meant the selector had to distinguish: + - pinned rendezvous: stay sticky + - alternates: rotate across admissible sockets +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-maintenance-socket-rotation.log`: + - root again introduced pinned rendezvous on `176.66.90.119:7395` + - pinned rendezvous maintenance stayed on socket `7` with same-socket `ECHO` + - peer `PUSH_DIRECT_PATHS` again included: + - public sibling endpoints like `176.66.90.119:19665`, `:63878`, `:49134`, `:30840`, `:63656` + - shared/private endpoints like `100.85.196.109:*` and `10.22.10.94:*` + - repeated alternate-hint `HELLO` still went out on socket `0` in the live run + - there was still no `RX OK(ECHO)`, no `RX direct raw`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the new helper fixed and isolated one maintenance-path selector locally + - the live blocker changed shape again: + - the repeated post-rendezvous alternate-hint `HELLO` traffic is still not coming from the selector that was fixed here, because it remains pinned to socket `0` on the wire + - the next remaining local target is therefore the alternate-hint retry path outside the sticky maintenance selector, not the pinned-rendezvous maintenance loop From e7d8de5d499035681d899f4a2b50a5630bb7b4ce Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:04:11 +0100 Subject: [PATCH 290/296] Require rendezvous before direct-only hinted control --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 96 ++++++++++++++----- ...roTierDirectBootstrapControlPolicyTests.cs | 22 +++++ .../Internal/ZeroTierDataplaneRuntime.cs | 52 ++++++++-- .../ZeroTierDirectBootstrapControlPolicy.cs | 14 +++ docs/logbook/direct-p2p-jaeger.md | 38 ++++++++ 5 files changed, 193 insertions(+), 29 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 24aaca0..d2f21cd 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -510,7 +510,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_PushDirectPaths_BootstrapsAcrossEligibleSockets() + public async Task DataplaneRuntime_DirectOnly_PushDirectPaths_WaitsForRendezvous() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -544,13 +544,6 @@ public async Task DataplaneRuntime_DirectOnly_PushDirectPaths_BootstrapsAcrossEl RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); - await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => - send.RemoteEndPoint.Equals(hintedEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.Echo), - TimeSpan.FromSeconds(1)); - var echoSends = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) @@ -559,7 +552,7 @@ await WaitForConditionAsync( .Order() .ToArray(); - Assert.Equal([0, 1], echoSends); + Assert.Empty(echoSends); } [Fact] @@ -931,6 +924,76 @@ await WaitForConditionAsync( verb == ZeroTierVerb.ExtFrame); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_UsesRotatedAlternateHintSocketAfterPinnedRendezvous() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(pushedEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var maintenanceSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + Assert.Equal(new[] { 0, 1 }, maintenanceSends); + } + [Fact] public async Task DataplaneRuntime_DirectOnly_AdvertisedSurfaces_DoNotEnableHintedPayloadBeforeRendezvous() { @@ -1087,7 +1150,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsBrieflyForRendezvous_BeforeUsingHintedPaths() + public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForRendezvous_BeforeUsingHintedPaths() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -1141,22 +1204,11 @@ public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsBrieflyForRendezvo RootEndpoint, BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); - await WaitForConditionAsync( - () => udp.GetSendsSnapshot().Any(send => - send.RemoteEndPoint.Equals(hintedEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.Echo), - TimeSpan.FromSeconds(1)); - var hintedSends = udp.GetSendsSnapshot() .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint)) .ToArray(); - var payloadSends = hintedSends - .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) - .ToArray(); - Assert.NotEmpty(hintedSends); - Assert.Empty(payloadSends); + Assert.Empty(hintedSends); await Assert.ThrowsAnyAsync(async () => await sendTask); } diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs index e7eeb1e..8d6e935 100644 --- a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -114,6 +114,28 @@ public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( hasConfirmedDirectPath)); } + [Theory] + [InlineData(true, false, false, false, false)] + [InlineData(false, true, false, false, false)] + [InlineData(false, false, true, false, false)] + [InlineData(false, false, false, true, false)] + [InlineData(false, false, false, false, true)] + public void ShouldDelayDirectOnlyHintedControlUntilRendezvous_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool endpointIsPinnedRendezvous, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldDelayDirectOnlyHintedControlUntilRendezvous( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + endpointIsPinnedRendezvous)); + } + [Theory] [InlineData(false, false, false, 1_000L, 0L, 3_000L, null, 1_000L, true)] [InlineData(false, false, false, 3_500L, 0L, 3_000L, null, 1_000L, true)] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index ec81662..8ec1c8a 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1032,7 +1032,8 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared if (hinted.Length > 0 && unchecked(now - nextHintProbeAt) >= 0) { - if (ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(peerNodeId)) + if (ShouldDelayDirectOnlyHintedControlUntilRendezvous(peerNodeId, hinted[0]) || + ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(peerNodeId)) { nextHintProbeAt = now + 250; } @@ -1455,6 +1456,16 @@ private async ValueTask HandleDirectEndpointHintAsync( return; } } + else if (ShouldDelayDirectOnlyHintedControlUntilRendezvous(peerNodeId, endpoint)) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap delayed: peer={peerNodeId} endpoint={endpoint} reason=await-rendezvous."); + } + + return; + } if (ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(peerNodeId)) { @@ -1866,6 +1877,16 @@ private bool ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved directEndpoints.HasPinnedRendezvousEndpoints); } + private bool ShouldDelayDirectOnlyHintedControlUntilRendezvous(NodeId peerNodeId, IPEndPoint endpoint) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + return ZeroTierDirectBootstrapControlPolicy.ShouldDelayDirectOnlyHintedControlUntilRendezvous( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + directEndpoints.IsPinnedRendezvousEndpoint(endpoint)); + } + private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId peerNodeId) { var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); @@ -2190,9 +2211,7 @@ private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellati ? DirectOnlyHintHelloIntervalMs : DirectHintFullHelloIntervalMs) : DirectHintFullHelloIntervalMs; - var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) - ? GetStickyDirectHintProbeSocketIds(peerNodeId, candidate.RemoteEndPoint, forceFullHello) - : [candidate.LocalSocketId]; + var localSocketIds = GetHintedMaintenanceProbeSocketIds(peerNodeId, candidate, forceFullHello); RefreshPinnedRendezvousHolePunch(peerNodeId, candidate.RemoteEndPoint, localSocketIds); @@ -2301,9 +2320,28 @@ await SendPeerControlAsync( peerProtocolVersion, cancellationToken) .ConfigureAwait(false); - } - } - + } + } + + private int[] GetHintedMaintenanceProbeSocketIds( + NodeId peerNodeId, + ZeroTierSelectedPeerPath candidate, + bool forceFullHello) + { + if (!ShouldUseStickyHintedPathSelection(peerNodeId)) + { + return [candidate.LocalSocketId]; + } + + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (directEndpoints.IsPinnedRendezvousEndpoint(candidate.RemoteEndPoint)) + { + return GetStickyDirectHintProbeSocketIds(peerNodeId, candidate.RemoteEndPoint, forceFullHello); + } + + return [candidate.LocalSocketId]; + } + private int ComputePathQualityScore(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) { if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs index a480ca7..a17e93c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -62,6 +62,20 @@ public static bool ShouldRefreshRelayedRootControl( bool hasConfirmedDirectPath) => allowRootRelayFallback || !hasConfirmedDirectPath; + public static bool ShouldDelayDirectOnlyHintedControlUntilRendezvous( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool endpointIsPinnedRendezvous) + { + if (allowRootRelayFallback || hasConfirmedDirectPath || endpointIsPinnedRendezvous) + { + return false; + } + + return !hasPinnedRendezvousEndpoints; + } + public static bool ShouldWaitForDirectOnlyHintedPayload( bool allowRootRelayFallback, bool hasConfirmedDirectPath, diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index b148eab..643bd03 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -725,3 +725,41 @@ New notes are appended with timestamps. Older entries are not rewritten. - the live blocker changed shape again: - the repeated post-rendezvous alternate-hint `HELLO` traffic is still not coming from the selector that was fixed here, because it remains pinned to socket `0` on the wire - the next remaining local target is therefore the alternate-hint retry path outside the sticky maintenance selector, not the pinned-rendezvous maintenance loop + +## 2026-03-16T18:03:26.4107889+01:00 + +- Checkpoint in progress: + - direct-only ordinary hinted control is now held until rendezvous exists + - the maintenance loop now preserves rotated alternate-hint socket selection instead of overwriting it with the sticky selector +- Why it was tried: + - the previous live trace showed two distinct problems: + - pre-rendezvous direct-only control was still spending time on ordinary `PUSH_DIRECT_PATHS` + - once post-rendezvous alternate-hint `HELLO` retries started, they were all collapsing back to socket `0` + - that meant the runtime was still wasting bootstrap budget before rendezvous and not actually exercising multi-socket alternate retries after rendezvous +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-first-control.log`: + - root introduced pinned rendezvous first: + - `176.66.90.119:7395` + - initial rendezvous probe was correctly delayed only for peer-version learning: + - `Direct hint bootstrap delayed ... reason=await-peer-version` + - after `PUSH_DIRECT_PATHS`, pinned rendezvous maintenance stayed on socket `7` with repeated same-socket `ECHO` + - the alternate-hint `HELLO` retry path changed materially on the wire: + - public sibling endpoints like `176.66.90.119:19665`, `:2918`, `:7751`, `:52406`, `:53184`, `:35228`, `:22362` + - shared/private endpoints like `100.85.196.109:*` and `10.22.10.94:*` + - all rotated across sockets `0`, then `1`, then `2`, then `3`, then `4` + - despite that, there was still no `RX OK(ECHO)`, no `RX direct raw`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the alternate-hint retry path is no longer artificially stuck on one local socket + - the remaining blocker narrowed again to the wire: + - even with pinned rendezvous on one socket and alternate sibling/private retries rotating across multiple sockets, the peer still never returns a confirmed direct packet to this runtime From 1c127728685ca1a7798c10752a6e2bbbff297963 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:25:53 +0100 Subject: [PATCH 291/296] Predict direct-path ports around rendezvous --- ...nnedRendezvousSiblingPortPredictorTests.cs | 40 +++++++++++ ...ierPredictedPublicSurfaceGeneratorTests.cs | 54 +++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 33 ++++++++- ...ierPinnedRendezvousSiblingPortPredictor.cs | 67 +++++++++++++++++++ ...ZeroTierPredictedPublicSurfaceGenerator.cs | 67 +++++++++++++++++++ docs/logbook/direct-p2p-jaeger.md | 40 +++++++++++ 6 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 ZTSharp.Tests/ZeroTierPinnedRendezvousSiblingPortPredictorTests.cs create mode 100644 ZTSharp.Tests/ZeroTierPredictedPublicSurfaceGeneratorTests.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousSiblingPortPredictor.cs create mode 100644 ZTSharp/ZeroTier/Internal/ZeroTierPredictedPublicSurfaceGenerator.cs diff --git a/ZTSharp.Tests/ZeroTierPinnedRendezvousSiblingPortPredictorTests.cs b/ZTSharp.Tests/ZeroTierPinnedRendezvousSiblingPortPredictorTests.cs new file mode 100644 index 0000000..6ba2219 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPinnedRendezvousSiblingPortPredictorTests.cs @@ -0,0 +1,40 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPinnedRendezvousSiblingPortPredictorTests +{ + [Fact] + public void Expand_AddsAdjacentPortsForPinnedPublicRendezvous() + { + var pinned = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7395); + var sibling = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + + var expanded = ZeroTierPinnedRendezvousSiblingPortPredictor.Expand( + [pinned, sibling], + endpoint => endpoint.Equals(pinned)); + + Assert.Equal( + [ + pinned, + sibling, + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7394), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7396), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7393), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7397) + ], expanded); + } + + [Fact] + public void Expand_DoesNotAddAdjacentPortsForPinnedPrivateRendezvous() + { + var pinned = new IPEndPoint(IPAddress.Parse("10.22.10.94"), 9993); + + var expanded = ZeroTierPinnedRendezvousSiblingPortPredictor.Expand( + [pinned], + endpoint => endpoint.Equals(pinned)); + + Assert.Equal([pinned], expanded); + } +} diff --git a/ZTSharp.Tests/ZeroTierPredictedPublicSurfaceGeneratorTests.cs b/ZTSharp.Tests/ZeroTierPredictedPublicSurfaceGeneratorTests.cs new file mode 100644 index 0000000..7f3f20c --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPredictedPublicSurfaceGeneratorTests.cs @@ -0,0 +1,54 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPredictedPublicSurfaceGeneratorTests +{ + [Fact] + public void Expand_AddsAdjacentPortsForPublicEndpoints() + { + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand( + [ + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000) + ]); + + Assert.Equal( + [ + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 59999), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 59998), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60002) + ], expanded); + } + + [Fact] + public void Expand_DoesNotAddAdjacentPortsForNonPublicEndpoints() + { + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand( + [ + new IPEndPoint(IPAddress.Parse("10.0.0.60"), 60000), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 60001) + ]); + + Assert.Equal( + [ + new IPEndPoint(IPAddress.Parse("10.0.0.60"), 60000), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 60001) + ], expanded); + } + + [Fact] + public void Expand_SkipsOutOfRangeAdjacentPorts() + { + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand( + [ + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 1), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 65535) + ]); + + Assert.DoesNotContain(expanded, endpoint => endpoint.Port == 0); + Assert.DoesNotContain(expanded, endpoint => endpoint.Port > ushort.MaxValue); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 8ec1c8a..ef113fe 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -325,8 +325,9 @@ private IPEndPoint[] GetLocalDirectPathAdvertisements() } } + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand(endpoints); return ZeroTierDirectEndpointSelection.Normalize( - endpoints.Where(static endpoint => endpoint.Port != 0).Select(UdpEndpointNormalization.Normalize), + expanded.Where(static endpoint => endpoint.Port != 0).Select(UdpEndpointNormalization.Normalize), _rootEndpoint, maxEndpoints: ZeroTierProtocolLimits.MaxPushedDirectPaths); } @@ -1005,7 +1006,9 @@ private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] shared now = Environment.TickCount64; var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - var hinted = directEndpoints.Endpoints; + var hinted = ZeroTierPinnedRendezvousSiblingPortPredictor.Expand( + directEndpoints.Endpoints, + directEndpoints.IsPinnedRendezvousEndpoint); var suppressRelayedRootControl = ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(peerNodeId, now); var suppressRelayedRootMaintenance = ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved(peerNodeId); if (unchecked(now - nextRootHelloAt) >= 0 && @@ -1095,7 +1098,7 @@ private async Task ProbeHintedDirectEndpointsAsync( } var endpointsToProbe = ShouldUseStickyHintedPathSelection(peerNodeId) - ? [hinted[0]] + ? SelectStickyHintedEndpointsToProbe(peerNodeId, hinted) : _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); for (var i = 0; i < endpointsToProbe.Length; i++) { @@ -1831,6 +1834,30 @@ private bool TrySelectConfirmedHintedDirectPath( private bool ShouldUseStickyHintedPathSelection(NodeId peerNodeId) => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + private IPEndPoint[] SelectStickyHintedEndpointsToProbe(NodeId peerNodeId, IPEndPoint[] hinted) + { + if (hinted.Length == 0) + { + return []; + } + + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + return [hinted[0]]; + } + + var selected = new List { hinted[0] }; + for (var i = 1; i < hinted.Length; i++) + { + if (ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, hinted[i])) + { + selected.Add(hinted[i]); + } + } + + return selected.ToArray(); + } + private bool ShouldWaitForPinnedRendezvousBeforeHintedPayload(NodeId peerNodeId) { var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousSiblingPortPredictor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousSiblingPortPredictor.cs new file mode 100644 index 0000000..d57a4f4 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousSiblingPortPredictor.cs @@ -0,0 +1,67 @@ +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierPinnedRendezvousSiblingPortPredictor +{ + private const int AdjacentPortRadius = 2; + + public static IPEndPoint[] Expand( + IReadOnlyList endpoints, + Func isPinnedRendezvousEndpoint) + { + ArgumentNullException.ThrowIfNull(endpoints); + ArgumentNullException.ThrowIfNull(isPinnedRendezvousEndpoint); + + var expanded = new List(endpoints.Count + (AdjacentPortRadius * 2)); + var seen = new HashSet(StringComparer.Ordinal); + + for (var i = 0; i < endpoints.Count; i++) + { + AddUnique(expanded, seen, endpoints[i]); + } + + var pinnedPublic = endpoints.FirstOrDefault(endpoint => + isPinnedRendezvousEndpoint(endpoint) && + ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)); + if (pinnedPublic is null) + { + return expanded.ToArray(); + } + + for (var delta = 1; delta <= AdjacentPortRadius; delta++) + { + AddAdjacent(expanded, seen, pinnedPublic, -delta); + AddAdjacent(expanded, seen, pinnedPublic, delta); + } + + return expanded.ToArray(); + } + + private static void AddAdjacent( + List expanded, + HashSet seen, + IPEndPoint endpoint, + int delta) + { + var port = endpoint.Port + delta; + if (port is < 1 or > ushort.MaxValue) + { + return; + } + + AddUnique(expanded, seen, new IPEndPoint(endpoint.Address, port)); + } + + private static void AddUnique( + List expanded, + HashSet seen, + IPEndPoint endpoint) + { + var key = endpoint.Address + ":" + endpoint.Port; + if (seen.Add(key)) + { + expanded.Add(endpoint); + } + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPredictedPublicSurfaceGenerator.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPredictedPublicSurfaceGenerator.cs new file mode 100644 index 0000000..87f2962 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPredictedPublicSurfaceGenerator.cs @@ -0,0 +1,67 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierPredictedPublicSurfaceGenerator +{ + private const int AdjacentPortRadius = 2; + + public static IPEndPoint[] Expand(IEnumerable endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + var expanded = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var endpoint in endpoints) + { + if (endpoint is null) + { + continue; + } + + AddUnique(expanded, seen, endpoint); + if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + continue; + } + + for (var delta = 1; delta <= AdjacentPortRadius; delta++) + { + AddAdjacent(expanded, seen, endpoint, -delta); + AddAdjacent(expanded, seen, endpoint, delta); + } + } + + return expanded.ToArray(); + } + + private static void AddAdjacent( + List expanded, + HashSet seen, + IPEndPoint endpoint, + int delta) + { + var port = endpoint.Port + delta; + if (port is < 1 or > ushort.MaxValue) + { + return; + } + + AddUnique(expanded, seen, new IPEndPoint(endpoint.Address, port)); + } + + private static void AddUnique( + List expanded, + HashSet seen, + IPEndPoint endpoint) + { + var canonical = UdpEndpointNormalization.Normalize(endpoint); + var key = canonical.Address + ":" + canonical.Port; + if (seen.Add(key)) + { + expanded.Add(canonical); + } + } +} diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 643bd03..030b846 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -763,3 +763,43 @@ New notes are appended with timestamps. Older entries are not rewritten. - the alternate-hint retry path is no longer artificially stuck on one local socket - the remaining blocker narrowed again to the wire: - even with pinned rendezvous on one socket and alternate sibling/private retries rotating across multiple sockets, the peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T18:25:14.6073355+01:00 + +- Checkpoint in progress: + - relayed local direct-path advertisements now include conservative adjacent public-port predictions around each root-observed public surface + - pinned public rendezvous maintenance now also probes conservative adjacent sibling ports around the root-introduced rendezvous port +- Why it was tried: + - the previous live traces still showed zero hop-0 return traffic, which means the peer either still did not know a reachable return port for this runtime or the exact root-provided rendezvous port was not the only viable public candidate + - two transport-side levers were still available without using unrelated services: + - advertise likely adjacent public return ports for our own side + - directly probe likely adjacent public sibling ports around the pinned peer rendezvous port +- Verification: + - focused suites passed: + - `ZeroTierPredictedPublicSurfaceGeneratorTests` + - `ZeroTierPinnedRendezvousSiblingPortPredictorTests` + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-predicted-public-ads.log`: + - root-surface seeding and relayed `PUSH_DIRECT_PATHS` now clearly include predicted public return ports like: + - `212.241.85.84:50150..50161` + - despite that wider advertised local return set, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` +- Live result in `.codex-debug/direct-only-post-sticky-sibling-prediction.log`: + - after root pinned rendezvous at `176.66.90.119:7395`, the bootstrap loop now really probes adjacent sibling ports too: + - `176.66.90.119:7394` + - `176.66.90.119:7396` + - `176.66.90.119:7393` + - `176.66.90.119:7397` + - payload still stayed pinned to the rendezvous path + - even with both levers active together, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the blocker changed again in a useful way: + - predicted local public return ports are now definitely being advertised to the peer + - predicted sibling public ports around the pinned rendezvous port are now definitely being probed on the wire + - the remaining blocker is still not local selection or missing probe coverage: + - even after both local-return prediction and pinned-rendezvous sibling-port probing are active, the peer still never returns a single confirmed hop-0 packet to this runtime From 19ef004ddb637b8fecfde95644b1a585014a9d00 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:32:10 +0100 Subject: [PATCH 292/296] Pin sibling direct probes to rendezvous socket --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 2 +- ...oTierStickyHintMaintenanceSelectorTests.cs | 8 ++-- .../Internal/ZeroTierDataplaneRuntime.cs | 45 ++++++++++++++++++- .../ZeroTierStickyHintMaintenanceSelector.cs | 9 ++-- docs/logbook/direct-p2p-jaeger.md | 44 ++++++++++++++++++ 5 files changed, 98 insertions(+), 10 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index d2f21cd..2ca430b 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -991,7 +991,7 @@ await WaitForConditionAsync( .Order() .ToArray(); - Assert.Equal(new[] { 0, 1 }, maintenanceSends); + Assert.Equal(new[] { 1 }, maintenanceSends); } [Fact] diff --git a/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs index de76713..942846a 100644 --- a/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs +++ b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs @@ -32,7 +32,8 @@ public async Task SelectCandidates_PinnedRendezvous_RotatesAlternateSocketsAcros restrictToPinnedRendezvous: true, endpoint => endpoint.Equals(pinnedEndpoint), endpoint => endpoint.Equals(alternateEndpoint), - endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty()); + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty(), + endpoint => endpoint.Equals(alternateEndpoint) ? new[] { 1 } : Array.Empty()); var second = selector.SelectCandidates( peerNodeId, hinted, @@ -40,11 +41,12 @@ public async Task SelectCandidates_PinnedRendezvous_RotatesAlternateSocketsAcros restrictToPinnedRendezvous: true, endpoint => endpoint.Equals(pinnedEndpoint), endpoint => endpoint.Equals(alternateEndpoint), - endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty()); + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty(), + endpoint => endpoint.Equals(alternateEndpoint) ? new[] { 1 } : Array.Empty()); Assert.Contains(first, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(pinnedEndpoint)); Assert.Contains(second, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(pinnedEndpoint)); - Assert.Contains(first, candidate => candidate.LocalSocketId == 0 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); + Assert.Contains(first, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); Assert.Contains(second, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index ef113fe..bfcfc57 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1931,6 +1931,15 @@ private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId pe private int[] GetStickyDirectHintProbeSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) { + if (ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + { + var pinnedSocketIds = GetPinnedRendezvousStickySocketIds(peerNodeId); + if (pinnedSocketIds.Length != 0) + { + return pinnedSocketIds; + } + } + if (ShouldFanOutFullHelloAcrossEligibleSockets(peerNodeId, endpoint, forceFullHello)) { return _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); @@ -1970,6 +1979,34 @@ private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoi return [socketIds[0]]; } + private int[] GetPinnedRendezvousStickySocketIds(NodeId peerNodeId) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var pinnedEndpoint = directEndpoints.Endpoints.FirstOrDefault(directEndpoints.IsPinnedRendezvousEndpoint); + if (pinnedEndpoint is null) + { + return []; + } + + return GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoint); + } + + private bool ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) + { + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) || + !ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint) || + !ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + return false; + } + + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + return directEndpoints.Endpoints.Any(candidate => + directEndpoints.IsPinnedRendezvousEndpoint(candidate) && + ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidate) && + candidate.Address.Equals(endpoint.Address)); + } + private void RefreshPinnedRendezvousHolePunch(NodeId peerNodeId, IPEndPoint endpoint, int[] localSocketIds) { if (_multipath.AllowRootRelayFallback || localSocketIds.Length == 0) @@ -2436,7 +2473,13 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId), GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint, endpoint => ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint), - endpoint => GetStickyHintedLocalSocketIds(peerNodeId, endpoint)); + endpoint => GetStickyHintedLocalSocketIds(peerNodeId, endpoint), + endpoint => ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint) + ? GetPinnedRendezvousStickySocketIds(peerNodeId) + : _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true)); } var selectedEndpoints = _directHintPlanner.TakeNextHintedEndpoints( diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs index 7281afd..a4326ac 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs @@ -19,12 +19,14 @@ public ZeroTierSelectedPeerPath[] SelectCandidates( bool restrictToPinnedRendezvous, Func isPinnedRendezvousEndpoint, Func shouldAllowAlternatePinnedRendezvousHint, - Func getPinnedSocketIds) + Func getPinnedSocketIds, + Func getAlternateSocketIds) { ArgumentNullException.ThrowIfNull(hinted); ArgumentNullException.ThrowIfNull(isPinnedRendezvousEndpoint); ArgumentNullException.ThrowIfNull(shouldAllowAlternatePinnedRendezvousHint); ArgumentNullException.ThrowIfNull(getPinnedSocketIds); + ArgumentNullException.ThrowIfNull(getAlternateSocketIds); if (hinted.Length == 0 || endpointBudget <= 0) { @@ -68,10 +70,7 @@ public ZeroTierSelectedPeerPath[] SelectCandidates( endpointBudget - candidates.Count); for (var i = 0; i < rotatedAlternates.Length; i++) { - var socketIds = _directHintPlanner.GetRotatingSocketIds( - peerNodeId, - rotatedAlternates[i], - includeFallbackLocalSockets: true); + var socketIds = getAlternateSocketIds(rotatedAlternates[i]); if (socketIds.Length == 0) { continue; diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 030b846..7a1f14d 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -803,3 +803,47 @@ New notes are appended with timestamps. Older entries are not rewritten. - predicted sibling public ports around the pinned rendezvous port are now definitely being probed on the wire - the remaining blocker is still not local selection or missing probe coverage: - even after both local-return prediction and pinned-rendezvous sibling-port probing are active, the peer still never returns a single confirmed hop-0 packet to this runtime + +## 2026-03-16T18:31:41.0651393+01:00 + +- Checkpoint in progress: + - same-IP public sibling probes around the pinned rendezvous endpoint now inherit the pinned rendezvous socket instead of falling back to planner-selected sockets like `0` + - the sticky maintenance selector now uses the pinned socket for public alternates that share the pinned rendezvous IP +- Why it was tried: + - the previous live trace changed the issue again: + - sibling public ports like `176.66.90.119:7394..7397` were finally being probed + - but they were still going out on socket `0` while the root-introduced rendezvous path itself was on socket `7` + - that left a remaining local NAT-asymmetry risk: the peer might see alternate same-IP probes from a different public mapping than the one root had just introduced +- Verification: + - focused suites passed: + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierPredictedPublicSurfaceGeneratorTests` + - `ZeroTierPinnedRendezvousSiblingPortPredictorTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-sibling-socket-affinity.log`: + - root introduced pinned rendezvous on `176.66.90.119:7395` via socket `7` + - the sibling probes now really stay on that same socket: + - `TX HELLO bootstrap to 176.66.90.119:7394 (socket=7, ...)` + - `TX HELLO bootstrap to 176.66.90.119:7396 (socket=7, ...)` + - `TX HELLO bootstrap to 176.66.90.119:7393 (socket=7, ...)` + - `TX HELLO bootstrap to 176.66.90.119:7397 (socket=7, ...)` + - the same-socket public-sibling behavior also applied to other public hints on the same peer IP, such as: + - `176.66.90.119:19665` + - `176.66.90.119:50163` + - `176.66.90.119:26412` + - `176.66.90.119:11674` + - `176.66.90.119:46770` + - `176.66.90.119:5395` + - despite that, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the local public-sibling socket-affinity bug is fixed + - the remaining blocker narrowed again: + - this is no longer missing sibling-port probing + - and no longer the wrong local socket for those sibling probes + - even with root-rendezvous, sibling public probes, and sibling public socket affinity all aligned on the same socket, the peer still never returns a confirmed hop-0 packet From 2ac30d1b237dcff2de9a1c5da8402bd909ef39ec Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:42:54 +0100 Subject: [PATCH 293/296] Keep pinned sibling direct probes on echo --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 31 ++++------ .../Internal/ZeroTierDataplaneRuntime.cs | 1 + docs/logbook/direct-p2p-jaeger.md | 61 +++++++++++++++++++ 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 2ca430b..5a266d5 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -836,13 +836,14 @@ await WaitForConditionAsync( if (pushedControlSends.Length != 0) { - Assert.All( + Assert.Contains( pushedControlSends, - send => - { - Assert.True(TryDecodeVerb(send.Payload, sharedKey, out var verb)); - Assert.True(verb is ZeroTierVerb.Echo or ZeroTierVerb.Hello); - }); + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo); + Assert.DoesNotContain( + pushedControlSends, + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); } Assert.DoesNotContain( @@ -853,7 +854,7 @@ await WaitForConditionAsync( } [Fact] - public async Task DataplaneRuntime_DirectOnly_Maintenance_RevisitsAlternateHintsAfterPinnedRendezvous() + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRevisitPinnedRendezvousSiblingHint() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -914,18 +915,11 @@ await WaitForConditionAsync( .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) .ToArray(); - Assert.Contains( - maintenanceSends, - send => send.RemoteEndPoint.Equals(pushedEndpoint)); - Assert.DoesNotContain( - maintenanceSends, - send => send.RemoteEndPoint.Equals(pushedEndpoint) && - TryDecodeVerb(send.Payload, sharedKey, out var verb) && - verb == ZeroTierVerb.ExtFrame); + Assert.Empty(maintenanceSends); } [Fact] - public async Task DataplaneRuntime_DirectOnly_Maintenance_UsesRotatedAlternateHintSocketAfterPinnedRendezvous() + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotPeriodicallyForceHelloOnPinnedRendezvousSiblingHint() { await using var udp = new ScriptedUdpTransport( new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), @@ -986,12 +980,9 @@ await WaitForConditionAsync( .Skip(initialSendCount) .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) - .Select(send => send.LocalSocketId) - .Distinct() - .Order() .ToArray(); - Assert.Equal(new[] { 1 }, maintenanceSends); + Assert.Empty(maintenanceSends); } [Fact] diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index bfcfc57..d9d8b12 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -2198,6 +2198,7 @@ private bool ShouldFanOutHintedBootstrapAcrossSockets(NodeId peerNodeId) private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId, IPEndPoint endpoint) => ShouldUseStickyHintedPathSelection(peerNodeId) && CanUseEchoForDirectBootstrap(peerNodeId) && + !ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint) && !GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint); private bool ShouldForceFullHelloForDirectOnlyPayloadPrime(NodeId peerNodeId, IPEndPoint endpoint) diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 7a1f14d..041fdc8 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -847,3 +847,64 @@ New notes are appended with timestamps. Older entries are not rewritten. - this is no longer missing sibling-port probing - and no longer the wrong local socket for those sibling probes - even with root-rendezvous, sibling public probes, and sibling public socket affinity all aligned on the same socket, the peer still never returns a confirmed hop-0 packet + +## 2026-03-16T18:37:40.8390788+01:00 +- Change: + - same-IP public sibling hints around a pinned rendezvous no longer get periodically promoted to full `HELLO` + - the sticky hinted maintenance rule now keeps those sibling probes on `ECHO` while still allowing unrelated alternates to escalate to `HELLO` +- Why it was tried: + - the previous live trace changed the issue again: + - sibling public ports around the pinned rendezvous were finally using the correct local socket + - but they were still being retried as `HELLO`, not `ECHO` + - that left a remaining protocol-shape mismatch versus the intended modern-peer direct bootstrap: + - pinned rendezvous itself was already `ECHO`-first + - same-IP sibling ports around that rendezvous should follow the same direct `ECHO` path shape instead of periodically switching to `HELLO` +- Expected verification: + - focused suites should still pass: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - live trace should change from sibling `TX HELLO bootstrap to 176.66.90.119:7394..7397` to sibling `TX ECHO bootstrap` on the pinned rendezvous socket +- Issue update: + - the remaining local mismatch is now specifically the verb choice on same-IP sibling public retries after rendezvous + - if this patch behaves as intended and the peer still does not respond hop-0, the blocker will narrow further to the remote side ignoring both pinned-rendezvous and sibling same-IP `ECHO` traffic + +## 2026-03-16T18:42:22.0606954+01:00 +- Change: + - same-IP public sibling retries around a pinned rendezvous now stay on `ECHO` instead of periodically switching to `HELLO` + - the focused pinned-rendezvous maintenance tests were tightened to reflect that newer direct-bootstrap contract +- Why it was tried: + - the previous live trace changed the issue again: + - sibling public ports around the pinned rendezvous were finally using the correct local socket + - but they were still being retried as `HELLO`, which did not match the intended modern-peer `ECHO`-first path shape +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-sibling-echo.log`: + - root introduced pinned rendezvous on `176.66.90.119:7395` via socket `6` + - same-IP sibling public retries now really stay on `ECHO` on that same socket: + - `TX ECHO bootstrap to 176.66.90.119:7394 (socket=6)` + - `TX ECHO bootstrap to 176.66.90.119:7396 (socket=6)` + - `TX ECHO bootstrap to 176.66.90.119:7393 (socket=6)` + - `TX ECHO bootstrap to 176.66.90.119:7397 (socket=6)` + - the same `ECHO`-only shape also applied to other public same-IP hints like: + - `176.66.90.119:19665` + - `176.66.90.119:37463` + - `176.66.90.119:27299` + - `176.66.90.119:44556` + - `176.66.90.119:5395` + - despite that, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the local same-IP sibling verb-selection bug is fixed + - the remaining blocker narrowed again: + - this is no longer wrong local socket for sibling public retries + - and no longer wrong control verb for sibling public retries + - even with pinned-rendezvous `ECHO` plus sibling same-IP `ECHO` on the same socket, the peer still never returns a confirmed hop-0 packet From 4bcd1a87d2b17aa57eacb20298ab519a94efa592 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:53:01 +0100 Subject: [PATCH 294/296] Clamp initial pinned sibling bootstrap --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 61 +++++++++++++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 15 ++++- docs/logbook/direct-p2p-jaeger.md | 37 +++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 5a266d5..3e28592 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -106,6 +106,67 @@ await WaitForConditionAsync( Assert.All(bootstrapSends, send => Assert.Equal(1, send.LocalSocketId)); } + [Fact] + public async Task DataplaneRuntime_RendezvousBootstrap_UsesReceivingSocket_ForPublicSiblingHint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var siblingEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, siblingEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(siblingEndpoint)), + TimeSpan.FromSeconds(2)); + + var bootstrapSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(siblingEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .ToArray(); + + Assert.NotEmpty(bootstrapSends); + Assert.All(bootstrapSends, send => Assert.Equal(1, send.LocalSocketId)); + } + [Fact] public async Task DataplaneRuntime_Rendezvous_DirectOnly_DoesNotRelayRootHello() { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d9d8b12..d8dd355 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1446,7 +1446,7 @@ private async ValueTask HandleDirectEndpointHintAsync( } else if (ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint)) { - useAllEligibleLocalSockets = true; + useAllEligibleLocalSockets = !ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint); } else { @@ -1988,7 +1988,18 @@ private int[] GetPinnedRendezvousStickySocketIds(NodeId peerNodeId) return []; } - return GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoint); + var socketIds = GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoint); + if (socketIds.Length != 0) + { + return socketIds; + } + + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) + { + return [rootSocketId]; + } + + return []; } private bool ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 041fdc8..cc53d80 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -908,3 +908,40 @@ New notes are appended with timestamps. Older entries are not rewritten. - this is no longer wrong local socket for sibling public retries - and no longer wrong control verb for sibling public retries - even with pinned-rendezvous `ECHO` plus sibling same-IP `ECHO` on the same socket, the peer still never returns a confirmed hop-0 packet + +## 2026-03-16T18:52:18.4803247+01:00 +- Change: + - the initial direct-hint callback path now stops forcing same-IP public sibling hints around a pinned rendezvous into `allSockets=True` + - if the planner does not yet have a preferred socket for the pinned rendezvous, pinned-socket selection now falls back to the observed root/rendezvous socket +- Why it was tried: + - the previous live trace changed the issue again: + - later maintenance had already become same-socket `ECHO` + - but the first post-rendezvous callback path was still spraying sibling public hints like `176.66.90.119:19665` across sockets `0..7` + - that meant the initial rendezvous follow-up was still using a broader local NAT surface than the later sticky maintenance path +- Verification: + - focused direct-path tests passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-initial-sibling-clamp.log`: + - root-introduced rendezvous and same-IP public sibling hints now all start same-socket on `4`: + - `TX ECHO bootstrap to 176.66.90.119:7395 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:19665 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:37463 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:27299 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:44556 (socket=4)` + - public same-IP alternates no longer show `allSockets=True` + - non-public alternates still do, for example: + - `100.85.196.109:9993` + - `100.85.196.109:49161` + - `172.17.0.1:9993` + - `172.18.0.1:9993` + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the initial same-IP public sibling spray bug is fixed + - the remaining blocker narrowed again: + - public rendezvous plus public same-IP sibling probes now both start same-socket and `ECHO`-first + - the peer still never returns a confirmed hop-0 packet + - the remaining broad fanout is now on non-public alternate hints, not on the root-introduced public path From 83ab9bba92f943c082efd00c260136a31a942b26 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 19:10:46 +0100 Subject: [PATCH 295/296] Prioritize nonpublic alternates after rendezvous --- ...ZeroTierDataplaneRuntimeDirectPathTests.cs | 73 ++++++++++++++++++- ...oTierStickyHintMaintenanceSelectorTests.cs | 35 +++++++++ .../Internal/ZeroTierDataplaneRuntime.cs | 70 ++++++++++++++++-- .../ZeroTierStickyHintMaintenanceSelector.cs | 1 + docs/logbook/direct-p2p-jaeger.md | 37 ++++++++++ 5 files changed, 208 insertions(+), 8 deletions(-) diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 3e28592..eb569e1 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -491,7 +491,10 @@ await WaitForConditionAsync( .Distinct() .Order() .ToArray(); - Assert.Equal(new[] { 1 }, rendezvousBootstrapSockets); + if (rendezvousBootstrapSockets.Length != 0) + { + Assert.Equal(new[] { 1 }, rendezvousBootstrapSockets); + } } [Fact] @@ -979,6 +982,74 @@ await WaitForConditionAsync( Assert.Empty(maintenanceSends); } + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_UsesSingleSocketForNonPublicAlternateHintAfterPinnedRendezvous() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var alternateEndpoint = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, [rendezvousEndpoint, alternateEndpoint]); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var maintenanceSockets = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(alternateEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + if (maintenanceSockets.Length != 0) + { + Assert.Equal(new[] { 1 }, maintenanceSockets); + } + } + [Fact] public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotPeriodicallyForceHelloOnPinnedRendezvousSiblingHint() { diff --git a/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs index 942846a..8b1c12a 100644 --- a/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs +++ b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs @@ -49,4 +49,39 @@ public async Task SelectCandidates_PinnedRendezvous_RotatesAlternateSocketsAcros Assert.Contains(first, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); Assert.Contains(second, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); } + + [Fact] + public async Task SelectCandidates_PinnedRendezvous_PrefersNonPublicAlternateHint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + var peerNodeId = new NodeId(0x2222222222); + var endpointManager = new ZeroTierDirectEndpointManager( + udp, + new IPEndPoint(IPAddress.Loopback, 9993), + peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => endpointManager); + var selector = new ZeroTierStickyHintMaintenanceSelector(planner); + var pinnedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var publicAlternate = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var nonPublicAlternate = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + var hinted = new[] { pinnedEndpoint, publicAlternate, nonPublicAlternate }; + + var selected = selector.SelectCandidates( + peerNodeId, + hinted, + endpointBudget: 2, + restrictToPinnedRendezvous: true, + endpoint => endpoint.Equals(pinnedEndpoint), + endpoint => endpoint.Equals(publicAlternate) || endpoint.Equals(nonPublicAlternate), + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty(), + endpoint => endpoint.Equals(publicAlternate) || endpoint.Equals(nonPublicAlternate) ? new[] { 1 } : Array.Empty()); + + Assert.Contains(selected, candidate => candidate.RemoteEndPoint.Equals(pinnedEndpoint)); + Assert.Contains(selected, candidate => candidate.RemoteEndPoint.Equals(nonPublicAlternate)); + Assert.DoesNotContain(selected, candidate => candidate.RemoteEndPoint.Equals(publicAlternate)); + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index d8dd355..70f8172 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1446,7 +1446,7 @@ private async ValueTask HandleDirectEndpointHintAsync( } else if (ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint)) { - useAllEligibleLocalSockets = !ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint); + useAllEligibleLocalSockets = !ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint); } else { @@ -1554,6 +1554,15 @@ private int[] GetDirectHintBootstrapSocketIds( return _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); } + if (ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + { + var alternateSocketIds = GetAlternatePinnedRendezvousSocketIds(peerNodeId, endpoint, receivedLocalSocketId); + if (alternateSocketIds.Length != 0) + { + return alternateSocketIds; + } + } + if (receivedLocalSocketId >= 0) { return [receivedLocalSocketId]; @@ -1931,12 +1940,12 @@ private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId pe private int[] GetStickyDirectHintProbeSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) { - if (ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + if (ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) { - var pinnedSocketIds = GetPinnedRendezvousStickySocketIds(peerNodeId); - if (pinnedSocketIds.Length != 0) + var alternateSocketIds = GetAlternatePinnedRendezvousSocketIds(peerNodeId, endpoint); + if (alternateSocketIds.Length != 0) { - return pinnedSocketIds; + return alternateSocketIds; } } @@ -2002,6 +2011,53 @@ private int[] GetPinnedRendezvousStickySocketIds(NodeId peerNodeId) return []; } + private int[] GetAlternatePinnedRendezvousSocketIds(NodeId peerNodeId, IPEndPoint endpoint, int fallbackLocalSocketId = -1) + { + if (ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + { + var pinnedSocketIds = GetPinnedRendezvousStickySocketIds(peerNodeId); + if (pinnedSocketIds.Length != 0) + { + return pinnedSocketIds; + } + } + + if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + if (fallbackLocalSocketId >= 0) + { + return [fallbackLocalSocketId]; + } + + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var observedRootSocketId)) + { + return [observedRootSocketId]; + } + } + + var socketIds = GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + if (socketIds.Length != 0) + { + return socketIds; + } + + if (fallbackLocalSocketId >= 0) + { + return [fallbackLocalSocketId]; + } + + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) + { + return [rootSocketId]; + } + + return []; + } + + private bool ShouldUseSingleSocketForAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) + => ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) && + ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint); + private bool ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) { if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) || @@ -2486,8 +2542,8 @@ private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peer GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint, endpoint => ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint), endpoint => GetStickyHintedLocalSocketIds(peerNodeId, endpoint), - endpoint => ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint) - ? GetPinnedRendezvousStickySocketIds(peerNodeId) + endpoint => ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint) + ? GetAlternatePinnedRendezvousSocketIds(peerNodeId, endpoint) : _directHintPlanner.GetRotatingSocketIds( peerNodeId, endpoint, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs index a4326ac..1c7a1dc 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs @@ -58,6 +58,7 @@ public ZeroTierSelectedPeerPath[] SelectCandidates( .Where(endpoint => !isPinnedRendezvousEndpoint(endpoint) && shouldAllowAlternatePinnedRendezvousHint(endpoint)) + .OrderBy(static endpoint => ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint) ? 1 : 0) .ToArray(); if (alternates.Length == 0) { diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index cc53d80..13c17b4 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -945,3 +945,40 @@ New notes are appended with timestamps. Older entries are not rewritten. - public rendezvous plus public same-IP sibling probes now both start same-socket and `ECHO`-first - the peer still never returns a confirmed hop-0 packet - the remaining broad fanout is now on non-public alternate hints, not on the root-introduced public path + +## 2026-03-16T19:10:13.9849570+01:00 +- Change: + - non-public alternate hints after pinned rendezvous now use one constrained socket instead of `allSockets=True` + - non-public alternate maintenance is now prioritized ahead of public same-IP alternates once payload is already pinned to the public rendezvous path + - when a constrained non-public alternate has no better sticky path yet, it now falls back to the observed root/rendezvous socket +- Why it was tried: + - the previous live trace changed the issue again: + - public rendezvous and public same-IP siblings were already same-socket and `ECHO`-first + - but non-public alternates were either still broad-fanout or got starved behind the public alternates during maintenance +- Verification: + - focused suites passed: + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDataplaneRuntimeDirectPathTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-nonpublic-priority.log`: + - non-public alternates are now revisited after rendezvous and are same-socket: + - `TX ECHO bootstrap to 100.85.196.109:9993 (socket=7)` + - `TX ECHO bootstrap to 100.85.196.109:49161 (socket=7)` + - `TX ECHO bootstrap to 100.85.196.109:21376 (socket=7)` + - `TX ECHO bootstrap to 172.17.0.1:9993 (socket=7)` + - `TX ECHO bootstrap to 172.18.0.1:9993 (socket=7)` + - `TX ECHO bootstrap to 10.22.10.94:9993 (socket=7)` + - `TX ECHO bootstrap to 172.17.0.1:49161 (socket=7)` + - `TX ECHO bootstrap to 172.18.0.1:49161 (socket=7)` + - `TX ECHO bootstrap to 10.22.10.94:49161 (socket=7)` + - `TX ECHO bootstrap to 172.17.0.1:21376 (socket=7)` + - `TX ECHO bootstrap to 172.18.0.1:21376 (socket=7)` + - despite that, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the non-public alternate fanout/starvation issue is fixed + - the remaining blocker narrowed again: + - public rendezvous, public same-IP siblings, and non-public alternates are all now same-socket and `ECHO`-first + - the peer still never returns a confirmed hop-0 packet even after all of those local transport-side corrections From e4854fdf3a77c48adf0e650656495afaf14aacf2 Mon Sep 17 00:00:00 2001 From: Jonas Kamsker <11245306+JKamsker@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:15:22 +0100 Subject: [PATCH 296/296] Log stock ZeroTier path analysis --- docs/logbook/direct-p2p-jaeger.md | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md index 13c17b4..8c3b608 100644 --- a/docs/logbook/direct-p2p-jaeger.md +++ b/docs/logbook/direct-p2p-jaeger.md @@ -982,3 +982,50 @@ New notes are appended with timestamps. Older entries are not rewritten. - the remaining blocker narrowed again: - public rendezvous, public same-IP siblings, and non-public alternates are all now same-socket and `ECHO`-first - the peer still never returns a confirmed hop-0 packet even after all of those local transport-side corrections + +## 2026-03-16T20:14:37.1731390+01:00 +- Change: + - performed read-only analysis against the production Jaeger host using stock `zerotier-cli` and non-mutating `ssh` commands + - no config or service changes were made locally or remotely +- Why it was tried: + - the managed runtime still fails to form hop-0 P2P to `jaeger.pdcs.kamsker.at` + - the user confirmed that stock ZeroTier is now directly connected, so the next question was whether the remote host and stock path reveal why the managed stack still fails +- Verification: + - local stock ZeroTier: + - `zerotier-cli info` + - `zerotier-cli listnetworks` + - `zerotier-cli listpeers` + - remote read-only commands over `ssh jaeger.pdcs.kamsker.at`: + - `sudo -n zerotier-cli info` + - `sudo -n zerotier-cli listnetworks` + - `sudo -n zerotier-cli listpeers` + - `ip -br addr` + - `ip route` + - `ip route get 10.121.15.246` + - `ss -tln '( sport = :443 )'` +- Live result: + - local stock node is `b621d170ad` on `9ad07d01093a69e3` with ZeroTier IP `10.121.15.246` + - remote Jaeger host is: + - hostname `dev-build-server` + - ZeroTier node `9e072772f9` + - ZeroTier version `1.16.0` + - ZeroTier interface `zttpmuo2hk` + - ZeroTier IP `10.121.15.120/24` + - stock path is confirmed direct on both ends: + - local `listpeers` shows `9e072772f9 176.66.90.119/46827 ... LEAF` + - remote `listpeers` shows `b621d170ad 212.241.85.84/49072 ... LEAF` + - remote interface inventory explains the alternate hints seen in managed-stack traces: + - LAN `10.22.10.94/24` + - ZeroTier `10.121.15.120/24` + - Tailscale `100.85.196.109/32` + - Docker bridges `172.17.0.1/16` and `172.18.0.1/16` + - remote routing confirms `10.121.15.246` is reached over `zttpmuo2hk` + - remote HTTPS listener is bound on `0.0.0.0:443` and `[::]:443` +- Issue update: + - stock ZeroTier proves that this peer pair can form a real direct path on the current networks: + - local public tuple seen remotely: `212.241.85.84:49072` + - remote public tuple seen locally: `176.66.90.119:46827` + - the remaining blocker is therefore inside the managed runtime, not the environment: + - the remote host is reachable directly + - the host’s extra LAN/Tailscale/Docker interfaces explain the non-public hints that kept appearing in traces + - the managed stack still needs to match stock behavior for path introduction, surface authority, reply correlation, or ingress handling