diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..82984b8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,8 @@ + +# ZTSharp +ZTSharp is a fully managed c# library that implements the Zerotier protocol without relying on unmanaged code or external dependencies. + +# Rules +- Do not use or reuse official Zerotier services, connections or identities from the current host. The library **must** be self-contained and not rely on any existing Zerotier configuration or services on the host machine. +- The source code must be written in clean, modular, easy to understand and maintainable style. It should follow best practices in c#. +- The source code must be high-performant and efficient, minimizing resource usage and maximizing speed (Eg use ArrayPool, Span, etc... instead of creating new arrays or lists all of the time (especially hot paths)). \ No newline at end of file diff --git a/ZTSharp.Tests/AtomicFileTests.cs b/ZTSharp.Tests/AtomicFileTests.cs index 02421aa..8f1b179 100644 --- a/ZTSharp.Tests/AtomicFileTests.cs +++ b/ZTSharp.Tests/AtomicFileTests.cs @@ -10,16 +10,23 @@ public async Task WriteAllBytesAsync_Throws_WhenAtomicMoveNeverSucceeds() var root = TestTempPaths.CreateGuidSuffixed("zt-atomic-file-"); Directory.CreateDirectory(root); - var destination = Path.Combine(root, "dest"); - Directory.CreateDirectory(destination); - - var ex = await Assert.ThrowsAsync(async () => + try { - await AtomicFile.WriteAllBytesAsync(destination, new byte[] { 1, 2, 3 }, CancellationToken.None); - }); + var destination = Path.Combine(root, "dest"); + Directory.CreateDirectory(destination); + + var ex = await Assert.ThrowsAsync(async () => + { + await AtomicFile.WriteAllBytesAsync(destination, new byte[] { 1, 2, 3 }, CancellationToken.None); + }); - Assert.Contains("Atomic replace failed", ex.Message, StringComparison.OrdinalIgnoreCase); - Assert.True(Directory.Exists(destination)); + Assert.Contains("Atomic replace failed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.True(Directory.Exists(destination)); + } + finally + { + Directory.Delete(root, recursive: true); + } } } diff --git a/ZTSharp.Tests/BoundedFileIOBomTests.cs b/ZTSharp.Tests/BoundedFileIOBomTests.cs new file mode 100644 index 0000000..8267c0d --- /dev/null +++ b/ZTSharp.Tests/BoundedFileIOBomTests.cs @@ -0,0 +1,29 @@ +using System.Text; +using ZTSharp.Internal; + +namespace ZTSharp.Tests; + +public sealed class BoundedFileIOBomTests +{ + [Fact] + public void TryReadAllText_StripsUtf8Bom() + { + var path = TestTempPaths.CreateGuidSuffixed("bounded-io-bom-"); + try + { + var payload = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes("hello")).ToArray(); + File.WriteAllBytes(path, payload); + + Assert.True(BoundedFileIO.TryReadAllText(path, maxBytes: 1024, Encoding.UTF8, out var text)); + Assert.Equal("hello", text); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} + diff --git a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs index 2aee83c..c850ef3 100644 --- a/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs +++ b/ZTSharp.Tests/ChannelWriterConcurrencyTests.cs @@ -34,9 +34,9 @@ public async Task InMemoryZtUdpClient_ReceivesDatagrams_UnderConcurrentSends() sendTasks[i] = sender.SendToAsync(payload, receiverId, remotePort: 30000); } - await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(2)); + await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(10)); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var seen = new bool[count]; for (var i = 0; i < count; i++) { @@ -68,7 +68,7 @@ public async Task InMemoryOverlayTcpListener_AcceptsMultipleConcurrentClients() await using var listener = new OverlayTcpListener(serverNode, networkId, serverPort); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var acceptTask = Task.Run(async () => { var accepted = new List(capacity: count); @@ -94,8 +94,8 @@ public async Task InMemoryOverlayTcpListener_AcceptsMultipleConcurrentClients() }, cts.Token); } - await Task.WhenAll(connectTasks).WaitAsync(TimeSpan.FromSeconds(5), cts.Token); - var acceptedClients = await acceptTask.WaitAsync(TimeSpan.FromSeconds(5), cts.Token); + await Task.WhenAll(connectTasks).WaitAsync(TimeSpan.FromSeconds(10), cts.Token); + var acceptedClients = await acceptTask.WaitAsync(TimeSpan.FromSeconds(10), cts.Token); Assert.Equal(count, acceptedClients.Count); foreach (var accepted in acceptedClients) @@ -158,9 +158,9 @@ void CaptureSyn(in RawFrame frame) await using var client = new OverlayTcpClient(clientNode, networkId, clientPort); await client.ConnectAsync(serverNode.NodeId.Value, serverPort); - await using var _ = await acceptTask.WaitAsync(TimeSpan.FromSeconds(2)); + await using var _ = await acceptTask.WaitAsync(TimeSpan.FromSeconds(10)); - var (connectionId, sourcePort) = await synInfoTcs.Task.WaitAsync(TimeSpan.FromSeconds(2)); + var (connectionId, sourcePort) = await synInfoTcs.Task.WaitAsync(TimeSpan.FromSeconds(10)); const int frames = 500; var payload = Encoding.ASCII.GetBytes("abcdefgh"); @@ -168,26 +168,23 @@ void CaptureSyn(in RawFrame frame) var sendTasks = new Task[frames]; for (var i = 0; i < frames; i++) { - sendTasks[i] = Task.Run(async () => - { - var dataFrame = new byte[OverlayTcpFrameCodec.HeaderLength + payload.Length]; - OverlayTcpFrameCodec.BuildHeader( - OverlayTcpFrameCodec.FrameType.Data, - sourcePort: serverPort, - destinationPort: clientPort, - destinationNodeId: clientNode.NodeId.Value, - connectionId, - dataFrame); - payload.CopyTo(dataFrame.AsSpan(OverlayTcpFrameCodec.HeaderLength)); - await serverNode.SendFrameAsync(networkId, dataFrame); - }); + var dataFrame = new byte[OverlayTcpFrameCodec.HeaderLength + payload.Length]; + OverlayTcpFrameCodec.BuildHeader( + OverlayTcpFrameCodec.FrameType.Data, + sourcePort: serverPort, + destinationPort: clientPort, + destinationNodeId: clientNode.NodeId.Value, + connectionId, + dataFrame); + payload.CopyTo(dataFrame.AsSpan(OverlayTcpFrameCodec.HeaderLength)); + sendTasks[i] = serverNode.SendFrameAsync(networkId, dataFrame); } - await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(2)); + await Task.WhenAll(sendTasks).WaitAsync(TimeSpan.FromSeconds(20)); var clientStream = client.GetStream(); var buffer = new byte[frames * payload.Length]; - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); var read = await StreamTestHelpers.ReadExactAsync(clientStream, buffer, buffer.Length, cts.Token); Assert.Equal(buffer.Length, read); } diff --git a/ZTSharp.Tests/DirectBootstrapProbeCursorTests.cs b/ZTSharp.Tests/DirectBootstrapProbeCursorTests.cs new file mode 100644 index 0000000..31f8983 --- /dev/null +++ b/ZTSharp.Tests/DirectBootstrapProbeCursorTests.cs @@ -0,0 +1,37 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class DirectBootstrapProbeCursorTests +{ + [Fact] + public void TakeNextEndpoints_RotatesAcrossCandidates() + { + var cursor = new DirectBootstrapProbeCursor(); + var endpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 59827) + }; + + var first = cursor.TakeNextEndpoints(endpoints, budget: 2); + var second = cursor.TakeNextEndpoints(endpoints, budget: 2); + + Assert.Equal(new[] { endpoints[0], endpoints[1] }, first); + Assert.Equal(new[] { endpoints[2], endpoints[0] }, second); + } + + [Fact] + public void TakeNextSocketIndex_RotatesPerEndpoint() + { + var cursor = new DirectBootstrapProbeCursor(); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + + Assert.Equal(0, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + Assert.Equal(1, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + Assert.Equal(2, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + Assert.Equal(0, cursor.TakeNextSocketIndex(endpoint, socketCount: 3)); + } +} diff --git a/ZTSharp.Tests/FileStateStoreSecurityTests.cs b/ZTSharp.Tests/FileStateStoreSecurityTests.cs index 3ad7aaa..58ba976 100644 --- a/ZTSharp.Tests/FileStateStoreSecurityTests.cs +++ b/ZTSharp.Tests/FileStateStoreSecurityTests.cs @@ -1,17 +1,13 @@ using System.Diagnostics; -using System.Net; namespace ZTSharp.Tests; public sealed class FileStateStoreSecurityTests { - [Fact] + [SkippableFact] public async Task ReadAsync_Throws_WhenPathTraversesJunction() { - if (!OperatingSystem.IsWindows()) - { - return; - } + Skip.IfNot(OperatingSystem.IsWindows(), "Junction traversal tests require Windows."); var root = TestTempPaths.CreateGuidSuffixed("zt-state-root-"); Directory.CreateDirectory(root); @@ -23,10 +19,7 @@ public async Task ReadAsync_Throws_WhenPathTraversesJunction() await File.WriteAllBytesAsync(secretPath, new byte[] { 1, 2, 3 }); var junction = Path.Combine(root, "escape"); - if (!TryCreateJunction(junction, outside)) - { - return; - } + Skip.IfNot(TryCreateJunction(junction, outside), "Failed to create junction (insufficient privileges or mklink unavailable)."); var store = new FileStateStore(root); _ = await Assert.ThrowsAsync(async () => await store.ReadAsync("escape/secret.bin")); @@ -76,7 +69,19 @@ private static bool TryCreateJunction(string junctionPath, string targetPath) return false; } - process.WaitForExit(milliseconds: 5000); + if (!process.WaitForExit(milliseconds: 5000)) + { + try + { + process.Kill(entireProcessTree: true); + } + catch + { + } + + return false; + } + return process.ExitCode == 0 && Directory.Exists(junctionPath); } catch diff --git a/ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs b/ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs new file mode 100644 index 0000000..ca5ccba --- /dev/null +++ b/ZTSharp.Tests/Ipv6CodecAhHeaderTests.cs @@ -0,0 +1,32 @@ +using System.Net; +using ZTSharp.ZeroTier.Net; + +namespace ZTSharp.Tests; + +public sealed class Ipv6CodecAhHeaderTests +{ + [Fact] + public void TryParseTransportPayload_RejectsInvalidAhHeaderLength() + { + var src = IPAddress.Parse("2001:db8::1"); + var dst = IPAddress.Parse("2001:db8::2"); + + var payload = new byte[8 + 8]; + payload[0] = UdpCodec.ProtocolNumber; // next header + payload[1] = 0; // AH payload len (0 => 8 bytes total, invalid; must be >= 12 bytes) + + var packet = new byte[40 + payload.Length]; + packet[0] = 0x60; // version + packet[4] = (byte)(payload.Length >> 8); + packet[5] = (byte)(payload.Length & 0xFF); + packet[6] = 51; // AH + packet[7] = 64; // hop limit + + src.GetAddressBytes().CopyTo(packet, 8); + dst.GetAddressBytes().CopyTo(packet, 24); + payload.CopyTo(packet, 40); + + Assert.False(Ipv6Codec.TryParseTransportPayload(packet, out _, out _, out _, out _, out _)); + } +} + diff --git a/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs b/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs index b77e175..46d57f7 100644 --- a/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs +++ b/ZTSharp.Tests/NodeNetworkLeaveOrderingTests.cs @@ -6,6 +6,34 @@ namespace ZTSharp.Tests; public sealed class NodeNetworkLeaveOrderingTests { + [Fact] + public async Task JoinNetworkAsync_DuplicateJoin_IsIdempotentAndDoesNotLeakRegistrations() + { + var store = new MemoryStateStore(); + var transport = new RecordingJoinTransport(); + var events = new NodeEventStream(_ => { }, NullLogger.Instance); + var peers = new NodePeerService(store); + var service = new NodeNetworkService(store, transport, events, peers); + + var networkId = 0xCAFE5003UL; + + await service.JoinNetworkAsync( + networkId, + localNodeId: 1, + localEndpoint: null, + onFrameReceived: (_, _, _, _) => Task.CompletedTask, + CancellationToken.None); + + await service.JoinNetworkAsync( + networkId, + localNodeId: 1, + localEndpoint: null, + onFrameReceived: (_, _, _, _) => Task.CompletedTask, + CancellationToken.None); + + Assert.Equal(1, transport.JoinCallCount); + } + [Fact] public async Task LeaveNetworkAsync_FailedLeave_DoesNotLoseRegistration_AndRetryUnsubscribes() { @@ -46,6 +74,38 @@ await service.JoinNetworkAsync( Assert.Equal(2, received); } + [Fact] + public async Task LeaveNetworkAsync_ConcurrentCalls_InvokeTransportLeaveOnce() + { + var store = new MemoryStateStore(); + var transport = new BlockingLeaveTransport(); + var events = new NodeEventStream(_ => { }, NullLogger.Instance); + var peers = new NodePeerService(store); + var service = new NodeNetworkService(store, transport, events, peers); + + var networkId = 0xCAFE5002UL; + + await service.JoinNetworkAsync( + networkId, + localNodeId: 1, + localEndpoint: null, + onFrameReceived: (_, _, _, _) => Task.CompletedTask, + CancellationToken.None); + + var leave1 = service.LeaveNetworkAsync(networkId, CancellationToken.None); + await transport.LeaveEntered; + + var leave2 = service.LeaveNetworkAsync(networkId, CancellationToken.None); + + await Task.Delay(25); + Assert.Equal(1, transport.LeaveCallCount); + + transport.AllowLeaveToProceed(); + await Task.WhenAll(leave1, leave2); + + Assert.Equal(1, transport.LeaveCallCount); + } + private sealed class FlakyLeaveTransport : INodeTransport, IDisposable { public InMemoryNodeTransport Inner { get; } = new(); @@ -79,4 +139,75 @@ public Task FlushAsync(CancellationToken cancellationToken = default) public void Dispose() => Inner.Dispose(); } + + private sealed class BlockingLeaveTransport : INodeTransport, IDisposable + { + private readonly TaskCompletionSource _leaveEntered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _allowLeaveToProceed = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _leaveEnteredSet; + private int _leaveCallCount; + + public InMemoryNodeTransport Inner { get; } = new(); + + public int LeaveCallCount => Volatile.Read(ref _leaveCallCount); + + public Task LeaveEntered => _leaveEntered.Task; + + public void AllowLeaveToProceed() => _allowLeaveToProceed.TrySetResult(); + + public Task JoinNetworkAsync( + ulong networkId, + ulong nodeId, + Func, CancellationToken, Task> onFrameReceived, + System.Net.IPEndPoint? localEndpoint = null, + CancellationToken cancellationToken = default) + => Inner.JoinNetworkAsync(networkId, nodeId, onFrameReceived, localEndpoint, cancellationToken); + + public async Task LeaveNetworkAsync(ulong networkId, Guid registrationId, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _leaveCallCount); + if (Interlocked.Exchange(ref _leaveEnteredSet, 1) == 0) + { + _leaveEntered.TrySetResult(); + } + + await _allowLeaveToProceed.Task.ConfigureAwait(false); + await Inner.LeaveNetworkAsync(networkId, registrationId, cancellationToken).ConfigureAwait(false); + } + + public Task SendFrameAsync(ulong networkId, ulong sourceNodeId, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Inner.SendFrameAsync(networkId, sourceNodeId, payload, cancellationToken); + + public Task FlushAsync(CancellationToken cancellationToken = default) + => Inner.FlushAsync(cancellationToken); + + public void Dispose() => Inner.Dispose(); + } + + private sealed class RecordingJoinTransport : INodeTransport + { + private int _joinCallCount; + + public int JoinCallCount => Volatile.Read(ref _joinCallCount); + + public Task JoinNetworkAsync( + ulong networkId, + ulong nodeId, + Func, CancellationToken, Task> onFrameReceived, + System.Net.IPEndPoint? localEndpoint = null, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _joinCallCount); + return Task.FromResult(Guid.NewGuid()); + } + + public Task LeaveNetworkAsync(ulong networkId, Guid registrationId, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendFrameAsync(ulong networkId, ulong sourceNodeId, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task FlushAsync(CancellationToken cancellationToken = default) + => Task.CompletedTask; + } } diff --git a/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs b/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs index 6662f2d..f141c2e 100644 --- a/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs +++ b/ZTSharp.Tests/OsUdpPeerRegistryBoundsTests.cs @@ -32,7 +32,10 @@ public void DirectoryPeers_AreCappedPerNetwork() for (var i = 0; i < OsUdpPeerRegistry.DirectoryMaxPeersPerNetwork + 50; i++) { - registry.RegisterDiscoveredPeer(networkId, sourceNodeId: (ulong)(i + 1), new IPEndPoint(IPAddress.Loopback, 9999)); + _ = registry.RegisterLocalAndGetKnownPeers( + networkId, + localNodeId: (ulong)(i + 1), + advertisedEndpoint: new IPEndPoint(IPAddress.Loopback, 9999)); } Assert.True(OsUdpPeerRegistry.GetDirectoryPeerCountForTests(networkId) <= OsUdpPeerRegistry.DirectoryMaxPeersPerNetwork); @@ -53,11 +56,11 @@ public void DirectoryPeers_ExpireOnTtlSweep() var registry = new OsUdpPeerRegistry(enablePeerDiscovery: true, UdpEndpointNormalization.Normalize, timeProvider: time); var networkId = 0x5678UL; - registry.RegisterDiscoveredPeer(networkId, sourceNodeId: 1, new IPEndPoint(IPAddress.Loopback, 9999)); + _ = registry.RegisterLocalAndGetKnownPeers(networkId, localNodeId: 1, advertisedEndpoint: new IPEndPoint(IPAddress.Loopback, 9999)); Assert.Equal(1, OsUdpPeerRegistry.GetDirectoryPeerCountForTests(networkId)); time.Advance(OsUdpPeerRegistry.DirectoryPeerTtl + TimeSpan.FromSeconds(1)); - registry.Cleanup(); + _ = registry.RegisterLocalAndGetKnownPeers(networkId: networkId + 1, localNodeId: 2, advertisedEndpoint: new IPEndPoint(IPAddress.Loopback, 9998)); Assert.Equal(0, OsUdpPeerRegistry.GetDirectoryPeerCountForTests(networkId)); } diff --git a/ZTSharp.Tests/OsUdpPeerRegistryLastSeenTests.cs b/ZTSharp.Tests/OsUdpPeerRegistryLastSeenTests.cs new file mode 100644 index 0000000..7fcd2cf --- /dev/null +++ b/ZTSharp.Tests/OsUdpPeerRegistryLastSeenTests.cs @@ -0,0 +1,46 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.Tests; + +[Collection("OsUdpPeerRegistry")] +public sealed class OsUdpPeerRegistryLastSeenTests +{ + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public ManualTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta); + } + + [Fact] + public void RefreshPeerLastSeen_KeepsActivePeerFromExpiring() + { + var time = new ManualTimeProvider(DateTimeOffset.UnixEpoch); + var registry = new OsUdpPeerRegistry(enablePeerDiscovery: false, UdpEndpointNormalization.Normalize, timeProvider: time); + var networkId = 0x1111UL; + var peerNodeId = 0x2222UL; + var endpoint = new IPEndPoint(IPAddress.Loopback, 9999); + + registry.AddOrUpdatePeer(networkId, peerNodeId, endpoint); + + time.Advance(TimeSpan.FromMinutes(4)); + registry.RefreshPeerLastSeen(networkId, peerNodeId); + + time.Advance(TimeSpan.FromMinutes(2)); + Assert.True(registry.TryGetPeers(networkId, out var peers)); + Assert.True(peers.ContainsKey(peerNodeId)); + + time.Advance(TimeSpan.FromMinutes(10)); + Assert.True(registry.TryGetPeers(networkId, out peers)); + Assert.False(peers.ContainsKey(peerNodeId)); + } +} + diff --git a/ZTSharp.Tests/OsUdpPeerRegistryNetworkTrimTests.cs b/ZTSharp.Tests/OsUdpPeerRegistryNetworkTrimTests.cs new file mode 100644 index 0000000..f06aa45 --- /dev/null +++ b/ZTSharp.Tests/OsUdpPeerRegistryNetworkTrimTests.cs @@ -0,0 +1,44 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.Tests; + +[Collection("OsUdpPeerRegistry")] +public sealed class OsUdpPeerRegistryNetworkTrimTests +{ + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public ManualTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta); + } + + [Fact] + public void SweepNetworkPeers_DoesNotEvictNetworks_WithLocalRegistration() + { + var time = new ManualTimeProvider(DateTimeOffset.UnixEpoch); + var registry = new OsUdpPeerRegistry(enablePeerDiscovery: false, UdpEndpointNormalization.Normalize, timeProvider: time); + + const int extraNetworks = 16; + var totalNetworks = 256 + extraNetworks; + + for (var i = 0; i < totalNetworks; i++) + { + var networkId = (ulong)(0x9000 + i); + registry.SetLocalNodeId(networkId, nodeId: 1); + registry.AddOrUpdatePeer(networkId, nodeId: (ulong)(0xA000 + i), endpoint: new IPEndPoint(IPAddress.Loopback, 10000 + i)); + time.Advance(TimeSpan.FromSeconds(1)); + } + + var firstNetworkId = 0x9000UL; + Assert.True(registry.TryGetPeers(firstNetworkId, out _)); + } +} + diff --git a/ZTSharp.Tests/OsUdpReceiveLoopLastSeenRefreshTests.cs b/ZTSharp.Tests/OsUdpReceiveLoopLastSeenRefreshTests.cs new file mode 100644 index 0000000..880bcec --- /dev/null +++ b/ZTSharp.Tests/OsUdpReceiveLoopLastSeenRefreshTests.cs @@ -0,0 +1,82 @@ +using System.Net; +using System.Net.Sockets; +using ZTSharp.Transport; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.Tests; + +public sealed class OsUdpReceiveLoopLastSeenRefreshTests +{ + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public ManualTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta); + } + + [Fact] + public async Task ReceiveLoop_RefreshesPeerLastSeen_EvenWhenDispatchThrows() + { + using var udp = new UdpClient(AddressFamily.InterNetwork); + + var time = new ManualTimeProvider(DateTimeOffset.UnixEpoch); + var peers = new OsUdpPeerRegistry(enablePeerDiscovery: false, UdpEndpointNormalization.Normalize, timeProvider: time); + var networkId = 0xABCDEF10UL; + var sourceNodeId = 0x1234UL; + var remoteEndpoint = new IPEndPoint(IPAddress.Loopback, 40100); + peers.AddOrUpdatePeer(networkId, sourceNodeId, remoteEndpoint); + + time.Advance(TimeSpan.FromMinutes(4)); + var appPayload = new byte[] { 0x42 }; + var frame = NodeFrameCodec.Encode(networkId, sourceNodeId, appPayload); + var result = new UdpReceiveResult(frame.ToArray(), remoteEndpoint); + + var dispatched = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task DispatchAsync(ulong _, ulong __, ReadOnlyMemory ___, CancellationToken ____) + { + dispatched.TrySetResult(true); + throw new InvalidOperationException("synthetic dispatch fault"); + } + + static Task SendDiscoveryAsync(ulong _, ulong __, IPEndPoint ___, OsUdpPeerDiscoveryProtocol.FrameType ____, CancellationToken _____) + => Task.CompletedTask; + + var callCount = 0; + async ValueTask ReceiveAsync(CancellationToken ct) + { + if (Interlocked.Increment(ref callCount) == 1) + { + return result; + } + + await Task.Delay(Timeout.InfiniteTimeSpan, ct); + throw new OperationCanceledException(ct); + } + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var loop = new OsUdpReceiveLoop( + udp, + enablePeerDiscovery: false, + peers, + DispatchAsync, + SendDiscoveryAsync, + receiveAsync: ReceiveAsync); + + var run = loop.RunAsync(cts.Token); + _ = await dispatched.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + time.Advance(TimeSpan.FromMinutes(2)); + Assert.True(peers.TryGetPeers(networkId, out var networkPeers)); + Assert.True(networkPeers.ContainsKey(sourceNodeId)); + + cts.Cancel(); + await run; + } +} diff --git a/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs b/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs index 1cb33cc..ed3c6fe 100644 --- a/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs +++ b/ZTSharp.Tests/OsUdpReceiveLoopSocketExceptionTests.cs @@ -80,6 +80,7 @@ public async Task OsUdpReceiveLoop_DoesNotBlockOnDiscoveryReplySends() var peers = new OsUdpPeerRegistry(enablePeerDiscovery: true, UdpEndpointNormalization.Normalize); peers.SetLocalNodeId(networkId, localNodeId); + peers.AddOrUpdatePeer(networkId, remoteNodeId, remoteEndpoint); var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var allowSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs index f837c2a..ba16ce4 100644 --- a/ZTSharp.Tests/OsUdpSocketFactoryTests.cs +++ b/ZTSharp.Tests/OsUdpSocketFactoryTests.cs @@ -1,17 +1,15 @@ using System.Net.Sockets; +using System.Reflection; using ZTSharp.Transport.Internal; namespace ZTSharp.Tests; public sealed class OsUdpSocketFactoryTests { - [Fact] + [SkippableFact] public void WindowsSioUdpConnResetInput_IsDword() { - if (!OperatingSystem.IsWindows()) - { - return; - } + Skip.IfNot(OperatingSystem.IsWindows(), "Windows-only IOControl buffer test."); var buffer = OsUdpSocketFactory.CreateWindowsSioUdpConnResetInputBuffer(disableConnReset: true); Assert.Equal(4, buffer.Length); @@ -19,7 +17,7 @@ public void WindowsSioUdpConnResetInput_IsDword() } [Fact] - public void CreateSocketCore_WhenDualModeFails_TriesIpv6OnlyBeforeIpv4() + public void CreateSocketCore_WhenDualModeFails_TriesIpv4BeforeIpv6Only() { var calls = new List(); @@ -36,13 +34,48 @@ public void CreateSocketCore_WhenDualModeFails_TriesIpv6OnlyBeforeIpv4() try { - Assert.Equal(new[] { "dual", "v6only" }, calls); - Assert.Equal(AddressFamily.InterNetworkV6, socket.Client.AddressFamily); + Assert.Equal(new[] { "dual", "v4" }, calls); + Assert.Equal(AddressFamily.InterNetwork, socket.Client.AddressFamily); } finally { socket.Dispose(); } } + + [SkippableFact] + public void CreateUdp6OnlyBound_SetsDualModeFalse() + { + Skip.IfNot(Socket.OSSupportsIPv6, "IPv6 not supported on this platform."); + + var method = typeof(OsUdpSocketFactory).GetMethod( + "CreateUdp6OnlyBound", + BindingFlags.NonPublic | BindingFlags.Static, + binder: null, + types: [typeof(int)], + modifiers: null); + Assert.NotNull(method); + + UdpClient? udp; + try + { + udp = (UdpClient?)method!.Invoke(null, new object[] { 0 }); + } + catch (TargetInvocationException ex) when (ex.InnerException is SocketException or PlatformNotSupportedException or NotSupportedException) + { + Skip.If(true, $"IPv6 appears supported, but binding an IPv6 UDP socket failed: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"); + return; + } + Assert.NotNull(udp); + + try + { + Assert.False(udp!.Client.DualMode); + } + finally + { + udp!.Dispose(); + } + } } diff --git a/ZTSharp.Tests/OsUdpSpoofingTests.cs b/ZTSharp.Tests/OsUdpSpoofingTests.cs index 721fa9f..e423230 100644 --- a/ZTSharp.Tests/OsUdpSpoofingTests.cs +++ b/ZTSharp.Tests/OsUdpSpoofingTests.cs @@ -1,4 +1,5 @@ using System.Net.Sockets; +using System.Net; using ZTSharp.Sockets; using ZTSharp.Transport; @@ -15,14 +16,16 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForUdpHandlers() { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await using var node2 = new Node(new NodeOptions { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await node1.StartAsync(); @@ -35,6 +38,7 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForUdpHandlers() var node2Endpoint = node2.LocalTransportEndpoint; Assert.NotNull(node2Endpoint); + var transport2 = Assert.IsType(node2.TransportForTests); await using var udp2 = new ZtUdpClient(node2, networkId, localPort: 12002); var datagram = "spoof"u8.ToArray(); @@ -51,9 +55,26 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForUdpHandlers() var frameBuffer = new byte[NodeFrameCodec.GetEncodedLength(udpPayload.Length)]; Assert.True(NodeFrameCodec.TryEncode(networkId, node1Id, udpPayload, frameBuffer, out var frameLength)); - using (var spoofUdp = CreateSpoofUdp(node2Endpoint!)) + using var spoofUdp = CreateSpoofUdp(node2Endpoint!); + var spoofLocalEndpoint = (IPEndPoint)spoofUdp.Client.LocalEndPoint!; + var datagramSeen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport2.SetDatagramObserverForTests(remote => { - await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + if (remote.Equals(spoofLocalEndpoint)) + { + datagramSeen.TrySetResult(true); + } + }); + + try + { + var sent = await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + Assert.Equal(frameLength, sent); + _ = await datagramSeen.Task.WaitAsync(TimeSpan.FromSeconds(1)); + } + finally + { + transport2.SetDatagramObserverForTests(null); } using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(250)); @@ -69,14 +90,16 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForOverlayTcpListeners { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await using var node2 = new Node(new NodeOptions { StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-node-"), StateStore = new MemoryStateStore(), - TransportMode = TransportMode.OsUdp + TransportMode = TransportMode.OsUdp, + EnableIpv6 = false }); await node1.StartAsync(); @@ -104,9 +127,27 @@ public async Task OsUdpTransport_DropsSpoofedSourceNodeId_ForOverlayTcpListeners var node2Endpoint = node2.LocalTransportEndpoint; Assert.NotNull(node2Endpoint); - using (var spoofUdp = CreateSpoofUdp(node2Endpoint!)) + var transport2 = Assert.IsType(node2.TransportForTests); + using var spoofUdp = CreateSpoofUdp(node2Endpoint!); + var spoofLocalEndpoint = (IPEndPoint)spoofUdp.Client.LocalEndPoint!; + var datagramSeen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport2.SetDatagramObserverForTests(remote => + { + if (remote.Equals(spoofLocalEndpoint)) + { + datagramSeen.TrySetResult(true); + } + }); + + try + { + var sent = await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + Assert.Equal(frameLength, sent); + _ = await datagramSeen.Task.WaitAsync(TimeSpan.FromSeconds(1)); + } + finally { - await spoofUdp.SendAsync(frameBuffer.AsMemory(0, frameLength), node2Endpoint!); + transport2.SetDatagramObserverForTests(null); } using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(250)); @@ -124,7 +165,10 @@ private static UdpClient CreateSpoofUdp(System.Net.IPEndPoint destination) } var v4 = new UdpClient(System.Net.Sockets.AddressFamily.InterNetwork); - v4.Client.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Any, 0)); + var spoofAddress = destination.Address.Equals(IPAddress.Parse("127.0.0.2")) + ? IPAddress.Parse("127.0.0.3") + : IPAddress.Parse("127.0.0.2"); + v4.Client.Bind(new System.Net.IPEndPoint(spoofAddress, 0)); return v4; } } diff --git a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs index e0c173e..af6de4b 100644 --- a/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs +++ b/ZTSharp.Tests/OverlayTcpIncomingBufferTests.cs @@ -20,4 +20,56 @@ public async Task WhenBufferOverflows_ReadThrowsIOException() await Assert.ThrowsAsync(() => incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask()); } + + [Fact] + public async Task FinGrace_AllowsLateDataFramesToBeRead() + { + var incoming = new OverlayTcpIncomingBuffer(); + incoming.MarkRemoteFinReceived(); + + var payload = new byte[] { 1, 2, 3 }; + Assert.True(incoming.TryWrite(payload)); + + var buffer = new byte[3]; + var read = await incoming.ReadAsync(buffer, CancellationToken.None); + Assert.Equal(3, read); + Assert.Equal(payload, buffer); + + var eof = await incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(1)); + Assert.Equal(0, eof); + } + + [Fact] + public async Task FinGrace_AfterSegmentConsumed_SetsRemoteClosed_WhenNoLateDataArrives() + { + var incoming = new OverlayTcpIncomingBuffer(); + + var payload = new byte[] { 1, 2, 3 }; + Assert.True(incoming.TryWrite(payload)); + + var buffer = new byte[3]; + var read = await incoming.ReadAsync(buffer, CancellationToken.None); + Assert.Equal(3, read); + Assert.Equal(payload, buffer); + + incoming.MarkRemoteFinReceived(); + + var eof = await incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(1)); + Assert.Equal(0, eof); + Assert.True(incoming.RemoteClosed); + } + + [Fact] + public async Task FinArrival_UnblocksReaderAwaitingReadAsync() + { + var incoming = new OverlayTcpIncomingBuffer(); + + var readTask = incoming.ReadAsync(new byte[1], CancellationToken.None).AsTask(); + + await Task.Yield(); + incoming.MarkRemoteFinReceived(); + + var eof = await readTask.WaitAsync(TimeSpan.FromSeconds(1)); + Assert.Equal(0, eof); + } } diff --git a/ZTSharp.Tests/ScriptedUdpTransport.cs b/ZTSharp.Tests/ScriptedUdpTransport.cs new file mode 100644 index 0000000..59ed3de --- /dev/null +++ b/ZTSharp.Tests/ScriptedUdpTransport.cs @@ -0,0 +1,79 @@ +using System.Net; +using System.Threading.Channels; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +internal sealed class ScriptedUdpTransport : IZeroTierUdpTransport +{ + private readonly Channel _receiveQueue = Channel.CreateUnbounded(); + private readonly List _sends = new(); + private readonly object _lock = new(); + + public ScriptedUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) + { + LocalSockets = localSockets; + } + + public IReadOnlyList LocalSockets { get; } + + public RecordedUdpSend[] GetSendsSnapshot() + { + lock (_lock) + { + return _sends.ToArray(); + } + } + + public void EnqueueInbound(int localSocketId, IPEndPoint remoteEndPoint, ReadOnlyMemory payload) + => _receiveQueue.Writer.TryWrite(new ZeroTierUdpDatagram(localSocketId, remoteEndPoint, payload.ToArray())); + + public ValueTask DisposeAsync() + { + _receiveQueue.Writer.TryComplete(); + return ValueTask.CompletedTask; + } + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => _receiveQueue.Reader.ReadAsync(cancellationToken); + + public async ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout); + return await _receiveQueue.Reader.ReadAsync(timeoutCts.Token); + } + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + RecordSend(localSocketId: 0, remoteEndpoint, payload, hopLimit: null); + return Task.CompletedTask; + } + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + RecordSend(localSocketId, remoteEndpoint, payload, hopLimit: null); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + RecordSend(localSocketId, remoteEndpoint, payload, hopLimit); + return Task.CompletedTask; + } + + private void RecordSend(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int? hopLimit) + { + lock (_lock) + { + _sends.Add(new RecordedUdpSend(localSocketId, remoteEndpoint, hopLimit, payload.ToArray())); + } + } +} + +internal readonly record struct RecordedUdpSend(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit, byte[] Payload); diff --git a/ZTSharp.Tests/SecretFilePermissionTests.cs b/ZTSharp.Tests/SecretFilePermissionTests.cs index 5fc3841..37240b7 100644 --- a/ZTSharp.Tests/SecretFilePermissionTests.cs +++ b/ZTSharp.Tests/SecretFilePermissionTests.cs @@ -4,12 +4,12 @@ namespace ZTSharp.Tests; public sealed class SecretFilePermissionTests { - [Fact] + [UnixFact] public void ZeroTierIdentityStore_Save_SetsUnixMode600_OnUnix() { if (OperatingSystem.IsWindows()) { - return; + throw new InvalidOperationException("UnixFact should have skipped this test on Windows."); } var root = TestTempPaths.CreateGuidSuffixed("zt-secret-perms-"); diff --git a/ZTSharp.Tests/StateStoreKeyNormalizationSecurityTests.cs b/ZTSharp.Tests/StateStoreKeyNormalizationSecurityTests.cs new file mode 100644 index 0000000..6aa3be7 --- /dev/null +++ b/ZTSharp.Tests/StateStoreKeyNormalizationSecurityTests.cs @@ -0,0 +1,17 @@ +namespace ZTSharp.Tests; + +public sealed class StateStoreKeyNormalizationSecurityTests +{ + [Theory] + [InlineData("file:stream")] + [InlineData("C:\\Windows\\System32")] + [InlineData("C:Windows\\System32")] + [InlineData("\\\\?\\C:\\Windows\\System32")] + [InlineData("\\\\server\\share\\file")] + [InlineData("/absolute/path")] + [InlineData("\\absolute\\path")] + public void NormalizeKey_Rejects_Rooted_Or_Ads_Like_Keys(string key) + { + _ = Assert.Throws(() => StateStoreKeyNormalization.NormalizeKey(key)); + } +} diff --git a/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs b/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs index d5bb0fb..de4f8e7 100644 --- a/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs +++ b/ZTSharp.Tests/StateStoreKeyNormalizationTests.cs @@ -6,6 +6,9 @@ public sealed class StateStoreKeyNormalizationTests [InlineData("con")] [InlineData("CON")] [InlineData("con.txt")] + [InlineData("conin$")] + [InlineData("conout$")] + [InlineData("clock$")] [InlineData("nul")] [InlineData("lpt1")] [InlineData("peers.d/con/peer")] diff --git a/ZTSharp.Tests/TunnelAndHttpTests.cs b/ZTSharp.Tests/TunnelAndHttpTests.cs index f251c12..5a88ced 100644 --- a/ZTSharp.Tests/TunnelAndHttpTests.cs +++ b/ZTSharp.Tests/TunnelAndHttpTests.cs @@ -306,7 +306,7 @@ public async Task InMemoryOverlayHttpHandler_LocalPortAllocator_RetriesUnderConc using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); var releaseTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var connectionTasks = new List(capacity: 3); + var connectionTasks = new Task?[3]; static async Task HandleConnectionAsync( TcpClient tcp, @@ -349,7 +349,7 @@ static async Task HandleConnectionAsync( for (var i = 0; i < 3; i++) { var tcp = await httpListener.AcceptTcpClientAsync(cts.Token).ConfigureAwait(false); - connectionTasks.Add(HandleConnectionAsync(tcp, releaseTcs.Task, cts.Token)); + connectionTasks[i] = HandleConnectionAsync(tcp, releaseTcs.Task, cts.Token); } }, cts.Token); @@ -402,7 +402,28 @@ static async Task HandleConnectionAsync( try { - await Task.WhenAll(connectionTasks.Concat(new[] { acceptTask, forwarderTask })).WaitAsync(TimeSpan.FromSeconds(2)); + var deadlineMs = Environment.TickCount64 + 2000; + static TimeSpan Remaining(long deadlineMs) + { + var remainingMs = deadlineMs - Environment.TickCount64; + return remainingMs <= 0 ? TimeSpan.Zero : TimeSpan.FromMilliseconds(remainingMs); + } + + try + { + await acceptTask.WaitAsync(Remaining(deadlineMs)); + } + catch (OperationCanceledException) when (cts.IsCancellationRequested) + { + } + + var snapshot = connectionTasks + .Where(t => t is not null) + .Select(t => t!) + .ToArray(); + await Task + .WhenAll(snapshot.Concat(new[] { forwarderTask })) + .WaitAsync(Remaining(deadlineMs)); } catch (OperationCanceledException) when (cts.IsCancellationRequested) { diff --git a/ZTSharp.Tests/UnixFactAttribute.cs b/ZTSharp.Tests/UnixFactAttribute.cs new file mode 100644 index 0000000..9103e4d --- /dev/null +++ b/ZTSharp.Tests/UnixFactAttribute.cs @@ -0,0 +1,16 @@ +using Xunit; + +namespace ZTSharp.Tests; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +internal sealed class UnixFactAttribute : FactAttribute +{ + public UnixFactAttribute() + { + if (OperatingSystem.IsWindows()) + { + Skip = "Unix only."; + } + } +} + diff --git a/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs b/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs index 06b3571..e269861 100644 --- a/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs +++ b/ZTSharp.Tests/UserSpaceTcpClientConnectTests.cs @@ -116,5 +116,41 @@ public async Task ConnectAsync_RetransmitsSyn_WhenNoSynAckArrives() Assert.Equal(TcpCodec.Flags.Ack, ackFlags); Assert.Equal(1001u, ackAck); } + + [Fact] + public async Task ConnectAsync_UsesConfiguredConnectTimeout() + { + await using var link = new InspectableIpv4Link(); + var localIp = IPAddress.Parse("10.0.0.1"); + var remoteIp = IPAddress.Parse("10.0.0.2"); + const ushort remotePort = 80; + const ushort localPort = 50000; + + await using var client = new UserSpaceTcpClient( + link, + localIp, + remoteIp, + remotePort, + localPort: localPort, + mss: 1200, + connectOptions: new UserSpaceTcpConnectOptions + { + ConnectTimeout = TimeSpan.FromMilliseconds(450), + InitialRetryDelay = TimeSpan.FromMilliseconds(200), + MaxRetryDelay = TimeSpan.FromMilliseconds(200) + }); + + var connectTask = client.ConnectAsync(); + + for (var attempt = 0; attempt < 3; attempt++) + { + var syn = await link.Outgoing.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(1)); + Assert.True(Ipv4Codec.TryParse(syn.Span, out _, out _, out _, out var payload)); + Assert.True(TcpCodec.TryParse(payload, out _, out _, out _, out _, out var flags, out _, out _)); + Assert.Equal(TcpCodec.Flags.Syn, flags); + } + + await Assert.ThrowsAsync(async () => await connectTask); + } } diff --git a/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs b/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs index 9cded1c..e3b9823 100644 --- a/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs +++ b/ZTSharp.Tests/VirtualNetworkInterfaceTests.cs @@ -5,7 +5,7 @@ public sealed class VirtualNetworkInterfaceTests [Fact] public async Task InMemoryVirtualInterface_DeliversPacket() { - var networkId = 0xCAFE0002UL; + var networkId = 0xCAFE0004UL; await using var node1 = new Node(new NodeOptions { diff --git a/ZTSharp.Tests/ZTSharp.Tests.csproj b/ZTSharp.Tests/ZTSharp.Tests.csproj index 5d69b5b..54a7ebb 100644 --- a/ZTSharp.Tests/ZTSharp.Tests.csproj +++ b/ZTSharp.Tests/ZTSharp.Tests.csproj @@ -11,6 +11,7 @@ + diff --git a/ZTSharp.Tests/ZeroTierApiTests.cs b/ZTSharp.Tests/ZeroTierApiTests.cs index 768e35d..ccea34b 100644 --- a/ZTSharp.Tests/ZeroTierApiTests.cs +++ b/ZTSharp.Tests/ZeroTierApiTests.cs @@ -73,4 +73,38 @@ await Assert.ThrowsAsync(async () => _ = await socket.ConnectTcpAsync(new IPEndPoint(IPAddress.IPv6Loopback, 80)).ConfigureAwait(false); }); } + + [Fact] + public async Task SuggestedHttpConnectTimeout_IsExtended_ForDirectOnlyMultipath() + { + await using var socket = await ZeroTierSocket.CreateAsync(new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-test-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = false + } + }); + + Assert.Equal(TimeSpan.FromSeconds(120), socket.SuggestedHttpConnectTimeout); + } + + [Fact] + public async Task SuggestedHttpConnectTimeout_RemainsShort_WhenRelayFallbackIsAllowed() + { + await using var socket = await ZeroTierSocket.CreateAsync(new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-test-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = true + } + }); + + Assert.Equal(TimeSpan.FromSeconds(10), socket.SuggestedHttpConnectTimeout); + } } diff --git a/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs b/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs index 67da90b..6e0652e 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneFragmentPolicyTests.cs @@ -49,6 +49,40 @@ public void Ipv6ExtensionHeaders_AreRecognized() Assert.False(Ipv6Codec.IsExtensionHeader(TcpCodec.ProtocolNumber)); } + [Fact] + public async Task Ipv6HopByHopHeader_DoesNotPreventUdpHandlerDispatch() + { + var localManagedIpV6 = IPAddress.Parse("fd00::2"); + await using var runtime = CreateRuntimeV6(localManagedIpV6); + var ip = GetIpHandler(runtime); + + const ushort localPort = 12011; + var udpChannel = Channel.CreateUnbounded(); + Assert.True(runtime.TryRegisterUdpPort(AddressFamily.InterNetworkV6, localPort, udpChannel.Writer)); + + var remoteIpV6 = IPAddress.Parse("fd00::1"); + var udp = UdpCodec.Encode(remoteIpV6, localManagedIpV6, sourcePort: 1111, destinationPort: localPort, payload: new byte[] { 1, 2, 3 }); + + var hbh = new byte[8]; + hbh[0] = UdpCodec.ProtocolNumber; // next header + hbh[1] = 0; // hdr ext len (8 bytes total) + + var payload = new byte[hbh.Length + udp.Length]; + hbh.CopyTo(payload, 0); + udp.CopyTo(payload, hbh.Length); + + var ipv6 = Ipv6Codec.Encode( + source: remoteIpV6, + destination: localManagedIpV6, + nextHeader: 0, // hop-by-hop + payload: payload, + hopLimit: 64); + + await ip.HandleIpv6PacketAsync(peerNodeId: new NodeId(0x3333333333), ipv6Packet: ipv6, cancellationToken: CancellationToken.None); + + Assert.True(udpChannel.Reader.TryRead(out _)); + } + private static void RewriteIpv4HeaderChecksum(byte[] packet) { var headerLength = (packet[0] & 0x0F) * 4; @@ -112,4 +146,21 @@ private static ZeroTierDataplaneRuntime CreateRuntime(IPAddress localManagedIpV4 localManagedIpsV4: new[] { localManagedIpV4 }, localManagedIpsV6: Array.Empty(), inlineCom: new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); + + [global::System.Diagnostics.CodeAnalysis.SuppressMessage( + "Reliability", + "CA2000:Dispose objects before losing scope", + Justification = "UDP transport ownership transfers to ZeroTierDataplaneRuntime, which is disposed by the caller.")] + private static ZeroTierDataplaneRuntime CreateRuntimeV6(IPAddress localManagedIpV6) + => new( + udp: new ZeroTierUdpTransport(localPort: 0, enableIpv6: false), + rootNodeId: new NodeId(0x1111111111), + rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootKey: new byte[48], + rootProtocolVersion: 12, + localIdentity: ZeroTierTestIdentities.CreateFastIdentity(0x2222222222), + networkId: 1, + localManagedIpsV4: Array.Empty(), + localManagedIpsV6: new[] { localManagedIpV6 }, + inlineCom: new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); } diff --git a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs index 1c61b7a..eb569e1 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRuntimeDirectPathTests.cs @@ -1,7 +1,10 @@ using System.Buffers.Binary; +using System.Linq; using System.Net; using System.Security.Cryptography; +using ZTSharp.ZeroTier; using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Net; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -9,35 +12,1549 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDataplaneRuntimeDirectPathTests { + private static readonly IPEndPoint RootEndpoint = new(IPAddress.Parse("203.0.113.1"), 9993); + + [Fact] + public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var peerNodeId = new NodeId(0x3333333333); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime(udp, localIdentity, rootNodeId, rootKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerNodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 0, RootEndpoint, packet); + + var holePunch = await WaitForSendAsync( + udp, + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4, + TimeSpan.FromSeconds(2)); + + Assert.Equal(0, holePunch.LocalSocketId); + Assert.Equal(2, holePunch.HopLimit); + } + + [Fact] + public async Task DataplaneRuntime_RendezvousBootstrap_UsesReceivingSocket_ForModernPeer() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.HopLimit is null), + TimeSpan.FromSeconds(2)); + + var bootstrapSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .Where(send => send.HopLimit is null) + .ToArray(); + + Assert.NotEmpty(bootstrapSends); + Assert.All(bootstrapSends, send => Assert.Equal(1, send.LocalSocketId)); + } + + [Fact] + public async Task DataplaneRuntime_RendezvousBootstrap_UsesReceivingSocket_ForPublicSiblingHint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var siblingEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, siblingEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(siblingEndpoint)), + TimeSpan.FromSeconds(2)); + + var bootstrapSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(siblingEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .ToArray(); + + Assert.NotEmpty(bootstrapSends); + Assert.All(bootstrapSends, send => Assert.Equal(1, send.LocalSocketId)); + } + + [Fact] + public async Task DataplaneRuntime_Rendezvous_DirectOnly_DoesNotRelayRootHello() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + await WaitForSendAsync( + udp, + send => send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + send.HopLimit == 2, + TimeSpan.FromSeconds(2)); + + Assert.DoesNotContain( + udp.GetSendsSnapshot(), + send => send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); + } + + [Fact] + public async Task DataplaneRuntime_Rendezvous_DelaysInitialDirectProbeUntilPeerVersionKnown() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity, peerProtocolVersion: 0, peerMajorVersion: 0, peerMinorVersion: 0, peerRevision: 0); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + await WaitForSendAsync( + udp, + send => send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + send.HopLimit == 2, + TimeSpan.FromSeconds(2)); + + await Task.Delay(150); + + Assert.DoesNotContain( + udp.GetSendsSnapshot(), + send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.HopLimit is null); + } + + [Fact] + public async Task DataplaneRuntime_RendezvousBootstrap_HelloReportsPeerEndpoint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity, peerProtocolVersion: 4); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); + + var helloSend = await WaitForSendAsync( + udp, + send => send.LocalSocketId == 1 && + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello, + TimeSpan.FromSeconds(2)); + + Assert.Equal(rendezvousEndpoint, ReadReportedHelloSurface(helloSend.Payload, sharedKey)); + } + + [Fact] + public async Task DataplaneRuntime_SeededHintedEndpoints_AreVisibleToMaintenanceBeforeHop0Confirmation() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var peerNodeId = new NodeId(0x3333333333); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4243); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1); + + runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoint); + + var candidates = runtime.GetHintedDirectCandidatesForMaintenance(peerNodeId); + var candidate = Assert.Single(candidates); + Assert.Equal(hintedEndpoint, candidate.RemoteEndPoint); + Assert.Equal(0, candidate.LocalSocketId); + } + + [Fact] + public async Task DataplaneRuntime_DirectHello_DoesNotConfirmUnknownPath() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint)); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var runtimeEndpoint = TestUdpEndpoints.ToLoopback(udp.LocalEndpoint); + var helloPacket = BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, runtimeEndpoint); + + await rootUdp.SendAsync(runtimeEndpoint, helloPacket); + _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + + Assert.False(runtime.TrySelectDirectPathForTests(peerIdentity.NodeId, flowId: 0, out _)); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_ExposesHintedPathCandidate_WithoutConfirmedHop0() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var peerNodeId = new NodeId(0x3333333333); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.SeedDirectEndpointsForTests(peerNodeId, hintedEndpoints); + + Assert.True(runtime.TrySelectDirectPathForTests(peerNodeId, flowId: 0, out var selected)); + Assert.Equal(hintedEndpoints[0], selected.RemoteEndPoint); + Assert.Equal(0, selected.LocalSocketId); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_PrefersRendezvousEndpoint_AfterLaterPushDirectPaths() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(2)); + + var endpoints = runtime.GetDirectEndpointsForTests(peerIdentity.NodeId); + Assert.Equal(rendezvousEndpoint, endpoints[0]); + + var rendezvousBootstrapSockets = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + (verb == ZeroTierVerb.Echo || verb == ZeroTierVerb.Hello)) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + if (rendezvousBootstrapSockets.Length != 0) + { + Assert.Equal(new[] { 1 }, rendezvousBootstrapSockets); + } + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_DoesNotUseHintedPayloadBeforeRendezvous() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4246) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + hintedEndpoints.Contains(send.RemoteEndPoint)), + TimeSpan.FromSeconds(1)); + + var sends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .ToArray(); + + var payloadSends = sends + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) + .ToArray(); + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_PushDirectPaths_WaitsForRendezvous() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); + + var echoSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + Assert.Empty(echoSends); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_PrefersPinnedRendezvous_ButWaitsForConfirmation() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), + TimeSpan.FromSeconds(2)); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + send.RemoteEndPoint.Equals(rendezvousEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + (verb == ZeroTierVerb.Echo || verb == ZeroTierVerb.Hello)), + TimeSpan.FromSeconds(1)); + + var directSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) + .ToArray(); + + var echoSends = directSends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Echo) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .ToArray(); + var payloadSends = directSends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .ToArray(); + + Assert.Contains((1, rendezvousEndpoint), echoSends); + Assert.DoesNotContain(echoSends, send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.LocalSocketId != 1); + Assert.Empty(payloadSends); + Assert.DoesNotContain(payloadSends, send => send.RemoteEndPoint.Equals(pushedEndpoint)); + Assert.DoesNotContain(payloadSends, send => send.LocalSocketId != 1); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_WithoutRelayedDirectPathPush() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(1)); + + var sendCountAfterPinnedRendezvous = udp.GetSendsSnapshot().Length; + + var directSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .ToArray(); + var payloadSends = directSends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .ToArray(); + + Assert.NotEmpty(directSends); + Assert.Empty(payloadSends); + Assert.DoesNotContain( + udp.GetSendsSnapshot().Skip(sendCountAfterPinnedRendezvous), + send => send.RemoteEndPoint.Equals(RootEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.PushDirectPaths); + await Assert.ThrowsAnyAsync(async () => await sendTask); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_MaintenanceStaysOnPinnedRendezvousSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) && send.Payload.Length == 4), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var directSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) + .ToArray(); + var rendezvousSends = directSends + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .ToArray(); + var pushedControlSends = directSends + .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .ToArray(); + + if (rendezvousSends.Length != 0) + { + Assert.DoesNotContain( + rendezvousSends, + send => send.LocalSocketId != 1); + Assert.Contains( + rendezvousSends, + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo); + Assert.DoesNotContain( + rendezvousSends, + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); + } + + if (pushedControlSends.Length != 0) + { + Assert.Contains( + pushedControlSends, + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Echo); + Assert.DoesNotContain( + pushedControlSends, + send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.Hello); + } + + Assert.DoesNotContain( + directSends, + send => send.RemoteEndPoint.Equals(pushedEndpoint) && + TryDecodeVerb(send.Payload, sharedKey, out var verb) && + verb == ZeroTierVerb.ExtFrame); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRevisitPinnedRendezvousSiblingHint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(pushedEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var maintenanceSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint) || send.RemoteEndPoint.Equals(pushedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .ToArray(); + + Assert.Empty(maintenanceSends); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_UsesSingleSocketForNonPublicAlternateHintAfterPinnedRendezvous() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var alternateEndpoint = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, [rendezvousEndpoint, alternateEndpoint]); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var maintenanceSockets = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(alternateEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + if (maintenanceSockets.Length != 0) + { + Assert.Equal(new[] { 1 }, maintenanceSockets); + } + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotPeriodicallyForceHelloOnPinnedRendezvousSiblingHint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(pushedEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var maintenanceSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(pushedEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb != ZeroTierVerb.ExtFrame) + .ToArray(); + + Assert.Empty(maintenanceSends); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_AdvertisedSurfaces_DoNotEnableHintedPayloadBeforeRendezvous() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await Task.Delay(300); + + var sends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .ToArray(); + var payloadSends = sends + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .ToArray(); + + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForRendezvousEvenAfterHintedPathsAppear() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + + await Task.Delay(300); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => + hintedEndpoints.Contains(send.RemoteEndPoint)), + TimeSpan.FromSeconds(1)); + + var sends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .ToArray(); + + var payloadSends = sends + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.ExtFrame) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint)) + .Distinct() + .OrderBy(send => send.RemoteEndPoint.Port) + .ThenBy(send => send.LocalSocketId) + .ToArray(); + + Assert.Empty(payloadSends); + await Assert.ThrowsAnyAsync(async () => await sendTask.WaitAsync(TimeSpan.FromSeconds(2))); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_SynPayload_WaitsForRendezvous_BeforeUsingHintedPaths() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var tcp = TcpCodec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + sourcePort: 50000, + destinationPort: 443, + sequenceNumber: 1, + acknowledgmentNumber: 0, + flags: TcpCodec.Flags.Syn, + windowSize: 65535, + options: ReadOnlySpan.Empty, + payload: ReadOnlySpan.Empty); + var ipv4 = Ipv4Codec.Encode( + IPAddress.Parse("10.0.0.1"), + IPAddress.Parse("10.0.0.2"), + TcpCodec.ProtocolNumber, + tcp, + identification: 1); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var sendTask = runtime.SendIpv4Async(peerIdentity.NodeId, ipv4, cts.Token).AsTask(); + udp.EnqueueInbound( + localSocketId: 0, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, hintedEndpoint)); + + var hintedSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(hintedEndpoint)) + .ToArray(); + + Assert.Empty(hintedSends); + await Assert.ThrowsAnyAsync(async () => await sendTask); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_UsesExtendedTcpConnectTimeout() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + Assert.Equal(TimeSpan.FromSeconds(30), runtime.GetTcpConnectOptionsForTests().ConnectTimeout); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_MaintenanceUsesPeriodicHintedHelloOnSingleSocket_ForModernPeers() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var hintedSends = udp.GetSendsSnapshot() + .Where(send => hintedEndpoints.Contains(send.RemoteEndPoint)) + .Select(send => new + { + send.LocalSocketId, + send.RemoteEndPoint, + Verb = TryDecodeVerb(send.Payload, sharedKey, out var verb) ? verb : (ZeroTierVerb?)null + }) + .ToArray(); + + var helloFanout = hintedSends + .Where(send => send.Verb == ZeroTierVerb.Hello) + .ToArray(); + + Assert.Equal(new[] { 0 }, helloFanout.Select(send => send.LocalSocketId).Distinct().Order().ToArray()); + Assert.Equal(new[] { hintedEndpoints[0] }, helloFanout.Select(send => send.RemoteEndPoint).Distinct().OrderBy(endpoint => endpoint.Port).ToArray()); + Assert.DoesNotContain(hintedSends, send => send.Verb == ZeroTierVerb.Echo); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_MaintenanceHello_ReportsHintedPeerEndpoint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new[] + { + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4245) + }; + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var helloSurfaces = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(hintedEndpoints[0])) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Hello) + .Select(send => (send.LocalSocketId, send.RemoteEndPoint, Surface: ReadReportedHelloSurface(send.Payload, sharedKey))) + .Distinct() + .OrderBy(send => send.LocalSocketId) + .ToArray(); + + Assert.All(helloSurfaces, send => Assert.Equal(send.RemoteEndPoint, send.Surface)); + } + [Fact] - public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayRendezvousForAdvertisedSurfaces() { - await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var punchReceiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + var rootNodeId = new NodeId(0x1111111111); - var peerNodeId = new NodeId(0x3333333333); + var hintedEndpoints = new[] { new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244) }; + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoints); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var rendezvousEndpoints = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Rendezvous) + .Select(send => ReadRendezvousEndpoint(send.Payload, sharedKey)) + .Distinct() + .OrderBy(endpoint => endpoint.Port) + .ToArray(); + + Assert.Empty(rendezvousEndpoints); + } + + [Fact] + public async Task DataplaneRuntime_DirectOnly_Maintenance_DoesNotRelayDirectPathPush_WhilePinnedRendezvousUnresolved() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); - var rootKey = new byte[48]; - RandomNumberGenerator.Fill(rootKey); + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); - await using var runtime = new ZeroTierDataplaneRuntime( + await using var runtime = CreateRuntime( udp, - rootNodeId: rootNodeId, - rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), - rootKey: rootKey, - rootProtocolVersion: 12, - localIdentity: localIdentity, - networkId: 0x9ad07d01093a69e3UL, - localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, - localManagedIpsV6: Array.Empty(), - inlineCom: Array.Empty()); + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); - var runtimeEndpoint = TestUdpEndpoints.ToLoopback(runtime.LocalUdp); - var rendezvousPayload = BuildRendezvousPayload(peerNodeId, TestUdpEndpoints.ToLoopback(punchReceiver.LocalEndpoint)); - var packet = ZeroTierPacketCodec.Encode( + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( new ZeroTierPacketHeader( PacketId: 1, Destination: localIdentity.NodeId, @@ -45,22 +1562,139 @@ public async Task DataplaneRuntime_HandlesRendezvous_AndSendsHolePunch() Flags: 0, Mac: 0, VerbRaw: (byte)ZeroTierVerb.Rendezvous), - rendezvousPayload); + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); - ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + udp.EnqueueInbound( + localSocketId: 1, + RootEndpoint, + BuildPushDirectPathsPacket(peerIdentity.NodeId, localIdentity.NodeId, sharedKey, pushedEndpoint)); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var pushDirectSendsAfterMaintenance = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.PushDirectPaths) + .ToArray(); + + Assert.Empty(pushDirectSendsAfterMaintenance); + } + + [Fact] + public async Task DataplaneRuntime_MultipathMaintenance_DoesNotSendPeerRendezvousViaRoot() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var hintedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4244); + var advertisedSurfaces = new[] + { + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + }; + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: advertisedSurfaces); + + runtime.PrimePeerForTests(peerIdentity); + runtime.SeedDirectEndpointsForTests(peerIdentity.NodeId, hintedEndpoint); + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await runtime.RunMultipathMaintenanceOnceForTestsAsync(); + + var rendezvousSends = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => TryDecodeVerb(send.Payload, sharedKey, out var verb) && verb == ZeroTierVerb.Rendezvous) + .ToArray(); + + Assert.Empty(rendezvousSends); + } + + [Fact] + public async Task DataplaneRuntime_PeerRendezvous_IsIgnored() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 62120); + var rootKey = RandomNumberGenerator.GetBytes(48); + + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true, AllowRootRelayFallback = false }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 3, + Destination: localIdentity.NodeId, + Source: peerIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, packet); - await rootUdp.SendAsync(runtimeEndpoint, packet); + await Task.Delay(TimeSpan.FromMilliseconds(200)); - var holePunch = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - Assert.Equal(4, holePunch.Payload.Length); + var directSends = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)) + .ToArray(); + + Assert.Empty(directSends); } [Fact] - public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsHolePunch() + public async Task DataplaneRuntime_MultipathMaintenance_IncludesTrustedRelayedPeer_AndLocalAdvertisements() { await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var punchReceiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); @@ -68,39 +1702,260 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsHolePunch() Assert.True(peerIdentity.LocallyValidate()); var rootNodeId = new NodeId(0x1111111111); - var rootKey = new byte[48]; - RandomNumberGenerator.Fill(rootKey); + var rootKey = RandomNumberGenerator.GetBytes(48); - await using var runtime = new ZeroTierDataplaneRuntime( + var publicSurface = new IPEndPoint(IPAddress.Parse("203.0.113.10"), 54321); + await using var runtime = CreateRuntime( udp, - rootNodeId: rootNodeId, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1, rootEndpoint: TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint), - rootKey: rootKey, - rootProtocolVersion: 12, - localIdentity: localIdentity, - networkId: 0x9ad07d01093a69e3UL, - localManagedIpsV4: new[] { IPAddress.Parse("10.0.0.1") }, - localManagedIpsV6: Array.Empty(), - inlineCom: Array.Empty()); + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(LocalSocketId: 0, SurfaceAddress: publicSurface) + ]); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var runtimeEndpoint = TestUdpEndpoints.ToLoopback(udp.LocalEndpoint); + var helloPacket = BuildRootRelayedHelloPacket(peerIdentity, localIdentity, sharedKey, runtimeEndpoint); + + await rootUdp.SendAsync(runtimeEndpoint, helloPacket); + _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + + var maintenancePeers = runtime.GetPeersForMultipathMaintenanceForTests(); + Assert.Contains(peerIdentity.NodeId, maintenancePeers); + + var advertisements = runtime.GetLocalDirectPathAdvertisementsForTests(); + Assert.Contains(publicSurface, advertisements); + } + + [Fact] + public async Task DataplaneRuntime_SendViaRootSockets_FansOutAcrossLocalSockets() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1, + initialExternalSurfaceObservations: + [ + new ZeroTierExternalSurfaceObservation(0, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000)), + new ZeroTierExternalSurfaceObservation(1, new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001)) + ]); + + var payload = new byte[] { 1, 2, 3 }; + await runtime.SendViaRootSocketsForTestsAsync(payload); + + var socketIds = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => send.Payload.SequenceEqual(payload)) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + Assert.Equal(new[] { 0, 1 }, socketIds); + } + + [Fact] + public async Task DataplaneRuntime_SendHelloViaRoot_UsesPeerRootSocketAffinity() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); - var runtimeEndpoint = TestUdpEndpoints.ToLoopback(runtime.LocalUdp); + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions { Enabled = true }, + planetId: 1, + planetTimestamp: 1); var sharedKey = new byte[48]; - ZeroTierC25519.Agree(peerIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); - var planet = new ZeroTierWorld( - ZeroTierWorldType.Planet, - id: 1, - timestamp: 1, - updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], - signature: new byte[ZeroTierWorld.C25519SignatureLength], - roots: Array.Empty()); + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + await runtime.SendHelloViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var socketIds = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && decoded.Header.Verb == ZeroTierVerb.Hello) + .Select(send => send.LocalSocketId) + .Distinct() + .ToArray(); + + Assert.Equal(new[] { 1 }, socketIds); + } + + [Fact] + public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithObservedRootPath_FansOutAcrossSockets() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = false + }, + planetId: 1, + planetTimestamp: 1); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + runtime.ObservePeerRootSocketForTests(peerIdentity.NodeId, localSocketId: 1); + await runtime.SendHelloViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var socketIds = udp.GetSendsSnapshot() + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && decoded.Header.Verb == ZeroTierVerb.Hello) + .Select(send => send.LocalSocketId) + .Distinct() + .Order() + .ToArray(); + + Assert.Equal(new[] { 0, 1 }, socketIds); + } + + [Fact] + public async Task DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithPinnedRendezvous_UsesObservedRootSocket() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + + var localIdentity = ZeroTierTestIdentities.CreateFastIdentity(0x2222222222); + Assert.True(ZeroTierIdentity.TryParse(ZeroTierTestIdentities.KnownGoodIdentity, out var peerIdentity)); + Assert.True(peerIdentity.HasPrivateKey); + Assert.True(peerIdentity.LocallyValidate()); + + var rootNodeId = new NodeId(0x1111111111); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var rootKey = RandomNumberGenerator.GetBytes(48); + await using var runtime = CreateRuntime( + udp, + localIdentity, + rootNodeId, + rootKey, + multipath: new ZeroTierMultipathOptions + { + Enabled = true, + AllowRootRelayFallback = false + }, + planetId: 1, + planetTimestamp: 1); + + runtime.PrimePeerForTests(peerIdentity); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, peerIdentity.PublicKey, sharedKey); + + var rendezvousPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localIdentity.NodeId, + Source: rootNodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.Rendezvous), + BuildRendezvousPayload(peerIdentity.NodeId, rendezvousEndpoint)); + ZeroTierPacketCrypto.Armor(rendezvousPacket, ZeroTierPacketCrypto.SelectOutboundKey(rootKey, remoteProtocolVersion: 12), encryptPayload: true); + + udp.EnqueueInbound(localSocketId: 1, RootEndpoint, rendezvousPacket); + + await WaitForConditionAsync( + () => udp.GetSendsSnapshot().Any(send => send.RemoteEndPoint.Equals(rendezvousEndpoint)), + TimeSpan.FromSeconds(2)); + + var initialSendCount = udp.GetSendsSnapshot().Length; + await runtime.SendHelloViaRootForTestsAsync(peerIdentity.NodeId, sharedKey); + + var socketIds = udp.GetSendsSnapshot() + .Skip(initialSendCount) + .Where(send => send.RemoteEndPoint.Equals(RootEndpoint)) + .Where(send => ZeroTierPacketCodec.TryDecode(send.Payload, out var decoded) && decoded.Header.Verb == ZeroTierVerb.Hello) + .Select(send => send.LocalSocketId) + .Distinct() + .ToArray(); + + Assert.Equal(new[] { 1 }, socketIds); + } + + private static ZeroTierDataplaneRuntime CreateRuntime( + IZeroTierUdpTransport udp, + ZeroTierIdentity localIdentity, + NodeId rootNodeId, + byte[] rootKey, + ZeroTierMultipathOptions? multipath = null, + ulong planetId = 0, + ulong planetTimestamp = 0, + IPEndPoint? rootEndpoint = null, + IReadOnlyList? initialExternalSurfaceObservations = null) + => new( + udp, + rootNodeId: rootNodeId, + rootEndpoint: rootEndpoint ?? RootEndpoint, + rootKey: rootKey, + rootProtocolVersion: 12, + localIdentity, + networkId: 0x9ad07d01093a69e3UL, + localManagedIpsV4: [IPAddress.Parse("10.0.0.1")], + localManagedIpsV6: Array.Empty(), + inlineCom: Array.Empty(), + multipath: multipath ?? new ZeroTierMultipathOptions(), + planetId: planetId, + planetTimestamp: planetTimestamp, + initialExternalSurfaceObservations: initialExternalSurfaceObservations); - var helloPacket = ZeroTierHelloPacketBuilder.BuildPacket( + private static byte[] BuildRootRelayedHelloPacket( + ZeroTierIdentity peerIdentity, + ZeroTierIdentity localIdentity, + byte[] sharedKey, + IPEndPoint physicalDestination) + => ZeroTierHelloPacketBuilder.BuildPacket( peerIdentity, destination: localIdentity.NodeId, - physicalDestination: runtimeEndpoint, - planet, + physicalDestination, + CreatePlanet(), timestamp: (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), sharedKey, advertisedProtocolVersion: ZeroTierHelloClient.AdvertisedProtocolVersion, @@ -109,42 +1964,90 @@ public async Task DataplaneRuntime_HandlesPushDirectPaths_AndSendsHolePunch() advertisedRevision: ZeroTierHelloClient.AdvertisedRevision, out _); - await rootUdp.SendAsync(runtimeEndpoint, helloPacket); - _ = await rootUdp.ReceiveAsync(TimeSpan.FromSeconds(2)); + private static bool TryDecodeVerb(byte[] packet, byte[] sharedKey, out ZeroTierVerb verb) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed)) + { + verb = default; + return false; + } - var pushPayload = BuildPushDirectPathsPayload(TestUdpEndpoints.ToLoopback(punchReceiver.LocalEndpoint)); - var pushPacket = ZeroTierPacketCodec.Encode( - new ZeroTierPacketHeader( - PacketId: 2, - Destination: localIdentity.NodeId, - Source: peerIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.PushDirectPaths), - pushPayload); + verb = parsed.Header.Verb; + return true; + } - ZeroTierPacketCrypto.Armor(pushPacket, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); + private static IPEndPoint? ReadReportedHelloSurface(byte[] packet, byte[] sharedKey) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed)) + { + throw new InvalidOperationException("Failed to decode HELLO packet."); + } - await rootUdp.SendAsync(runtimeEndpoint, pushPacket); + var payload = parsed.Payload.Span; + ZeroTierIdentityCodec.Deserialize(payload.Slice(13), out var identityLength); + if (!ZeroTierInetAddressCodec.TryDeserialize(payload.Slice(13 + identityLength), out var surface, out _)) + { + throw new InvalidOperationException("Failed to decode advertised HELLO surface."); + } - var holePunch = await punchReceiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - Assert.Equal(4, holePunch.Payload.Length); + return surface; } - private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) + private static IPEndPoint ReadRendezvousEndpoint(byte[] packet, byte[] sharedKey) { - var addressBytes = endpoint.Address.GetAddressBytes(); - var payload = new byte[1 + 5 + 2 + 1 + addressBytes.Length]; + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed) || + !ZeroTierRendezvousCodec.TryParse(parsed.Payload.Span, out var rendezvous)) + { + throw new InvalidOperationException("Failed to decode RENDEZVOUS packet."); + } - payload[0] = 0; - ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, 5), with.Value); - BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(1 + 5, 2), (ushort)endpoint.Port); - payload[1 + 5 + 2] = (byte)addressBytes.Length; - addressBytes.CopyTo(payload.AsSpan(1 + 5 + 2 + 1)); + return rendezvous.Endpoint; + } - return payload; + private static IPEndPoint[] ReadPushedDirectPathEndpoints(byte[] packet, byte[] sharedKey) + { + var decoded = packet.ToArray(); + if (!ZeroTierPacketCrypto.Dearmor(decoded, sharedKey) || + !ZeroTierPacketCodec.TryDecode(decoded, out var parsed) || + !ZeroTierPushDirectPathsCodec.TryParse(parsed.Payload.Span, out var paths)) + { + throw new InvalidOperationException("Failed to decode PUSH_DIRECT_PATHS packet."); + } + + return paths + .Select(path => path.Endpoint) + .ToArray(); + } + + private static byte[] BuildPushDirectPathsPacket( + NodeId source, + NodeId destination, + byte[] sharedKey, + IPEndPoint endpoint) + { + var packet = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 2, + Destination: destination, + Source: source, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.PushDirectPaths), + BuildPushDirectPathsPayload(endpoint)); + + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion: 12), encryptPayload: true); + return packet; } + private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) + => ZeroTierRendezvousCodec.BuildPayload(with, endpoint); + private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint) { if (endpoint.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork) @@ -158,23 +2061,64 @@ private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint) throw new ArgumentOutOfRangeException(nameof(endpoint), "Invalid IPv4 address bytes."); } - var payload = new byte[2 + 1 + 2 + 1 + 1 + 6]; + var payload = new byte[13]; var span = payload.AsSpan(); BinaryPrimitives.WriteUInt16BigEndian(span.Slice(0, 2), 1); - var ptr = 2; - span[ptr++] = 0; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), 0); - ptr += 2; + span[2] = 0; + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(3, 2), 0); + span[5] = 4; + span[6] = 6; + addressBytes.CopyTo(span.Slice(7, 4)); + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(11, 2), (ushort)endpoint.Port); + + return payload; + } + + private static ZeroTierWorld CreatePlanet() + => new( + ZeroTierWorldType.Planet, + id: 1, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: Array.Empty()); + + private static async Task WaitForSendAsync( + ScriptedUdpTransport udp, + Func predicate, + TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + var send = udp.GetSendsSnapshot().FirstOrDefault(predicate); + if (send != default) + { + return send; + } + + await Task.Delay(20); + } + + throw new TimeoutException("Timed out waiting for matching UDP send."); + } - span[ptr++] = 4; - span[ptr++] = 6; + private static async Task WaitForConditionAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + if (condition()) + { + return; + } - addressBytes.CopyTo(span.Slice(ptr, 4)); - ptr += 4; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), (ushort)endpoint.Port); + await Task.Delay(20); + } - return payload; + throw new TimeoutException("Timed out waiting for condition."); } } + diff --git a/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs b/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs index 50ae769..4968590 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneRxLoopTests.cs @@ -188,7 +188,7 @@ public async Task PeerLoopAsync_ContinuesAfterProcessorFault() } [Fact] - public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint() + public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint_WhenDirectPeerRxEnabled() { await using var rxUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var rootSender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); @@ -215,7 +215,8 @@ public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint() rootKey: new byte[48], localNodeId, rootClient, - peerDatagrams: new NoopPeerDatagrams()); + peerDatagrams: new NoopPeerDatagrams(), + acceptDirectPeerDatagrams: true); var peerChannel = Channel.CreateUnbounded(new UnboundedChannelOptions { @@ -224,7 +225,7 @@ public async Task DispatcherLoopAsync_ForwardsPeerDatagrams_FromAnyEndpoint() }); using var cts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); var peerNodeId = new NodeId(0x3333333333); var peerPacket = ZeroTierPacketCodec.Encode( @@ -257,16 +258,84 @@ await Assert.ThrowsAsync(async () => await dispatcher.WaitAsync(TimeSpan.FromSeconds(2)); } + [Fact] + public async Task DispatcherLoopAsync_ForwardsOnlyRootPeerDatagrams_WhenDirectPeerRxDisabled() + { + await using var rxUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var rootSender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + await using var attackerSender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var rootEndpoint = TestUdpEndpoints.ToLoopback(rootSender.LocalEndpoint); + var rootNodeId = new NodeId(0x1111111111); + var localNodeId = new NodeId(0x2222222222); + + var rootClient = new ZeroTierDataplaneRootClient( + rxUdp, + rootNodeId, + rootEndpoint, + rootKey: new byte[48], + rootProtocolVersion: 12, + localNodeId, + networkId: 1, + inlineCom: Array.Empty()); + + var loops = new ZeroTierDataplaneRxLoops( + rxUdp, + rootNodeId, + rootEndpoint, + rootKey: new byte[48], + localNodeId, + rootClient, + peerDatagrams: new NoopPeerDatagrams(), + acceptDirectPeerDatagrams: false); + + var peerChannel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = true + }); + + using var cts = new CancellationTokenSource(); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); + + var peerNodeId = new NodeId(0x3333333333); + var peerPacket = ZeroTierPacketCodec.Encode( + new ZeroTierPacketHeader( + PacketId: 1, + Destination: localNodeId, + Source: peerNodeId, + Flags: 0, + Mac: 0, + VerbRaw: 0), + ReadOnlySpan.Empty); + + var receiverEndpoint = TestUdpEndpoints.ToLoopback(rxUdp.LocalEndpoint); + await attackerSender.SendAsync(receiverEndpoint, peerPacket); + await rootSender.SendAsync(receiverEndpoint, peerPacket); + + var forwarded = await ReadAsyncWithTimeout(peerChannel.Reader, TimeSpan.FromSeconds(2)); + Assert.Equal(rootEndpoint, forwarded.RemoteEndPoint); + + await Assert.ThrowsAsync(async () => + { + _ = await ReadAsyncWithTimeout(peerChannel.Reader, TimeSpan.FromMilliseconds(100)); + }); + + cts.Cancel(); + await dispatcher.WaitAsync(TimeSpan.FromSeconds(2)); + } + [Fact] public async Task DispatcherLoopAsync_DropsPeerDatagrams_WhenPeerQueueIsFull_AndCountsDrops() { await using var rxUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var sender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + var rootEndpoint = TestUdpEndpoints.ToLoopback(sender.LocalEndpoint); var rootClient = new ZeroTierDataplaneRootClient( rxUdp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], rootProtocolVersion: 12, localNodeId: new NodeId(0x2222222222), @@ -277,7 +346,7 @@ public async Task DispatcherLoopAsync_DropsPeerDatagrams_WhenPeerQueueIsFull_And var loops = new ZeroTierDataplaneRxLoops( rxUdp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], localNodeId: new NodeId(0x2222222222), rootClient: rootClient, @@ -293,7 +362,7 @@ public async Task DispatcherLoopAsync_DropsPeerDatagrams_WhenPeerQueueIsFull_And Assert.True(peerChannel.Writer.TryWrite(new ZeroTierUdpDatagram(LocalSocketId: 0, new IPEndPoint(IPAddress.Loopback, 1), new byte[1]))); using var cts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); var localNodeId = new NodeId(0x2222222222); var peerNodeId = new NodeId(0x3333333333); @@ -360,7 +429,7 @@ public async Task ResolveNodeIdAsync_CachesResult_WhenRootResponseComesFromAlter }); using var dispatcherCts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, dispatcherCts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, dispatcherCts.Token), CancellationToken.None); var remoteNodeId = new NodeId(0xaaaaaaaaaa); var serverTask = RunGatherOkServerOnceAsync( diff --git a/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs b/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs index 41fd306..e79d847 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneShutdownTests.cs @@ -14,10 +14,11 @@ public async Task DispatcherLoopAsync_ExitsCleanly_WhenUdpTransportIsDisposed() await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); await using var sender = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + var rootEndpoint = TestUdpEndpoints.ToLoopback(sender.LocalEndpoint); var rootClient = new ZeroTierDataplaneRootClient( udp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], rootProtocolVersion: 12, localNodeId: new NodeId(0x2222222222), @@ -27,7 +28,7 @@ public async Task DispatcherLoopAsync_ExitsCleanly_WhenUdpTransportIsDisposed() var loops = new ZeroTierDataplaneRxLoops( udp, rootNodeId: new NodeId(0x1111111111), - rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootEndpoint: rootEndpoint, rootKey: new byte[48], localNodeId: new NodeId(0x2222222222), rootClient: rootClient, @@ -40,7 +41,7 @@ public async Task DispatcherLoopAsync_ExitsCleanly_WhenUdpTransportIsDisposed() }); using var cts = new CancellationTokenSource(); - var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel.Writer, cts.Token), CancellationToken.None); + var dispatcher = Task.Run(() => loops.DispatcherLoopAsync(peerChannel, cts.Token), CancellationToken.None); var peerPacket = ZeroTierPacketCodec.Encode( new ZeroTierPacketHeader( diff --git a/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs b/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs index a121fd0..7d72d60 100644 --- a/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs +++ b/ZTSharp.Tests/ZeroTierDataplaneUdpChecksumTests.cs @@ -51,6 +51,29 @@ public async Task ValidUdpChecksum_IsDelivered_ToUdpHandler() Assert.Equal(new NodeId(0x3333333333), delivered.PeerNodeId); } + [Fact] + public async Task ValidUdpChecksum_WithTrailingBytes_IsDelivered_ToUdpHandler() + { + var localManagedIpV4 = IPAddress.Parse("10.0.0.2"); + await using var runtime = CreateRuntime(localManagedIpV4); + var ip = GetIpHandler(runtime); + + const ushort localPort = 12002; + var udpChannel = Channel.CreateUnbounded(); + Assert.True(runtime.TryRegisterUdpPort(AddressFamily.InterNetwork, localPort, udpChannel.Writer)); + + var remoteIp = IPAddress.Parse("10.0.0.1"); + var udp = UdpCodec.Encode(remoteIp, localManagedIpV4, sourcePort: 1111, destinationPort: localPort, payload: new byte[] { 1, 2, 3 }); + var padded = new byte[udp.Length + 8]; + udp.CopyTo(padded, 0); + + var ipv4 = Ipv4Codec.Encode(remoteIp, localManagedIpV4, UdpCodec.ProtocolNumber, padded, identification: 1); + await ip.HandleIpv4PacketAsync(peerNodeId: new NodeId(0x3333333333), ipv4Packet: ipv4, cancellationToken: CancellationToken.None); + + Assert.True(udpChannel.Reader.TryRead(out var delivered)); + Assert.Equal(new NodeId(0x3333333333), delivered.PeerNodeId); + } + private static ZeroTierDataplaneIpHandler GetIpHandler(ZeroTierDataplaneRuntime runtime) { var peerPackets = GetPrivateField(runtime, "_peerPackets"); diff --git a/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs new file mode 100644 index 0000000..8d6e935 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectBootstrapControlPolicyTests.cs @@ -0,0 +1,169 @@ +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectBootstrapControlPolicyTests +{ + [Theory] + [InlineData(true, false, true, false)] + [InlineData(false, true, true, false)] + [InlineData(false, false, false, false)] + [InlineData(false, false, true, true)] + public void ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints)); + } + + [Theory] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + [InlineData(false, true, true)] + [InlineData(false, false, false)] + public void ShouldRelayRootControlOnRendezvous_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldRelayRootControlOnRendezvous( + allowRootRelayFallback, + hasConfirmedDirectPath)); + } + + [Theory] + [InlineData(false, false, true, true, 1_200L, 1_000L, 2_000L, true)] + [InlineData(false, false, true, true, 3_500L, 1_000L, 2_000L, false)] + [InlineData(false, false, true, false, 1_200L, 1_000L, 2_000L, false)] + [InlineData(false, true, true, true, 1_200L, 1_000L, 2_000L, false)] + [InlineData(true, false, true, true, 1_200L, 1_000L, 2_000L, false)] + [InlineData(false, false, false, true, 1_200L, 1_000L, 2_000L, false)] + public void ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousRelayQuietMs, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + canUseEchoForDirectBootstrap, + nowMs, + pinnedRendezvousObservedAtMs, + pinnedRendezvousRelayQuietMs)); + } + + [Theory] + [InlineData(false, false, true, false, 1_200L, 1_000L, 750L, true)] + [InlineData(false, false, true, false, 1_800L, 1_000L, 750L, false)] + [InlineData(false, false, true, true, 1_200L, 1_000L, 750L, false)] + [InlineData(false, true, true, false, 1_200L, 1_000L, 750L, false)] + [InlineData(true, false, true, false, 1_200L, 1_000L, 750L, false)] + public void ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousVersionGraceMs, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + canUseEchoForDirectBootstrap, + nowMs, + pinnedRendezvousObservedAtMs, + pinnedRendezvousVersionGraceMs)); + } + + [Theory] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + public void ShouldRefreshRelayedRootControl_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( + allowRootRelayFallback, + hasConfirmedDirectPath)); + } + + [Theory] + [InlineData(true, false, false, false, false)] + [InlineData(false, true, false, false, false)] + [InlineData(false, false, true, false, false)] + [InlineData(false, false, false, true, false)] + [InlineData(false, false, false, false, true)] + public void ShouldDelayDirectOnlyHintedControlUntilRendezvous_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool endpointIsPinnedRendezvous, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldDelayDirectOnlyHintedControlUntilRendezvous( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + endpointIsPinnedRendezvous)); + } + + [Theory] + [InlineData(false, false, false, 1_000L, 0L, 3_000L, null, 1_000L, true)] + [InlineData(false, false, false, 3_500L, 0L, 3_000L, null, 1_000L, true)] + [InlineData(false, false, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, true)] + [InlineData(false, false, true, 2_100L, 0L, 3_000L, 1_000L, 1_000L, false)] + [InlineData(false, true, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, false)] + [InlineData(true, false, true, 1_500L, 0L, 3_000L, 1_000L, 1_000L, false)] + public void ShouldWaitForDirectOnlyHintedPayload_ReturnsExpectedValue( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + long nowMs, + long? bootstrapStartedAtMs, + long directOnlyRendezvousGraceMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousSettleMs, + bool expected) + { + Assert.Equal( + expected, + ZeroTierDirectBootstrapControlPolicy.ShouldWaitForDirectOnlyHintedPayload( + allowRootRelayFallback, + hasConfirmedDirectPath, + hasPinnedRendezvousEndpoints, + nowMs, + bootstrapStartedAtMs, + directOnlyRendezvousGraceMs, + pinnedRendezvousObservedAtMs, + pinnedRendezvousSettleMs)); + } +} diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs new file mode 100644 index 0000000..f481fb3 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerHopLimitTests.cs @@ -0,0 +1,83 @@ +using System.Buffers.Binary; +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointManagerHopLimitTests +{ + [Fact] + public async Task Rendezvous_UsesLowTtlHolePunch() + { + var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 7, + receivedVia: relay, + CancellationToken.None); + + var send = Assert.Single(udp.Sends); + Assert.Equal(7, send.LocalSocketId); + Assert.Equal(endpoint, send.RemoteEndPoint); + Assert.Equal(2, send.HopLimit); + } + + private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) + { + var addressBytes = endpoint.Address.GetAddressBytes(); + var payload = new byte[1 + 5 + 2 + 1 + addressBytes.Length]; + + payload[0] = 0; + ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, 5), with.Value); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(6, 2), (ushort)endpoint.Port); + payload[8] = (byte)addressBytes.Length; + addressBytes.CopyTo(payload.AsSpan(9)); + + return payload; + } + + private sealed class RecordingUdpTransport : IZeroTierUdpTransport + { + public List Sends { get; } = new(); + + public IReadOnlyList LocalSockets { get; } = + new[] { new ZeroTierUdpLocalSocket(Id: 7, LocalEndpoint: new IPEndPoint(IPAddress.Loopback, 0)) }; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => SendAsync(localSocketId: 7, remoteEndpoint, payload, cancellationToken); + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, HopLimit: null)); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, hopLimit)); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private readonly record struct SendCall(int LocalSocketId, IPEndPoint RemoteEndPoint, ReadOnlyMemory Payload, int? HopLimit); +} diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs index 65e6844..e138360 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerPushFlagsTests.cs @@ -1,69 +1,316 @@ -using System.Buffers.Binary; -using System.Net; -using ZTSharp.ZeroTier.Internal; -using ZTSharp.ZeroTier.Protocol; -using ZTSharp.ZeroTier.Transport; +using System.Buffers.Binary; +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointManagerPushFlagsTests +{ + [Fact] + public async Task PushDirectPaths_ForgetFlag_RemovesEndpoint() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.Contains(manager.Endpoints, ep => ep.Equals(endpoint)); + + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagForgetPath), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.DoesNotContain(manager.Endpoints, ep => ep.Equals(endpoint)); + } + + [Fact] + public async Task PushDirectPaths_RejectedEndpoint_IsIgnored() + { + await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + shouldAcceptEndpoint: static endpoint => !endpoint.Address.ToString().StartsWith("172.", StringComparison.Ordinal)); + + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993), flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + + Assert.Empty(manager.Endpoints); + } + + [Fact] + public async Task PushDirectPaths_KnownEndpoint_DoesNotPersistOrdinarySocketAffinity() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hintedEndpoints = new List(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, _, endpoint, _, _, _) => + { + hintedEndpoints.Add(endpoint); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + var payload = BuildPushDirectPathsPayload(endpoint, flags: 0); + + await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 0, CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId: 1, CancellationToken.None); + + Assert.Equal(new[] { endpoint, endpoint }, hintedEndpoints); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(udp.Sends); + } -namespace ZTSharp.Tests; + [Fact] + public async Task PushDirectPaths_NewEndpoint_DoesNotRememberReceivingSocket() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(udp.Sends); + } -public sealed class ZeroTierDirectEndpointManagerPushFlagsTests -{ [Fact] - public async Task PushDirectPaths_ForgetFlag_RemovesEndpoint() + public async Task PushDirectPaths_NewerEndpoints_ArePreferredAheadOfStaleOnes() { - await using var udp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); - await using var receiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + var udp = new RecordingUdpTransport(); var relay = new IPEndPoint(IPAddress.Loopback, 9999); var peerNodeId = new NodeId(0x1111111111); var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); - var endpoint = TestUdpEndpoints.ToLoopback(receiver.LocalEndpoint); - await manager.HandlePushDirectPathsFromRemoteAsync(BuildPushDirectPathsPayload(endpoint, flags: 0), CancellationToken.None); - - _ = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); - Assert.Contains(manager.Endpoints, ep => ep.Equals(endpoint)); + var oldEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + var newEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9123); await manager.HandlePushDirectPathsFromRemoteAsync( - BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagForgetPath), + BuildPushDirectPathsPayload(oldEndpoint, flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(newEndpoint, flags: 0), + receivedLocalSocketId: 1, CancellationToken.None); - Assert.DoesNotContain(manager.Endpoints, ep => ep.Equals(endpoint)); + Assert.Equal(newEndpoint, manager.Endpoints[0]); + Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(oldEndpoint)); + Assert.Empty(manager.GetPreferredLocalSocketIds(newEndpoint)); } - private const byte ZtPushDirectPathsFlagForgetPath = 0x01; - - private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flags) + [Fact] + public async Task PushDirectPaths_DoesNotDemote_RendezvousEndpoint() { - if (endpoint.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork) - { - throw new ArgumentOutOfRangeException(nameof(endpoint), "Test helper supports IPv4 only."); - } + var udp = new RecordingUdpTransport(); - var addressBytes = endpoint.Address.GetAddressBytes(); - if (addressBytes.Length != 4) - { - throw new ArgumentOutOfRangeException(nameof(endpoint), "Invalid IPv4 address bytes."); - } - - var payload = new byte[2 + 1 + 2 + 1 + 1 + 6]; - var span = payload.AsSpan(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(0, 2), 1); + var rendezvousEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var pushedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); - var ptr = 2; - span[ptr++] = flags; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), 0); - ptr += 2; + await manager.HandleRendezvousFromRootAsync( + ZeroTierRendezvousCodec.BuildPayload(peerNodeId, rendezvousEndpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(pushedEndpoint, flags: 0), + receivedLocalSocketId: 0, + CancellationToken.None); - span[ptr++] = 4; - span[ptr++] = 6; + Assert.Equal(rendezvousEndpoint, manager.Endpoints[0]); + Assert.Contains(manager.Endpoints, endpoint => endpoint.Equals(pushedEndpoint)); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(rendezvousEndpoint)); + } - addressBytes.CopyTo(span.Slice(ptr, 4)); - ptr += 4; - BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), (ushort)endpoint.Port); + [Fact] + public async Task PushDirectPaths_NormalHint_AllowsAllEligibleSockets() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 1, + CancellationToken.None); - return payload; + Assert.Single(hints); + Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); } -} + [Fact] + public async Task PushDirectPaths_PrivateNormalHint_AllowsAllEligibleSockets() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: 0), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, false, true, 1), hints[0]); + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + } + + [Fact] + public async Task PushDirectPaths_ClusterRedirect_UsesReceivingSocketAndForcesFullHello() + { + var udp = new RecordingUdpTransport(); + + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, endpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((endpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 8212); + await manager.HandlePushDirectPathsFromRemoteAsync( + BuildPushDirectPathsPayload(endpoint, flags: ZtPushDirectPathsFlagClusterRedirect), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, true, false, 1), hints[0]); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + Assert.Empty(udp.Sends); + } + + private const byte ZtPushDirectPathsFlagForgetPath = 0x01; + private const byte ZtPushDirectPathsFlagClusterRedirect = 0x02; + + private static byte[] BuildPushDirectPathsPayload(IPEndPoint endpoint, byte flags) + { + if (endpoint.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork) + { + throw new ArgumentOutOfRangeException(nameof(endpoint), "Test helper supports IPv4 only."); + } + + var addressBytes = endpoint.Address.GetAddressBytes(); + if (addressBytes.Length != 4) + { + throw new ArgumentOutOfRangeException(nameof(endpoint), "Invalid IPv4 address bytes."); + } + + var payload = new byte[2 + 1 + 2 + 1 + 1 + 6]; + var span = payload.AsSpan(); + + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(0, 2), 1); + + var ptr = 2; + span[ptr++] = flags; + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), 0); + ptr += 2; + + span[ptr++] = 4; + span[ptr++] = 6; + + addressBytes.CopyTo(span.Slice(ptr, 4)); + ptr += 4; + BinaryPrimitives.WriteUInt16BigEndian(span.Slice(ptr, 2), (ushort)endpoint.Port); + + return payload; + } + + private sealed class RecordingUdpTransport : IZeroTierUdpTransport + { + public List<(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit)> Sends { get; } = new(); + + public IReadOnlyList LocalSockets { get; } = + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Loopback, 0)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Loopback, 0)) + ]; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => SendAsync(0, remoteEndpoint, payload, cancellationToken); + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + Sends.Add((localSocketId, remoteEndpoint, null)); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) + { + Sends.Add((localSocketId, remoteEndpoint, hopLimit)); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} + diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs new file mode 100644 index 0000000..79ee033 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectEndpointManagerSocketAffinityTests.cs @@ -0,0 +1,156 @@ +using System.Buffers.Binary; +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointManagerSocketAffinityTests +{ + [Fact] + public async Task Rendezvous_UsesReceivingLocalSocket_ForInitialHolePunch() + { + await using var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + + var send = Assert.Single(udp.Sends); + Assert.Equal(1, send.LocalSocketId); + Assert.Equal(endpoint, send.RemoteEndPoint); + Assert.Equal(2, send.HopLimit); + Assert.Equal(new[] { 1 }, manager.GetPreferredLocalSocketIds(endpoint)); + } + + [Fact] + public async Task Rendezvous_RequestsReceivingSocket_ForDirectBootstrap() + { + await using var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); + var hints = new List<(IPEndPoint Endpoint, bool ForceFullHello, bool UseAllEligibleLocalSockets, int LocalSocketId)>(); + var manager = new ZeroTierDirectEndpointManager( + udp, + relay, + peerNodeId, + handleDirectEndpointHintAsync: (_, localSocketId, hintedEndpoint, forceFullHello, useAllEligibleLocalSockets, _) => + { + hints.Add((hintedEndpoint, forceFullHello, useAllEligibleLocalSockets, localSocketId)); + return ValueTask.CompletedTask; + }); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + + Assert.Single(hints); + Assert.Equal((endpoint, false, false, 1), hints[0]); + } + + [Fact] + public async Task RefreshPinnedRendezvousHolePunch_UsesRequestedSocket() + { + await using var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("100.64.0.40"), 4242); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandleRendezvousFromRootAsync( + BuildRendezvousPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: relay, + CancellationToken.None); + + manager.RefreshPinnedRendezvousHolePunch(endpoint, preferredLocalSocketIds: [0]); + + Assert.Equal(2, udp.Sends.Count); + var refreshSend = udp.Sends[1]; + Assert.Equal(0, refreshSend.LocalSocketId); + Assert.Equal(endpoint, refreshSend.RemoteEndPoint); + Assert.Equal(2, refreshSend.HopLimit); + } + + [Fact] + public async Task PushDirectPaths_DoesNotRememberReceivingSocketAffinity_ForNewEndpoints() + { + await using var udp = new RecordingUdpTransport(); + var relay = new IPEndPoint(IPAddress.Loopback, 9999); + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("203.0.113.40"), 4242); + var manager = new ZeroTierDirectEndpointManager(udp, relay, peerNodeId); + + await manager.HandlePushDirectPathsFromRemoteAsync( + ZeroTierPushDirectPathsCodec.BuildPayload(new[] { endpoint }), + receivedLocalSocketId: 1, + CancellationToken.None); + + Assert.Empty(manager.GetPreferredLocalSocketIds(endpoint)); + } + + private static byte[] BuildRendezvousPayload(NodeId with, IPEndPoint endpoint) + { + var addressBytes = endpoint.Address.GetAddressBytes(); + var payload = new byte[1 + 5 + 2 + 1 + addressBytes.Length]; + + payload[0] = 0; + ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, 5), with.Value); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(1 + 5, 2), (ushort)endpoint.Port); + payload[1 + 5 + 2] = (byte)addressBytes.Length; + addressBytes.CopyTo(payload.AsSpan(1 + 5 + 2 + 1)); + + return payload; + } + + private sealed class RecordingUdpTransport : IZeroTierUdpTransport + { + public IReadOnlyList LocalSockets { get; } = + [ + new(0, new IPEndPoint(IPAddress.Any, 10000)), + new(1, new IPEndPoint(IPAddress.Any, 10001)) + ]; + + public List Sends { get; } = new(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + { + Sends.Add(new RecordedSend(localSocketId, remoteEndpoint, HopLimit: null, payload.ToArray())); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + Sends.Add(new RecordedSend(localSocketId, remoteEndpoint, hopLimit, payload.ToArray())); + return Task.CompletedTask; + } + } + + private sealed record RecordedSend(int LocalSocketId, IPEndPoint RemoteEndPoint, int? HopLimit, byte[] Payload); +} diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs new file mode 100644 index 0000000..e42ab65 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectEndpointPolicyTests.cs @@ -0,0 +1,159 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectEndpointPolicyTests +{ + [Fact] + public void ShouldAccept_AcceptsPublicEndpoint_WhenLocalNetworksUnknown() + { + var policy = new ZeroTierDirectEndpointPolicy(Array.Empty()); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_RejectsPublicIpv6Endpoint_WithoutLocalIpv6Network() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("2606:4700:4700::1111"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsPublicIpv6Endpoint_WithLocalIpv6Network() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("2001:db8::1"), 64) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("2606:4700:4700::1111"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsPrivateEndpoint_OnMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("172.17.0.10"), 16) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_RejectsPrivateEndpoint_OutsideLocalNetworks() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsSharedEndpoint_OnMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("100.85.196.14"), 10) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_RejectsSharedEndpoint_OutsideLocalNetworks() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsSharedEndpoint_WhenLocalAddressIsAlsoInSharedSpace() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("100.74.185.14"), 32) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAdvertiseLocalEndpoint_AcceptsSharedSpacePeerOutsideExactSubnet() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("100.74.185.14"), 32) + ]); + + var accepted = policy.ShouldAdvertiseLocalEndpoint( + new IPEndPoint(IPAddress.Parse("100.74.185.14"), 50060), + [new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)]); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_AcceptsUlaEndpoint_OnMatchingLocalNetwork() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("fd7a:115c:a1e0::1"), 48) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldAccept_RejectsUlaEndpoint_OutsideLocalNetworks() + { + var policy = new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("fd00::1"), 64) + ]); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993)); + + Assert.False(accepted); + } + + [Fact] + public void ShouldAccept_RejectsLinkLocalEndpoint() + { + var policy = new ZeroTierDirectEndpointPolicy(Array.Empty()); + + var accepted = policy.ShouldAccept(new IPEndPoint(IPAddress.Parse("fe80::1"), 9993)); + + Assert.False(accepted); + } +} diff --git a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs index a1ea9a0..d5b7bfd 100644 --- a/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs +++ b/ZTSharp.Tests/ZeroTierDirectEndpointSelectionTests.cs @@ -7,7 +7,7 @@ namespace ZTSharp.Tests; public sealed class ZeroTierDirectEndpointSelectionTests { [Fact] - public void Normalize_CanonicalizesIpv4MappedIpv6_ForUniquenessAndOrdering() + public void Normalize_CanonicalizesIpv4MappedIpv6_AndPreservesFirstSeenOrder() { var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); var endpoints = new[] @@ -30,5 +30,76 @@ public void Normalize_CanonicalizesIpv4MappedIpv6_ForUniquenessAndOrdering() Assert.Equal(new IPEndPoint(IPAddress.Parse("8.8.8.8"), 9993), normalized[0]); Assert.Equal(new IPEndPoint(IPAddress.Parse("10.0.0.1"), 9993), normalized[1]); } + + + [Fact] + public void Normalize_PrioritizesGlobalCandidatesBeforePrivateCandidates() + { + var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); + var endpoints = new[] + { + new IPEndPoint(IPAddress.Parse("172.17.0.1"), 9993), + new IPEndPoint(IPAddress.Parse("172.18.0.1"), 9993), + new IPEndPoint(IPAddress.Parse("10.22.10.94"), 9993), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 48705), + }; + + var normalized = ZeroTierDirectEndpointSelection.Normalize(endpoints, relay, maxEndpoints: 3); + + Assert.Equal(new IPEndPoint(IPAddress.Parse("176.66.90.119"), 48705), normalized[0]); + Assert.Contains(normalized, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993))); + } + + [Fact] + public void Normalize_KeepsUpToPerScopeAndFamilyBudget() + { + var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); + var endpoints = Enumerable.Range(0, 10) + .Select(port => new IPEndPoint(IPAddress.Parse("176.66.90.119"), 40000 + port)) + .Concat(Enumerable.Range(0, 10) + .Select(port => new IPEndPoint(IPAddress.Parse("2001:4860:4860::8888"), 50000 + port))) + .ToArray(); + + var normalized = ZeroTierDirectEndpointSelection.Normalize( + endpoints, + relay, + maxEndpoints: 32, + maxPerScopeAndFamily: 8); + + Assert.Equal(16, normalized.Length); + Assert.Equal(8, normalized.Count(endpoint => endpoint.AddressFamily == AddressFamily.InterNetwork)); + Assert.Equal(8, normalized.Count(endpoint => endpoint.AddressFamily == AddressFamily.InterNetworkV6)); + } + + [Fact] + public void Normalize_KeepsLoopback_WhenCallerExplicitlyAcceptsIt() + { + var relay = new IPEndPoint(IPAddress.Parse("1.1.1.1"), 9993); + var loopback = new IPEndPoint(IPAddress.Loopback, 4242); + + var normalized = ZeroTierDirectEndpointSelection.Normalize( + new[] { loopback }, + relay, + maxEndpoints: 4, + shouldInclude: endpoint => endpoint.Equals(loopback)); + + Assert.Equal([loopback], normalized); + } + + [Theory] + [InlineData("10.0.0.1", true)] + [InlineData("100.85.196.109", true)] + [InlineData("172.17.0.1", true)] + [InlineData("192.168.0.1", true)] + [InlineData("176.66.90.119", true)] + [InlineData("169.254.1.2", false)] + [InlineData("127.0.0.1", false)] + public void IsUsablePathEndpoint_FollowsUpstreamPathScopeRules(string ip, bool expected) + { + var endpoint = new IPEndPoint(IPAddress.Parse(ip), 9993); + Assert.Equal(expected, ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)); + } } + diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs new file mode 100644 index 0000000..0ecd96a --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerRouteTests.cs @@ -0,0 +1,107 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectHintPathPlannerRouteTests +{ + [Fact] + public void GetPreferredAndFallbackSocketIds_UsesRouteSelectedSocket_ForPublicEndpoint() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("192.168.224.1"), 10001))); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => manager, + new ZeroTierDirectPathSocketAdmissibility( + new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24), + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("192.168.224.1"), 24) + ]), + new ZeroTierDirectRouteResolver(_ => IPAddress.Parse("10.0.0.112")))); + + var sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + + Assert.Equal([0], sockets); + } + + [Fact] + public void GetPreferredAndFallbackSocketIds_IncludesAdmissibleSockets_WithoutSurfaceObservations() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001)), + new ZeroTierUdpLocalSocket(2, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10002))); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => manager, + new ZeroTierDirectPathSocketAdmissibility( + new ZeroTierDirectEndpointPolicy( + [new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24)]), + new ZeroTierDirectRouteResolver(_ => IPAddress.Parse("10.0.0.112")))); + + var sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + + Assert.Equal([0, 1, 2], sockets); + } + + [Fact] + public void GetRotatingSocketIds_RotatesAcrossAdmissibleSockets_WithoutSurfaceObservations() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001)), + new ZeroTierUdpLocalSocket(2, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10002))); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId), + new ZeroTierDirectPathSocketAdmissibility( + new ZeroTierDirectEndpointPolicy( + [new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24)]), + new ZeroTierDirectRouteResolver(_ => IPAddress.Parse("10.0.0.112")))); + + var seen = new HashSet(); + for (var i = 0; i < 6; i++) + { + seen.Add(planner.GetRotatingSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true).Single()); + } + + Assert.Equal([0, 1, 2], seen.OrderBy(static id => id).ToArray()); + } + + private sealed class StubUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) : IZeroTierUdpTransport + { + public IReadOnlyList LocalSockets { get; } = localSockets; + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs new file mode 100644 index 0000000..bed5b42 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectHintPathPlannerTests.cs @@ -0,0 +1,128 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectHintPathPlannerTests +{ + [Fact] + public void GetNextHintedCandidatesForMaintenance_RotatesSockets_AndSkipsObservedPaths() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint1 = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + var endpoint2 = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var endpoint3 = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + manager.SeedEndpoints([endpoint1, endpoint2, endpoint3]); + + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => manager); + + var first = planner.GetNextHintedCandidatesForMaintenance( + peerNodeId, + [new ZeroTierPeerPhysicalPath(0, endpoint1, LastSeenUnixMs: 1)], + endpointBudget: 3); + var second = planner.GetNextHintedCandidatesForMaintenance(peerNodeId, Array.Empty(), endpointBudget: 2); + + Assert.Equal( + [new ZeroTierSelectedPeerPath(0, endpoint2), new ZeroTierSelectedPeerPath(0, endpoint3)], + first); + Assert.Equal( + [new ZeroTierSelectedPeerPath(1, endpoint1), new ZeroTierSelectedPeerPath(1, endpoint2)], + second); + } + + [Fact] + public void GetPreferredAndFallbackSocketIds_FiltersByAddressFamily() + { + var peerNodeId = new NodeId(0x1111111111); + var ipv4Endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + var ipv6Endpoint = new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("fd00::1"), 10001))); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner(udp, _ => manager); + + var ipv4Sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv4Endpoint); + var ipv6Sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv6Endpoint); + + Assert.Equal([0], ipv4Sockets); + Assert.Equal([1], ipv6Sockets); + } + + [Fact] + public void GetPreferredAndFallbackSocketIds_ReturnsEmpty_WhenNoSocketCanReachEndpoint() + { + var peerNodeId = new NodeId(0x1111111111); + var ipv6Endpoint = new IPEndPoint(IPAddress.Parse("fd7a:115c:a1e0::b401:c4a1"), 9993); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000))); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner(udp, _ => manager); + + var sockets = planner.GetPreferredAndFallbackSocketIds(peerNodeId, ipv6Endpoint); + + Assert.Empty(sockets); + } + + [Fact] + public async Task GetPreferredSocketIds_ReturnsOnlyPersistedPreferredSockets() + { + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 11830); + + var udp = new StubUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10001))); + var manager = new ZeroTierDirectEndpointManager(udp, new IPEndPoint(IPAddress.Loopback, 9993), peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner(udp, _ => manager); + + Assert.Empty(planner.GetPreferredSocketIds(peerNodeId, endpoint)); + + await manager.HandleRendezvousFromRootAsync( + ZeroTierRendezvousCodec.BuildPayload(peerNodeId, endpoint), + receivedLocalSocketId: 1, + receivedVia: new IPEndPoint(IPAddress.Loopback, 9993), + CancellationToken.None); + + Assert.Equal([1], planner.GetPreferredSocketIds(peerNodeId, endpoint)); + Assert.Equal([1], planner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint)); + } + + private sealed class StubUdpTransport : IZeroTierUdpTransport + { + public StubUdpTransport(params ZeroTierUdpLocalSocket[] localSockets) + { + LocalSockets = localSockets; + } + + public IReadOnlyList LocalSockets { get; } + + public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task SendWithHopLimitAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, int hopLimit, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs b/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs new file mode 100644 index 0000000..c1fa17c --- /dev/null +++ b/ZTSharp.Tests/ZeroTierDirectPathSocketAdmissibilityTests.cs @@ -0,0 +1,78 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierDirectPathSocketAdmissibilityTests +{ + [Fact] + public void ShouldUsePath_AllowsOnlySocketSelectedByRoute_ForPublicEndpoint() + { + var admissibility = CreateAdmissibility(endpoint => + endpoint.Address.Equals(IPAddress.Parse("176.66.90.119")) + ? IPAddress.Parse("10.0.0.112") + : null); + + var accepted = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + var rejected = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("192.168.224.1"), 10001)), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + + Assert.True(accepted); + Assert.False(rejected); + } + + [Fact] + public void ShouldUsePath_FallsBackToAddressFamily_WhenRouteUnknown() + { + var admissibility = CreateAdmissibility(_ => null); + + var accepted = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldUsePath_AcceptsSharedEndpoint_WhenRouteUsesAcceptedLocalAddress() + { + var admissibility = CreateAdmissibility(endpoint => + endpoint.Address.Equals(IPAddress.Parse("100.85.196.109")) + ? IPAddress.Parse("10.0.0.112") + : null); + + var accepted = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 10000)), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.True(accepted); + } + + [Fact] + public void ShouldUsePath_RejectsSharedEndpoint_WhenRouteUsesUnacceptedLocalAddress() + { + var admissibility = CreateAdmissibility(endpoint => + endpoint.Address.Equals(IPAddress.Parse("100.85.196.109")) + ? IPAddress.Parse("100.74.185.14") + : null); + + var rejected = admissibility.ShouldUsePath( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)); + + Assert.False(rejected); + } + + private static ZeroTierDirectPathSocketAdmissibility CreateAdmissibility(Func resolveLocalAddress) + => new( + new ZeroTierDirectEndpointPolicy( + [ + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24), + new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("192.168.224.1"), 24) + ]), + new ZeroTierDirectRouteResolver(resolveLocalAddress)); +} diff --git a/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs b/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs index a363d49..a2d1df3 100644 --- a/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs +++ b/ZTSharp.Tests/ZeroTierExternalSurfaceAddressTrackerTests.cs @@ -13,9 +13,21 @@ public void Observe_TracksDistinctSurfaceAddressesPerLocalSocket() var tracker = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromSeconds(10), nowUnixMs: () => now); var localSocketId = 2; - tracker.Observe(new NodeId(0x1111111111), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); - tracker.Observe(new NodeId(0x2222222222), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); - tracker.Observe(new NodeId(0x3333333333), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.2"), 10001)); + tracker.Observe( + new NodeId(0x1111111111), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.10"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + new NodeId(0x2222222222), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.11"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + new NodeId(0x3333333333), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.12"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.2"), 10001)); var snapshot = tracker.GetSnapshot(localSocketId); Assert.Equal(2, snapshot.Length); @@ -28,11 +40,36 @@ public void Observe_ExpiresEntries() var tracker = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromMilliseconds(100), nowUnixMs: () => now); var localSocketId = 2; - tracker.Observe(new NodeId(0x1111111111), localSocketId, new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + new NodeId(0x1111111111), + localSocketId, + new IPEndPoint(IPAddress.Parse("203.0.113.10"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); Assert.Single(tracker.GetSnapshot(localSocketId)); now += 2_000; Assert.Empty(tracker.GetSnapshot(localSocketId)); } + + [Fact] + public void Observe_KeepsDistinctReporterPathsForSamePeerAndSocket() + { + var tracker = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromSeconds(10)); + var peerNodeId = new NodeId(0x1111111111); + + tracker.Observe( + peerNodeId, + localSocketId: 2, + new IPEndPoint(IPAddress.Parse("203.0.113.10"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.1"), 10000)); + tracker.Observe( + peerNodeId, + localSocketId: 2, + new IPEndPoint(IPAddress.Parse("203.0.113.11"), 9993), + new IPEndPoint(IPAddress.Parse("198.51.100.2"), 10001)); + + var snapshot = tracker.GetSnapshot(localSocketId: 2); + Assert.Equal(2, snapshot.Length); + } } diff --git a/ZTSharp.Tests/ZeroTierHelloClientTests.cs b/ZTSharp.Tests/ZeroTierHelloClientTests.cs index f392bb0..408d3d3 100644 --- a/ZTSharp.Tests/ZeroTierHelloClientTests.cs +++ b/ZTSharp.Tests/ZeroTierHelloClientTests.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Net; +using System.Linq; using ZTSharp.ZeroTier.Internal; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -47,11 +48,14 @@ public async Task HelloRootsAsync_SendsHello_And_ParsesOk() Assert.Equal(rootIdentity.NodeId, ok.RootNodeId); Assert.Equal(rootEndpoint, ok.RootEndpoint); - Assert.Equal(11, ok.RemoteProtocolVersion); - Assert.Equal(1, ok.RemoteMajorVersion); - Assert.Equal(12, ok.RemoteMinorVersion); - Assert.Equal(0, ok.RemoteRevision); + Assert.Equal(ZeroTierHelloClient.AdvertisedProtocolVersion, ok.RemoteProtocolVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedMajorVersion, ok.RemoteMajorVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedMinorVersion, ok.RemoteMinorVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedRevision, ok.RemoteRevision); Assert.Equal(new IPEndPoint(IPAddress.Loopback, 9993), ok.ExternalSurfaceAddress); + var observation = Assert.Single(ok.ExternalSurfaceObservations); + Assert.Equal(0, observation.LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 9993), observation.SurfaceAddress); await serverTask; } @@ -139,7 +143,113 @@ public async Task HelloAsync_SendsHello_AndCompletesOnOk() timeout: TimeSpan.FromSeconds(2), cancellationToken: CancellationToken.None); - Assert.Equal(11, remoteProtocolVersion); + Assert.Equal(ZeroTierHelloClient.AdvertisedProtocolVersion, remoteProtocolVersion); + + await serverTask; + } + + [Fact] + public async Task HelloRootsAsync_WithMultiTransport_CollectsSurfaceObservationsPerSocket() + { + Assert.True(ZeroTierIdentity.TryParse(KnownGoodIdentity, out var localIdentity)); + Assert.NotNull(localIdentity.PrivateKey); + + var rootIdentity = new ZeroTierIdentity( + new NodeId(0x1122334455), + (byte[])localIdentity.PublicKey.Clone(), + (byte[])localIdentity.PrivateKey.Clone()); + + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true); + await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 0); + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 1); + await using var multi = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); + + var rootEndpoint = TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint); + var planet = new ZeroTierWorld( + ZeroTierWorldType.Planet, + id: 1, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: new[] + { + new ZeroTierWorldRoot(rootIdentity, new[] { rootEndpoint }) + }); + + var serverTask = RunHelloServerTwiceAsync(rootUdp, rootIdentity, localIdentity); + + var ok = await ZeroTierHelloClient.HelloRootsAsync( + multi, + localIdentity, + planet, + timeout: TimeSpan.FromSeconds(2), + cancellationToken: CancellationToken.None); + + Assert.Equal(2, ok.ExternalSurfaceObservations.Count); + Assert.Equal(0, ok.ExternalSurfaceObservations[0].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 11000), ok.ExternalSurfaceObservations[0].SurfaceAddress); + Assert.Equal(1, ok.ExternalSurfaceObservations[1].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 11001), ok.ExternalSurfaceObservations[1].SurfaceAddress); + + await serverTask; + } + + [Fact] + public async Task ProbeRootExternalSurfacesAsync_RetriesSocketsUntilAllMappingsAreObserved() + { + Assert.True(ZeroTierIdentity.TryParse(KnownGoodIdentity, out var localIdentity)); + Assert.NotNull(localIdentity.PrivateKey); + + var rootIdentity = new ZeroTierIdentity( + new NodeId(0x1122334455), + (byte[])localIdentity.PublicKey.Clone(), + (byte[])localIdentity.PrivateKey.Clone()); + + await using var rootUdp = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true); + await using var socket0 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 0); + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: true, localSocketId: 1); + await using var multi = new ZeroTierUdpMultiTransport(new[] { socket0, socket1 }); + + var rootEndpoint = TestUdpEndpoints.ToLoopback(rootUdp.LocalEndpoint); + var planet = new ZeroTierWorld( + ZeroTierWorldType.Planet, + id: 1, + timestamp: 1, + updatesMustBeSignedBy: new byte[ZeroTierWorld.C25519PublicKeyLength], + signature: new byte[ZeroTierWorld.C25519SignatureLength], + roots: new[] + { + new ZeroTierWorldRoot(rootIdentity, new[] { rootEndpoint }) + }); + + var rootKey = new byte[48]; + ZeroTierC25519.Agree(localIdentity.PrivateKey!, rootIdentity.PublicKey, rootKey); + + var firstSocketPort = socket0.LocalEndpoint.Port; + var delayedSocketPort = socket1.LocalEndpoint.Port; + var serverTask = RunHelloSurfaceProbeRetryServerAsync( + rootUdp, + rootIdentity, + localIdentity, + firstSocketPort, + delayedSocketPort); + + var observations = await ZeroTierHelloClient.ProbeRootExternalSurfacesAsync( + multi, + localIdentity, + planet, + rootIdentity.NodeId, + rootEndpoint, + rootKey, + knownObservations: Array.Empty(), + timeout: TimeSpan.FromSeconds(3), + cancellationToken: CancellationToken.None); + + Assert.Equal(2, observations.Count); + Assert.Equal(0, observations[0].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 12000), observations[0].SurfaceAddress); + Assert.Equal(1, observations[1].LocalSocketId); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 12001), observations[1].SurfaceAddress); await serverTask; } @@ -162,30 +272,13 @@ private static async Task RunHelloServerOnceAsync( var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian( helloPacketBytes.AsSpan(ZeroTierPacketHeader.Length + 5, 8)); - var surface = new IPEndPoint(IPAddress.Loopback, 9993); - var surfaceLength = ZeroTierInetAddressCodec.GetSerializedLength(surface); - - var okPayloadLength = 1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength; - var okPayload = new byte[okPayloadLength]; - okPayload[0] = (byte)ZeroTierVerb.Hello; - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1, 8), hello.Header.PacketId); - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(9, 8), helloTimestamp); - okPayload[17] = 11; - okPayload[18] = 1; - okPayload[19] = 12; - BinaryPrimitives.WriteUInt16BigEndian(okPayload.AsSpan(20, 2), 0); - _ = ZeroTierInetAddressCodec.Serialize(surface, okPayload.AsSpan(22)); - - var okHeader = new ZeroTierPacketHeader( - PacketId: 2, - Destination: localIdentity.NodeId, - Source: rootIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Ok); - - var okPacket = ZeroTierPacketCodec.Encode(okHeader, okPayload); - ZeroTierPacketCrypto.Armor(okPacket, sharedKey, encryptPayload: true); + var okPacket = BuildHelloOkPacket( + rootNodeId: rootIdentity.NodeId, + localNodeId: localIdentity.NodeId, + sharedKey: sharedKey, + inRePacketId: hello.Header.PacketId, + helloTimestampEcho: helloTimestamp, + surface: new IPEndPoint(IPAddress.Loopback, 9993)); await transport.SendAsync(helloDatagram.RemoteEndPoint, okPacket).ConfigureAwait(false); } @@ -244,37 +337,118 @@ private static async Task RunHelloRootsMismatchServerOnceAsync( await rootBUdp.SendAsync(helloBDatagram.RemoteEndPoint, okFromB).ConfigureAwait(false); } + private static async Task RunHelloServerTwiceAsync( + ZeroTierUdpTransport transport, + ZeroTierIdentity rootIdentity, + ZeroTierIdentity localIdentity) + { + var firstHelloTask = transport.ReceiveAsync(TimeSpan.FromSeconds(2)).AsTask(); + var secondHelloTask = transport.ReceiveAsync(TimeSpan.FromSeconds(2)).AsTask(); + + await Task.WhenAll(firstHelloTask, secondHelloTask).ConfigureAwait(false); + + var hellos = new[] { await firstHelloTask.ConfigureAwait(false), await secondHelloTask.ConfigureAwait(false) } + .OrderBy(static datagram => datagram.LocalSocketId) + .ThenBy(static datagram => datagram.RemoteEndPoint.Port) + .ToArray(); + + for (var i = 0; i < hellos.Length; i++) + { + var helloDatagram = hellos[i]; + var helloPacketBytes = helloDatagram.Payload; + Assert.True(ZeroTierPacketCodec.TryDecode(helloPacketBytes, out var hello)); + Assert.Equal(ZeroTierVerb.Hello, hello.Header.Verb); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(rootIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); + + Assert.True(ZeroTierPacketCrypto.Dearmor(helloPacketBytes, sharedKey)); + + var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian( + helloPacketBytes.AsSpan(ZeroTierPacketHeader.Length + 5, 8)); + + var okPacket = BuildHelloOkPacket( + rootNodeId: rootIdentity.NodeId, + localNodeId: localIdentity.NodeId, + sharedKey: sharedKey, + inRePacketId: hello.Header.PacketId, + helloTimestampEcho: helloTimestamp, + surface: new IPEndPoint(IPAddress.Loopback, 11000 + i)); + + await transport.SendAsync(helloDatagram.RemoteEndPoint, okPacket).ConfigureAwait(false); + } + } + + private static async Task RunHelloSurfaceProbeRetryServerAsync( + ZeroTierUdpTransport transport, + ZeroTierIdentity rootIdentity, + ZeroTierIdentity localIdentity, + int immediateSocketPort, + int delayedSocketPort) + { + var helloCountsByRemotePort = new Dictionary(); + var responsesSent = 0; + + while (responsesSent < 2) + { + var helloDatagram = await transport.ReceiveAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); + var helloPacketBytes = helloDatagram.Payload; + Assert.True(ZeroTierPacketCodec.TryDecode(helloPacketBytes, out var hello)); + Assert.Equal(ZeroTierVerb.Hello, hello.Header.Verb); + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(rootIdentity.PrivateKey!, localIdentity.PublicKey, sharedKey); + + Assert.True(ZeroTierPacketCrypto.Dearmor(helloPacketBytes, sharedKey)); + + var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian( + helloPacketBytes.AsSpan(ZeroTierPacketHeader.Length + 5, 8)); + + var remotePort = helloDatagram.RemoteEndPoint.Port; + helloCountsByRemotePort.TryGetValue(remotePort, out var helloCount); + helloCount++; + helloCountsByRemotePort[remotePort] = helloCount; + + IPEndPoint? surface = remotePort switch + { + _ when remotePort == immediateSocketPort && helloCount == 1 => new IPEndPoint(IPAddress.Loopback, 12000), + _ when remotePort == delayedSocketPort && helloCount == 2 => new IPEndPoint(IPAddress.Loopback, 12001), + _ => null + }; + + if (surface is null) + { + continue; + } + + var okPacket = BuildHelloOkPacket( + rootNodeId: rootIdentity.NodeId, + localNodeId: localIdentity.NodeId, + sharedKey: sharedKey, + inRePacketId: hello.Header.PacketId, + helloTimestampEcho: helloTimestamp, + surface: surface); + + await transport.SendAsync(helloDatagram.RemoteEndPoint, okPacket).ConfigureAwait(false); + responsesSent++; + } + } + private static byte[] BuildHelloOkPacket( NodeId rootNodeId, NodeId localNodeId, byte[] sharedKey, ulong inRePacketId, - ulong helloTimestampEcho) + ulong helloTimestampEcho, + IPEndPoint? surface = null) { - var surface = new IPEndPoint(IPAddress.Loopback, 9993); - var surfaceLength = ZeroTierInetAddressCodec.GetSerializedLength(surface); - - var okPayloadLength = 1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength; - var okPayload = new byte[okPayloadLength]; - okPayload[0] = (byte)ZeroTierVerb.Hello; - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1, 8), inRePacketId); - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(9, 8), helloTimestampEcho); - okPayload[17] = 11; - okPayload[18] = 1; - okPayload[19] = 12; - BinaryPrimitives.WriteUInt16BigEndian(okPayload.AsSpan(20, 2), 0); - _ = ZeroTierInetAddressCodec.Serialize(surface, okPayload.AsSpan(22)); - - var okHeader = new ZeroTierPacketHeader( - PacketId: 2, - Destination: localNodeId, - Source: rootNodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)ZeroTierVerb.Ok); - - var okPacket = ZeroTierPacketCodec.Encode(okHeader, okPayload); - ZeroTierPacketCrypto.Armor(okPacket, sharedKey, encryptPayload: true); - return okPacket; + return ZeroTierHelloOkPacketBuilder.BuildPacket( + packetId: 2, + destination: localNodeId, + source: rootNodeId, + inRePacketId: inRePacketId, + helloTimestampEcho: helloTimestampEcho, + externalSurfaceAddress: surface ?? new IPEndPoint(IPAddress.Loopback, 9993), + sharedKey: sharedKey); } } diff --git a/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs b/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs index f6fa031..d1c2550 100644 --- a/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs +++ b/ZTSharp.Tests/ZeroTierHelloOkPacketBuilderTests.cs @@ -54,7 +54,8 @@ public void BuildPacket_EncodesOkHello() Assert.True(ZeroTierInetAddressCodec.TryDeserialize(payload.Slice(22), out var parsedSurface, out var read)); Assert.Equal(surface, parsedSurface); - Assert.Equal(payload.Length - 22, read); + Assert.Equal(payload.Length - 24, read); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(22 + read, 2))); } } diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs new file mode 100644 index 0000000..1276ede --- /dev/null +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementPlannerTests.cs @@ -0,0 +1,66 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierLocalDirectPathAdvertisementPlannerTests +{ + private static readonly IPEndPoint PublicLocal = new(IPAddress.Parse("212.241.85.84"), 50060); + private static readonly IPEndPoint PrivateLocal = new(IPAddress.Parse("10.0.0.112"), 50060); + + [Fact] + public void SelectForPeer_DropsPrivateLocalAdvertisements_ForPublicPeer() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [new IPEndPoint(IPAddress.Parse("176.66.90.119"), 14849)], + []); + + Assert.Equal([PublicLocal], selected); + } + + [Fact] + public void SelectForPeer_KeepsMatchingPrivateLocalAdvertisements_ForSameSubnetPeer() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [new IPEndPoint(IPAddress.Parse("10.0.0.7"), 14849)], + []); + + Assert.Equal([PublicLocal, PrivateLocal], selected); + } + + [Fact] + public void SelectForPeer_KeepsPrivateLocalAdvertisements_ForSharedOrPrivatePeerHints() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993)], + []); + + Assert.Equal([PublicLocal, PrivateLocal], selected); + } + + [Fact] + public void SelectForPeer_DropsPrivateLocalAdvertisements_WhenPeerEndpointsUnknown() + { + var planner = CreatePlanner(); + + var selected = planner.SelectForPeer( + [PublicLocal, PrivateLocal], + [], + []); + + Assert.Equal([PublicLocal], selected); + } + + private static ZeroTierLocalDirectPathAdvertisementPlanner CreatePlanner() + => new(new ZeroTierDirectEndpointPolicy( + [new ZeroTierDirectEndpointPolicy.LocalNetwork(IPAddress.Parse("10.0.0.112"), 24)])); +} diff --git a/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs new file mode 100644 index 0000000..236e84e --- /dev/null +++ b/ZTSharp.Tests/ZeroTierLocalDirectPathAdvertisementSourceTests.cs @@ -0,0 +1,56 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierLocalDirectPathAdvertisementSourceTests +{ + [Fact] + public void GetSnapshot_ExcludesNonPublicLocalAddresses() + { + var source = new ZeroTierLocalDirectPathAdvertisementSource( + getLocalAddresses: () => + [ + IPAddress.Parse("212.241.85.84"), + IPAddress.Parse("100.74.185.14"), + IPAddress.Parse("10.5.0.2"), + IPAddress.Parse("172.25.128.1"), + IPAddress.Parse("192.168.224.1"), + IPAddress.Parse("fd7a:115c:a1e0::5132:b90e") + ]); + + var endpoints = source.GetSnapshot( + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 51158)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 51159)) + ]); + + Assert.Contains(endpoints, endpoint => endpoint.Equals(new IPEndPoint(IPAddress.Parse("212.241.85.84"), 51158))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("100.74.185.14"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("10.5.0.2"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("172.25.128.1"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("192.168.224.1"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("fd7a:115c:a1e0::5132:b90e"))); + } + + [Fact] + public void GetSnapshot_DoesNotProjectPublicHostAddressesOntoPrivateBoundSockets() + { + var source = new ZeroTierLocalDirectPathAdvertisementSource( + getLocalAddresses: () => + [ + IPAddress.Parse("212.241.85.84"), + IPAddress.Parse("198.51.100.10") + ]); + + var endpoints = source.GetSnapshot( + [ + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 62984)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Parse("10.0.0.112"), 62985)) + ]); + + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("212.241.85.84"))); + Assert.DoesNotContain(endpoints, endpoint => endpoint.Address.Equals(IPAddress.Parse("198.51.100.10"))); + } +} diff --git a/ZTSharp.Tests/ZeroTierNetworkCredentialsPacketBuilderTests.cs b/ZTSharp.Tests/ZeroTierNetworkCredentialsPacketBuilderTests.cs new file mode 100644 index 0000000..d49dbae --- /dev/null +++ b/ZTSharp.Tests/ZeroTierNetworkCredentialsPacketBuilderTests.cs @@ -0,0 +1,33 @@ +using System.Buffers.Binary; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Protocol; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierNetworkCredentialsPacketBuilderTests +{ + [Fact] + public void BuildPacket_EncodesInlineComAndEmptyCredentialLists() + { + var sharedKey = Enumerable.Range(1, 48).Select(static value => (byte)value).ToArray(); + var inlineCom = new byte[] { 0x7f, 0x01, 0x02, 0x03 }; + + var packet = ZeroTierNetworkCredentialsPacketBuilder.BuildPacket( + packetId: 0x0102030405060708, + destination: new NodeId(0x1111111111), + source: new NodeId(0x2222222222), + inlineCom: inlineCom, + sharedKey: sharedKey); + + Assert.True(ZeroTierPacketCrypto.Dearmor(packet, sharedKey)); + var payload = packet.AsSpan(ZeroTierPacketHeader.IndexPayload); + Assert.True(payload.Slice(0, inlineCom.Length).SequenceEqual(inlineCom)); + Assert.Equal(0, payload[inlineCom.Length]); + + var counts = payload.Slice(inlineCom.Length + 1); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(0, 2))); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(2, 2))); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(4, 2))); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(counts.Slice(6, 2))); + } +} diff --git a/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs b/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs index 98450d8..985adc1 100644 --- a/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerBondPolicyEngineTests.cs @@ -82,6 +82,46 @@ public void ActiveBackup_PrefersLowestLatency() Assert.Equal(epFast, selected.RemoteEndPoint); } + [Fact] + public void ActiveBackup_IsSticky_AndSwitchesAfterHold() + { + var peer = new NodeId(0x1111111111); + var epA = new IPEndPoint(IPAddress.Parse("203.0.113.1"), 1001); + var epB = new IPEndPoint(IPAddress.Parse("203.0.113.2"), 1002); + + var paths = new[] + { + new ZeroTierPeerPhysicalPath(LocalSocketId: 1, epA, LastSeenUnixMs: 1), + new ZeroTierPeerPhysicalPath(LocalSocketId: 2, epB, LastSeenUnixMs: 1), + }; + + var now = 1_000L; + var latency = new Dictionary + { + [epA] = 10, + [epB] = 100 + }; + + var engine = new ZeroTierPeerBondPolicyEngine( + getLatencyMs: (_, _, ep) => latency.TryGetValue(ep, out var l) ? l : null, + getRemoteUtility: (_, _, _) => 0, + nowMs: () => now); + + Assert.True(engine.TrySelectSinglePath(peer, (ZeroTierPeerPhysicalPath[])paths.Clone(), flowId: 0, ZeroTierBondPolicy.ActiveBackup, out var s0)); + Assert.Equal(epA, s0.RemoteEndPoint); + + latency[epA] = 200; + latency[epB] = 10; + now += 1_000; + + Assert.True(engine.TrySelectSinglePath(peer, (ZeroTierPeerPhysicalPath[])paths.Clone(), flowId: 0, ZeroTierBondPolicy.ActiveBackup, out var s1)); + Assert.Equal(epA, s1.RemoteEndPoint); + + now += 20_000; + Assert.True(engine.TrySelectSinglePath(peer, (ZeroTierPeerPhysicalPath[])paths.Clone(), flowId: 0, ZeroTierBondPolicy.ActiveBackup, out var s2)); + Assert.Equal(epB, s2.RemoteEndPoint); + } + [Fact] public void BalanceAware_ChoosesOnlyWithinSlack() { @@ -134,4 +174,23 @@ public void Broadcast_ReturnsStableSortedPaths() Assert.Equal(2, broadcast[1].LocalSocketId); Assert.Equal(epB, broadcast[1].RemoteEndPoint); } + + [Fact] + public void Broadcast_SortsLinkLocalEndpointsByScopeId() + { + var addrBytes = IPAddress.Parse("fe80::1").GetAddressBytes(); + var epScope2 = new IPEndPoint(new IPAddress(addrBytes, scopeid: 2), 1001); + var epScope1 = new IPEndPoint(new IPAddress(addrBytes, scopeid: 1), 1001); + + var paths = new[] + { + new ZeroTierPeerPhysicalPath(LocalSocketId: 1, epScope2, LastSeenUnixMs: 1), + new ZeroTierPeerPhysicalPath(LocalSocketId: 1, epScope1, LastSeenUnixMs: 1), + }; + + var broadcast = ZeroTierPeerBondPolicyEngine.GetBroadcastPaths((ZeroTierPeerPhysicalPath[])paths.Clone()); + Assert.Equal(2, broadcast.Length); + Assert.Equal(epScope1, broadcast[0].RemoteEndPoint); + Assert.Equal(epScope2, broadcast[1].RemoteEndPoint); + } } diff --git a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs index 434b4ae..8c0215d 100644 --- a/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerEchoManagerTests.cs @@ -26,13 +26,17 @@ public async Task EchoProbe_UsesOnWirePacketId_AndUpdatesRttOnOk() Assert.Single(udp.Sends); Assert.Equal(1, udp.Sends[0].LocalSocketId); Assert.Equal(endpoint, udp.Sends[0].RemoteEndPoint); + var sentBytes = udp.Sends[0].Payload.ToArray(); + Assert.True(ZeroTierPacketCrypto.Dearmor(sentBytes, sharedKey)); + Assert.True(ZeroTierPacketCodec.TryDecode(sentBytes, out var sent)); + Assert.Equal(ZeroTierVerb.Echo, sent.Header.Verb); + Assert.Empty(sent.Payload.ToArray()); var onWireEchoPacketId = BinaryPrimitives.ReadUInt64BigEndian(udp.Sends[0].Payload.Span.Slice(0, 8)); now += 25; - Span okTail = stackalloc byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(okTail, 1_000UL); - mgr.HandleEchoOk(peerNodeId, localSocketId: 1, endpoint, onWireEchoPacketId, okTail); + Span okTail = stackalloc byte[0]; + Assert.True(mgr.HandleEchoOk(peerNodeId, localSocketId: 1, endpoint, onWireEchoPacketId, okTail)); Assert.True(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, endpoint, out var rttMs)); Assert.Equal(25, rttMs); @@ -42,7 +46,7 @@ public async Task EchoProbe_UsesOnWirePacketId_AndUpdatesRttOnOk() } [Fact] - public async Task HandleEchoRequest_SendsOkEchoWithTimestampEcho() + public async Task HandleEchoRequest_SendsOkEchoWithPayloadPassthrough() { var udp = new RecordingUdpTransport(); var localNodeId = new NodeId(0x2222222222); @@ -74,12 +78,70 @@ await mgr.HandleEchoRequestAsync( Assert.Equal(ZeroTierVerb.Ok, verb); var payload = decrypted.AsSpan(ZeroTierPacketHeader.IndexPayload); - Assert.True(payload.Length >= 1 + 8 + 8); + Assert.True(payload.Length >= 1 + 8); Assert.Equal(ZeroTierVerb.Echo, (ZeroTierVerb)(payload[0] & 0x1F)); Assert.Equal(123UL, BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(1, 8))); Assert.Equal(777UL, BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(1 + 8, 8))); } + [Fact] + public async Task HandleEchoRequest_SendsOkEchoForEmptyPayload() + { + var udp = new RecordingUdpTransport(); + var localNodeId = new NodeId(0x2222222222); + var peerNodeId = new NodeId(0x1111111111); + var mgr = new ZeroTierPeerEchoManager(udp, localNodeId, getPeerProtocolVersion: _ => 12, nowUnixMs: () => 1_000); + + var endpoint = new IPEndPoint(IPAddress.Loopback, 5555); + var sharedKey = Enumerable.Repeat((byte)7, 48).ToArray(); + + await mgr.HandleEchoRequestAsync( + peerNodeId, + localSocketId: 2, + endpoint, + inRePacketId: 123, + ReadOnlyMemory.Empty, + sharedKey, + CancellationToken.None); + + Assert.Single(udp.Sends); + + var decrypted = udp.Sends[0].Payload.ToArray(); + Assert.True(ZeroTierPacketCrypto.Dearmor(decrypted, sharedKey)); + + var payload = decrypted.AsSpan(ZeroTierPacketHeader.IndexPayload); + Assert.True(payload.Length >= 1 + 8); + Assert.Equal(ZeroTierVerb.Echo, (ZeroTierVerb)(payload[0] & 0x1F)); + Assert.Equal(123UL, BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(1, 8))); + Assert.Equal(1 + 8, payload.Length); + } + + [Fact] + public async Task EchoOk_AcceptsReplyKeyMatch_FromRemappedEndpoint() + { + var udp = new RecordingUdpTransport(); + + var now = 1_000L; + var localNodeId = new NodeId(0x2222222222); + var peerNodeId = new NodeId(0x1111111111); + var mgr = new ZeroTierPeerEchoManager(udp, localNodeId, getPeerProtocolVersion: _ => 12, nowUnixMs: () => now); + + var sendEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4242); + var replyEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 4243); + var sharedKey = Enumerable.Repeat((byte)7, 48).ToArray(); + + await mgr.TrySendEchoProbeAsync(peerNodeId, localSocketId: 1, sendEndpoint, sharedKey, CancellationToken.None); + + var onWireEchoPacketId = BinaryPrimitives.ReadUInt64BigEndian(udp.Sends[0].Payload.Span.Slice(0, 8)); + var remappedReplyPacketId = (onWireEchoPacketId & 0xffffffff00000000UL) | 0x00000000000000a5UL; + + now += 25; + Assert.True(mgr.HandleEchoOk(peerNodeId, localSocketId: 1, replyEndpoint, remappedReplyPacketId, ReadOnlySpan.Empty)); + + Assert.True(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, replyEndpoint, out var rttMs)); + Assert.Equal(25, rttMs); + Assert.False(mgr.TryGetLastRttMs(peerNodeId, localSocketId: 1, sendEndpoint, out _)); + } private sealed class RecordingUdpTransport : IZeroTierUdpTransport { public List Sends { get; } = new(); @@ -98,12 +160,24 @@ public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, C public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) { - Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload)); + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, HopLimit: null)); + return Task.CompletedTask; + } + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + Sends.Add(new SendCall(localSocketId, remoteEndpoint, payload, hopLimit)); return Task.CompletedTask; } public ValueTask DisposeAsync() => ValueTask.CompletedTask; } - private readonly record struct SendCall(int LocalSocketId, IPEndPoint RemoteEndPoint, ReadOnlyMemory Payload); + private readonly record struct SendCall(int LocalSocketId, IPEndPoint RemoteEndPoint, ReadOnlyMemory Payload, int? HopLimit); } + diff --git a/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs index 48aac34..cdb3bea 100644 --- a/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerPathNegotiationManagerTests.cs @@ -21,6 +21,22 @@ public void HandleInboundRequest_StoresUtilityPerPath() Assert.Equal((short)123, utility); } + [Fact] + public void TryGetRemoteUtility_DoesNotReturnStaleUtilityBeyondTtl() + { + var now = 1_000L; + var mgr = new ZeroTierPeerPathNegotiationManager(nowMs: () => now); + + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("203.0.113.9"), 9999); + + mgr.HandleInboundRequest(peerNodeId, localSocketId: 2, endpoint, remoteUtility: 123); + + now += 300_001; + + Assert.False(mgr.TryGetRemoteUtility(peerNodeId, localSocketId: 2, endpoint, out _)); + } + [Fact] public void TryMarkSent_RateLimitsPerPath() { diff --git a/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs b/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs index 3c5d4a6..a7c7559 100644 --- a/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerPhysicalPathTrackerTests.cs @@ -37,4 +37,19 @@ public void ObserveHop0_TracksMultipleSocketsAndEndpoints() Assert.Contains(snapshot, p => p.LocalSocketId == 0 && p.RemoteEndPoint.Port == 10000); Assert.Contains(snapshot, p => p.LocalSocketId == 1 && p.RemoteEndPoint.Port == 10001); } + + [Fact] + public void ObserveKnownHop0_DoesNotPromoteUnknownPaths() + { + var tracker = new ZeroTierPeerPhysicalPathTracker(ttl: TimeSpan.FromSeconds(10)); + var peer = new NodeId(0x1111111111); + var path = new IPEndPoint(IPAddress.Loopback, 10000); + + Assert.False(tracker.ObserveKnownHop0(peer, localSocketId: 0, path)); + Assert.Empty(tracker.GetSnapshot(peer)); + + Assert.True(tracker.ObserveHop0(peer, localSocketId: 0, path)); + Assert.True(tracker.ObserveKnownHop0(peer, localSocketId: 0, path)); + Assert.Single(tracker.GetSnapshot(peer)); + } } diff --git a/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs b/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs index 7463ef6..8a2fae4 100644 --- a/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs +++ b/ZTSharp.Tests/ZeroTierPeerQosManagerTests.cs @@ -71,4 +71,25 @@ public void TryBuildOutboundPayload_EncodesLittleEndianPacketIdAndHoldingTime() Assert.False(mgr.TryBuildOutboundPayload(peerNodeId, localSocketId: 1, endpoint, out _)); } + + [Fact] + public void TryGetLastLatencyAverageMs_ReturnsFalse_WhenAverageIsStale() + { + var now = 1_000L; + var mgr = new ZeroTierPeerQosManager(nowMs: () => now); + + var peerNodeId = new NodeId(0x1111111111); + var endpoint = new IPEndPoint(IPAddress.Parse("203.0.113.9"), 9999); + + mgr.RecordOutgoingPacket(peerNodeId, localSocketId: 1, endpoint, packetId: 3); + + now = 1_500; + Span payload = stackalloc byte[8 + 2]; + BinaryPrimitives.WriteUInt64LittleEndian(payload.Slice(0, 8), 3UL); + BinaryPrimitives.WriteUInt16LittleEndian(payload.Slice(8, 2), 200); + mgr.HandleInboundMeasurement(peerNodeId, localSocketId: 1, endpoint, payload); + + now = 1_500 + 120_000 + 1; + Assert.False(mgr.TryGetLastLatencyAverageMs(peerNodeId, localSocketId: 1, endpoint, out _)); + } } diff --git a/ZTSharp.Tests/ZeroTierPeerRootSocketAffinityTests.cs b/ZTSharp.Tests/ZeroTierPeerRootSocketAffinityTests.cs new file mode 100644 index 0000000..1715352 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPeerRootSocketAffinityTests.cs @@ -0,0 +1,45 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPeerRootSocketAffinityTests +{ + [Fact] + public void Observe_IgnoresNonRootEndpoints() + { + var peerNodeId = new NodeId(0x1111111111); + var affinity = new ZeroTierPeerRootSocketAffinity(new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993)); + + affinity.Observe(peerNodeId, localSocketId: 4, new IPEndPoint(IPAddress.Parse("176.66.90.119"), 35998)); + + Assert.False(affinity.TryGet(peerNodeId, out _)); + } + + [Fact] + public void Observe_RootEndpoint_RemembersLatestSocket() + { + var peerNodeId = new NodeId(0x1111111111); + var rootEndpoint = new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993); + var affinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); + + affinity.Observe(peerNodeId, localSocketId: 3, rootEndpoint); + affinity.Observe(peerNodeId, localSocketId: 6, rootEndpoint); + + Assert.True(affinity.TryGet(peerNodeId, out var localSocketId)); + Assert.Equal(6, localSocketId); + } + + [Fact] + public void Remove_ClearsRememberedSocket() + { + var peerNodeId = new NodeId(0x1111111111); + var rootEndpoint = new IPEndPoint(IPAddress.Parse("84.17.53.155"), 9993); + var affinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); + + affinity.Observe(peerNodeId, localSocketId: 6, rootEndpoint); + affinity.Remove(peerNodeId); + + Assert.False(affinity.TryGet(peerNodeId, out _)); + } +} diff --git a/ZTSharp.Tests/ZeroTierPinnedRendezvousAlternateHintPolicyTests.cs b/ZTSharp.Tests/ZeroTierPinnedRendezvousAlternateHintPolicyTests.cs new file mode 100644 index 0000000..6db8e83 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPinnedRendezvousAlternateHintPolicyTests.cs @@ -0,0 +1,36 @@ +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPinnedRendezvousAlternateHintPolicyTests +{ + [Fact] + public void ShouldProbe_PublicAlternateHint_RequiresPinnedPublicSibling() + { + Assert.True( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: true, + hasPinnedPublicSibling: true, + hasUsableSocketPath: true)); + Assert.False( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: true, + hasPinnedPublicSibling: false, + hasUsableSocketPath: true)); + } + + [Fact] + public void ShouldProbe_NonPublicAlternateHint_UsesSocketAdmissibility() + { + Assert.True( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: false, + hasPinnedPublicSibling: false, + hasUsableSocketPath: true)); + Assert.False( + ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic: false, + hasPinnedPublicSibling: true, + hasUsableSocketPath: false)); + } +} diff --git a/ZTSharp.Tests/ZeroTierPinnedRendezvousSiblingPortPredictorTests.cs b/ZTSharp.Tests/ZeroTierPinnedRendezvousSiblingPortPredictorTests.cs new file mode 100644 index 0000000..6ba2219 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPinnedRendezvousSiblingPortPredictorTests.cs @@ -0,0 +1,40 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPinnedRendezvousSiblingPortPredictorTests +{ + [Fact] + public void Expand_AddsAdjacentPortsForPinnedPublicRendezvous() + { + var pinned = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7395); + var sibling = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + + var expanded = ZeroTierPinnedRendezvousSiblingPortPredictor.Expand( + [pinned, sibling], + endpoint => endpoint.Equals(pinned)); + + Assert.Equal( + [ + pinned, + sibling, + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7394), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7396), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7393), + new IPEndPoint(IPAddress.Parse("176.66.90.119"), 7397) + ], expanded); + } + + [Fact] + public void Expand_DoesNotAddAdjacentPortsForPinnedPrivateRendezvous() + { + var pinned = new IPEndPoint(IPAddress.Parse("10.22.10.94"), 9993); + + var expanded = ZeroTierPinnedRendezvousSiblingPortPredictor.Expand( + [pinned], + endpoint => endpoint.Equals(pinned)); + + Assert.Equal([pinned], expanded); + } +} diff --git a/ZTSharp.Tests/ZeroTierPredictedPublicSurfaceGeneratorTests.cs b/ZTSharp.Tests/ZeroTierPredictedPublicSurfaceGeneratorTests.cs new file mode 100644 index 0000000..7f3f20c --- /dev/null +++ b/ZTSharp.Tests/ZeroTierPredictedPublicSurfaceGeneratorTests.cs @@ -0,0 +1,54 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierPredictedPublicSurfaceGeneratorTests +{ + [Fact] + public void Expand_AddsAdjacentPortsForPublicEndpoints() + { + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand( + [ + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000) + ]); + + Assert.Equal( + [ + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60000), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 59999), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60001), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 59998), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 60002) + ], expanded); + } + + [Fact] + public void Expand_DoesNotAddAdjacentPortsForNonPublicEndpoints() + { + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand( + [ + new IPEndPoint(IPAddress.Parse("10.0.0.60"), 60000), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 60001) + ]); + + Assert.Equal( + [ + new IPEndPoint(IPAddress.Parse("10.0.0.60"), 60000), + new IPEndPoint(IPAddress.Parse("100.85.196.109"), 60001) + ], expanded); + } + + [Fact] + public void Expand_SkipsOutOfRangeAdjacentPorts() + { + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand( + [ + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 1), + new IPEndPoint(IPAddress.Parse("212.241.85.84"), 65535) + ]); + + Assert.DoesNotContain(expanded, endpoint => endpoint.Port == 0); + Assert.DoesNotContain(expanded, endpoint => endpoint.Port > ushort.MaxValue); + } +} diff --git a/ZTSharp.Tests/ZeroTierReplyCorrelationTests.cs b/ZTSharp.Tests/ZeroTierReplyCorrelationTests.cs new file mode 100644 index 0000000..eb3ed35 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierReplyCorrelationTests.cs @@ -0,0 +1,17 @@ +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierReplyCorrelationTests +{ + [Fact] + public void Matches_UsesUpper32Bits() + { + const ulong expectedPacketId = 0x0123456789abcdefUL; + const ulong matchingReplyPacketId = 0x01234567000000aaUL; + const ulong differentReplyPacketId = 0x89abcdef000000aaUL; + + Assert.True(ZeroTierReplyCorrelation.Matches(expectedPacketId, matchingReplyPacketId)); + Assert.False(ZeroTierReplyCorrelation.Matches(expectedPacketId, differentReplyPacketId)); + } +} diff --git a/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs b/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs index bd86d1f..7cc21be 100644 --- a/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs +++ b/ZTSharp.Tests/ZeroTierSizeCapHardeningTests.cs @@ -52,11 +52,12 @@ public async Task PeerSecurity_DropsOversizedHello_WithoutCachingKeys() networkId: 1, inlineCom: Array.Empty()); - var peerSecurity = new ZeroTierDataplanePeerSecurity(udp, rootClient, localIdentity); + using var peerSecurity = new ZeroTierDataplanePeerSecurity(udp, rootClient, localIdentity); var packetBytes = new byte[ZeroTierProtocolLimits.MaxPacketBytes + 1]; await peerSecurity.HandleHelloAsync( peerNodeId: new NodeId(0xaaaaaaaaaa), + localSocketId: 0, helloPacketId: 1, packetBytes: packetBytes, remoteEndPoint: new IPEndPoint(IPAddress.Loopback, 12345), diff --git a/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs b/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs new file mode 100644 index 0000000..c35f8be --- /dev/null +++ b/ZTSharp.Tests/ZeroTierSocketFactoryMultipathValidationTests.cs @@ -0,0 +1,45 @@ +using ZTSharp.ZeroTier; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierSocketFactoryMultipathValidationTests +{ + [Fact] + public async Task CreateAsync_RejectsDuplicateNonZeroLocalUdpPorts() + { + var options = new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-state-root-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = true, + UdpSocketCount = 2, + LocalUdpPorts = new[] { 12345, 12345 } + } + }; + + await Assert.ThrowsAsync(() => + ZeroTierSocketFactory.CreateAsync(options, CancellationToken.None)); + } + + [Fact] + public async Task CreateAsync_MultipathDisabled_AllowsDuplicateNonZeroLocalUdpPorts() + { + var options = new ZeroTierSocketOptions + { + StateRootPath = TestTempPaths.CreateGuidSuffixed("zt-state-root-"), + NetworkId = 1, + Multipath = new ZeroTierMultipathOptions + { + Enabled = false, + UdpSocketCount = 2, + LocalUdpPorts = new[] { 12345, 12345 } + } + }; + + await using var socket = await ZeroTierSocketFactory.CreateAsync(options, CancellationToken.None); + Assert.NotNull(socket); + } +} diff --git a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs index ad3e546..232e5cc 100644 --- a/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierSocketRuntimeBootstrapperUdpTransportTests.cs @@ -1,5 +1,8 @@ +using System.Net; +using System.Net.Sockets; using ZTSharp.ZeroTier; using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; namespace ZTSharp.Tests; @@ -8,7 +11,7 @@ public sealed class ZeroTierSocketRuntimeBootstrapperUdpTransportTests [Fact] public async Task CreateUdpTransport_Default_ReturnsSingleSocket() { - var transport = ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport(new ZeroTierMultipathOptions(), enableIpv6: false); + var transport = await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync(new ZeroTierMultipathOptions(), enableIpv6: false); try { Assert.Single(transport.LocalSockets); @@ -22,7 +25,7 @@ public async Task CreateUdpTransport_Default_ReturnsSingleSocket() [Fact] public async Task CreateUdpTransport_MultipathEnabled_UsesMultipleSockets() { - var transport = ZeroTierSocketRuntimeBootstrapper.CreateUdpTransport( + var transport = await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 3 }, enableIpv6: false); @@ -36,4 +39,138 @@ public async Task CreateUdpTransport_MultipathEnabled_UsesMultipleSockets() await transport.DisposeAsync(); } } + + [Fact] + public async Task CreateUdpTransport_MultipathEnabled_RequiresAtLeastOneSocket() + { + await Assert.ThrowsAsync(async () => + await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 0 }, + enableIpv6: false)); + } + + [Fact] + public async Task CreateUdpTransport_SingleSocket_HonorsLocalUdpPorts() + { + const int attempts = 25; + for (var i = 0; i < attempts; i++) + { + var port = GetAvailableUdpPort(); + var transport = default(IZeroTierUdpTransport); + try + { + transport = await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 1, LocalUdpPorts = new[] { port } }, + enableIpv6: false); + + Assert.Single(transport.LocalSockets); + Assert.Equal(port, transport.LocalSockets[0].LocalEndpoint.Port); + return; + } + catch (SocketException) + { + } + finally + { + if (transport is not null) + { + await transport.DisposeAsync(); + } + } + } + + Assert.Fail($"Failed to bind a UDP socket to an available port after {attempts} attempts."); + } + + [Fact] + public async Task CreateUdpTransport_MultipathEnabled_RejectsDuplicateNonZeroLocalUdpPorts() + { + await Assert.ThrowsAsync(async () => + await ZeroTierSocketRuntimeBootstrapper.CreateUdpTransportAsync( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 2, LocalUdpPorts = new[] { 12345, 12345 } }, + enableIpv6: false)); + } + + [Fact] + public void CreateUdpSocketBindings_MultipathEnabled_UsesDiscoveredBindAddressesBeforeWildcard() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + getLocalBindAddresses: () => + [ + IPAddress.Parse("192.0.2.10"), + IPAddress.Parse("2001:db8::10") + ]); + + Assert.Collection( + bindings, + binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress), + binding => Assert.Equal(IPAddress.Parse("2001:db8::10"), binding.LocalAddress), + binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress), + binding => Assert.Equal(IPAddress.Parse("2001:db8::10"), binding.LocalAddress)); + } + + [Fact] + public void CreateUdpSocketBindings_WithPreferredAddressFamily_ReusesMatchingAddressesOnly() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + preferredBindAddressFamily: AddressFamily.InterNetwork, + getLocalBindAddresses: () => + [ + IPAddress.Parse("192.0.2.10"), + IPAddress.Parse("2001:db8::10") + ]); + + Assert.All(bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress)); + } + + [Fact] + public void CreateUdpSocketBindings_WithPreferredLocalBindAddress_ReusesRoutedAddressOnly() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + preferredBindAddressFamily: AddressFamily.InterNetwork, + preferredLocalBindAddress: IPAddress.Parse("192.0.2.11"), + getLocalBindAddresses: () => + [ + IPAddress.Parse("192.0.2.10"), + IPAddress.Parse("192.0.2.11"), + IPAddress.Parse("2001:db8::10") + ]); + + Assert.All(bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.11"), binding.LocalAddress)); + } + + [Fact] + public void CreateUdpSocketBindings_WithUnavailablePreferredAddressFamily_KeepsOriginalRotation() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 4 }, + enableIpv6: true, + preferredBindAddressFamily: AddressFamily.InterNetworkV6, + getLocalBindAddresses: () => [IPAddress.Parse("192.0.2.10")]); + + Assert.All(bindings, binding => Assert.Equal(IPAddress.Parse("192.0.2.10"), binding.LocalAddress)); + } + + [Fact] + public void CreateUdpSocketBindings_NoDiscoveredAddresses_FallsBackToWildcard() + { + var bindings = ZeroTierSocketRuntimeBootstrapper.CreateUdpSocketBindings( + new ZeroTierMultipathOptions { Enabled = true, UdpSocketCount = 2 }, + enableIpv6: false, + getLocalBindAddresses: static () => Array.Empty()); + + Assert.All(bindings, binding => Assert.Null(binding.LocalAddress)); + } + + private static int GetAvailableUdpPort() + { + using var udp = new UdpClient(new IPEndPoint(IPAddress.Loopback, 0)); + return ((IPEndPoint)udp.Client.LocalEndPoint!).Port; + } } diff --git a/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs new file mode 100644 index 0000000..8b1c12a --- /dev/null +++ b/ZTSharp.Tests/ZeroTierStickyHintMaintenanceSelectorTests.cs @@ -0,0 +1,87 @@ +using System.Net; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierStickyHintMaintenanceSelectorTests +{ + [Fact] + public async Task SelectCandidates_PinnedRendezvous_RotatesAlternateSocketsAcrossCalls() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + var peerNodeId = new NodeId(0x2222222222); + var endpointManager = new ZeroTierDirectEndpointManager( + udp, + new IPEndPoint(IPAddress.Loopback, 9993), + peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => endpointManager); + var selector = new ZeroTierStickyHintMaintenanceSelector(planner); + var pinnedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var alternateEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var hinted = new[] { pinnedEndpoint, alternateEndpoint }; + + var first = selector.SelectCandidates( + peerNodeId, + hinted, + endpointBudget: 2, + restrictToPinnedRendezvous: true, + endpoint => endpoint.Equals(pinnedEndpoint), + endpoint => endpoint.Equals(alternateEndpoint), + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty(), + endpoint => endpoint.Equals(alternateEndpoint) ? new[] { 1 } : Array.Empty()); + var second = selector.SelectCandidates( + peerNodeId, + hinted, + endpointBudget: 2, + restrictToPinnedRendezvous: true, + endpoint => endpoint.Equals(pinnedEndpoint), + endpoint => endpoint.Equals(alternateEndpoint), + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty(), + endpoint => endpoint.Equals(alternateEndpoint) ? new[] { 1 } : Array.Empty()); + + Assert.Contains(first, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(pinnedEndpoint)); + Assert.Contains(second, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(pinnedEndpoint)); + Assert.Contains(first, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); + Assert.Contains(second, candidate => candidate.LocalSocketId == 1 && candidate.RemoteEndPoint.Equals(alternateEndpoint)); + } + + [Fact] + public async Task SelectCandidates_PinnedRendezvous_PrefersNonPublicAlternateHint() + { + await using var udp = new ScriptedUdpTransport( + new ZeroTierUdpLocalSocket(0, new IPEndPoint(IPAddress.Any, 10000)), + new ZeroTierUdpLocalSocket(1, new IPEndPoint(IPAddress.Any, 10001))); + var peerNodeId = new NodeId(0x2222222222); + var endpointManager = new ZeroTierDirectEndpointManager( + udp, + new IPEndPoint(IPAddress.Loopback, 9993), + peerNodeId); + var planner = new ZeroTierDirectHintPathPlanner( + udp, + _ => endpointManager); + var selector = new ZeroTierStickyHintMaintenanceSelector(planner); + var pinnedEndpoint = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 51623); + var publicAlternate = new IPEndPoint(IPAddress.Parse("176.66.90.119"), 19665); + var nonPublicAlternate = new IPEndPoint(IPAddress.Parse("100.85.196.109"), 9993); + var hinted = new[] { pinnedEndpoint, publicAlternate, nonPublicAlternate }; + + var selected = selector.SelectCandidates( + peerNodeId, + hinted, + endpointBudget: 2, + restrictToPinnedRendezvous: true, + endpoint => endpoint.Equals(pinnedEndpoint), + endpoint => endpoint.Equals(publicAlternate) || endpoint.Equals(nonPublicAlternate), + endpoint => endpoint.Equals(pinnedEndpoint) ? new[] { 1 } : Array.Empty(), + endpoint => endpoint.Equals(publicAlternate) || endpoint.Equals(nonPublicAlternate) ? new[] { 1 } : Array.Empty()); + + Assert.Contains(selected, candidate => candidate.RemoteEndPoint.Equals(pinnedEndpoint)); + Assert.Contains(selected, candidate => candidate.RemoteEndPoint.Equals(nonPublicAlternate)); + Assert.DoesNotContain(selected, candidate => candidate.RemoteEndPoint.Equals(publicAlternate)); + } +} diff --git a/ZTSharp.Tests/ZeroTierUdpLocalBindAddressSourceTests.cs b/ZTSharp.Tests/ZeroTierUdpLocalBindAddressSourceTests.cs new file mode 100644 index 0000000..5b9f9c3 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierUdpLocalBindAddressSourceTests.cs @@ -0,0 +1,58 @@ +using System.Net; +using System.Net.NetworkInformation; +using ZTSharp.ZeroTier.Internal; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierUdpLocalBindAddressSourceTests +{ + [Fact] + public void SelectAddresses_RequiresDefaultGateway_AndExcludesOverlayAdapters() + { + var addresses = ZeroTierUdpLocalBindAddressSource.SelectAddresses( + [ + new( + Name: "WLAN", + Description: "Wi-Fi", + NetworkInterfaceType.Wireless80211, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("10.0.0.112")], + GatewayAddresses: [IPAddress.Parse("10.0.0.1")]), + new( + Name: "vEthernet (Default Switch)", + Description: "Hyper-V Virtual Ethernet Adapter", + NetworkInterfaceType.Ethernet, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("172.25.128.1")], + GatewayAddresses: Array.Empty()), + new( + Name: "Tailscale", + Description: "Tailscale Tunnel", + NetworkInterfaceType.Tunnel, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("100.74.185.14")], + GatewayAddresses: [IPAddress.Parse("100.100.100.100")]) + ], + enableIpv6: false); + + Assert.Equal([IPAddress.Parse("10.0.0.112")], addresses); + } + + [Fact] + public void SelectAddresses_Excludes_LinkLocal_Addresses() + { + var addresses = ZeroTierUdpLocalBindAddressSource.SelectAddresses( + [ + new( + Name: "Ethernet", + Description: "Realtek", + NetworkInterfaceType.Ethernet, + OperationalStatus.Up, + UnicastAddresses: [IPAddress.Parse("169.254.1.20"), IPAddress.Parse("fe80::1")], + GatewayAddresses: [IPAddress.Parse("10.0.0.1")]) + ], + enableIpv6: true); + + Assert.Empty(addresses); + } +} diff --git a/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs b/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs index 8a71692..20e124e 100644 --- a/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierUdpMultiTransportTests.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Reflection; using ZTSharp.ZeroTier.Transport; namespace ZTSharp.Tests; @@ -50,5 +51,110 @@ public async Task SendAsync_UsesSelectedSocket_AsUdpSourceEndpoint() var datagram2 = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); Assert.Equal(socket2.LocalEndpoint.Port, datagram2.RemoteEndPoint.Port); } + + [Fact] + public async Task SendAsync_WithoutSocketId_FansOutAcrossAllSockets() + { + await using var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + await using var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + await using var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + await using var receiver = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + var receiverEndpoint = TestUdpEndpoints.ToLoopback(receiver.LocalEndpoint); + + await multi.SendAsync(receiverEndpoint, new byte[] { 0x33 }); + + var seenPorts = new HashSet(); + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(2); + while (seenPorts.Count < 2 && DateTimeOffset.UtcNow < deadline) + { + var datagram = await receiver.ReceiveAsync(TimeSpan.FromSeconds(2)); + seenPorts.Add(datagram.RemoteEndPoint.Port); + } + + Assert.Contains(socket1.LocalEndpoint.Port, seenPorts); + Assert.Contains(socket2.LocalEndpoint.Port, seenPorts); + } + + [Fact] + public async Task DisposeAsync_DisposesUnderlyingSockets_AndMarksTransportDisposed() + { + var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + try + { + await multi.DisposeAsync(); + + Assert.Throws(() => _ = multi.LocalSockets); + await Assert.ThrowsAsync(() => + multi.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x00 })); + + await Assert.ThrowsAsync(() => + socket1.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x01 })); + await Assert.ThrowsAsync(() => + socket2.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x02 })); + } + finally + { + await multi.DisposeAsync(); + } + } + + [Fact] + public async Task DisposeAsync_RemainsBestEffort_WhenForwarderFaults() + { + var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + try + { + var forwardersField = typeof(ZeroTierUdpMultiTransport).GetField("_forwarders", BindingFlags.NonPublic | BindingFlags.Instance); + Assert.NotNull(forwardersField); + + var forwarders = (Task[]?)forwardersField!.GetValue(multi); + Assert.NotNull(forwarders); + Assert.NotEmpty(forwarders!); + + forwarders![0] = Task.WhenAll(forwarders[0], Task.FromException(new InvalidOperationException("boom"))); + + await multi.DisposeAsync(); + + await Assert.ThrowsAsync(() => + socket1.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x01 })); + await Assert.ThrowsAsync(() => + socket2.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x02 })); + } + finally + { + await multi.DisposeAsync(); + } + } + + [Fact] + public async Task DisposeAsync_CanBeCalledConcurrently_WithoutThrowing() + { + var socket1 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 1); + var socket2 = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false, localSocketId: 2); + var multi = new ZeroTierUdpMultiTransport(new[] { socket1, socket2 }); + + try + { + var tasks = Enumerable.Range(0, 20) + .Select(_ => multi.DisposeAsync().AsTask()) + .ToArray(); + + await Task.WhenAll(tasks); + + await Assert.ThrowsAsync(() => + socket1.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x01 })); + } + finally + { + await multi.DisposeAsync(); + } + } } diff --git a/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs b/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs new file mode 100644 index 0000000..866ccc0 --- /dev/null +++ b/ZTSharp.Tests/ZeroTierUdpSocketReceiveTests.cs @@ -0,0 +1,69 @@ +using System.Net; +using ZTSharp.ZeroTier; +using ZTSharp.ZeroTier.Internal; +using ZTSharp.ZeroTier.Net; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.Tests; + +public sealed class ZeroTierUdpSocketReceiveTests +{ + [Fact] + public async Task ReceiveFromAsync_WildcardSemantics_AllowAnyLocalManagedIp() + { + var localIpA = IPAddress.Parse("10.0.0.2"); + var localIpB = IPAddress.Parse("10.0.0.3"); + await using var runtime = CreateRuntime(localIpA, localIpB); + var ip = GetIpHandler(runtime); + + const ushort localPort = 12010; + await using var socket = new ZeroTierUdpSocket(runtime, IPAddress.Any, localPort); + + var buffer = new byte[32]; + var receiveTask = socket.ReceiveFromAsync(buffer, TimeSpan.FromSeconds(1)).AsTask(); + + var remoteIp = IPAddress.Parse("10.0.0.1"); + var payload = new byte[] { 1, 2, 3, 4 }; + var udp = UdpCodec.Encode(remoteIp, localIpB, sourcePort: 1111, destinationPort: localPort, payload); + var ipv4 = Ipv4Codec.Encode(remoteIp, localIpB, protocol: UdpCodec.ProtocolNumber, payload: udp, identification: 1); + + await ip.HandleIpv4PacketAsync(peerNodeId: new NodeId(0x3333333333), ipv4Packet: ipv4, cancellationToken: CancellationToken.None); + + var received = await receiveTask; + Assert.Equal(payload.Length, received.ReceivedBytes); + Assert.Equal(new IPEndPoint(remoteIp, 1111), received.RemoteEndPoint); + Assert.Equal(payload, buffer.AsSpan(0, payload.Length).ToArray()); + } + + private static ZeroTierDataplaneIpHandler GetIpHandler(ZeroTierDataplaneRuntime runtime) + { + var peerPackets = GetPrivateField(runtime, "_peerPackets"); + return GetPrivateField(peerPackets, "_ip"); + } + + private static T GetPrivateField(object instance, string fieldName) + { + var field = instance.GetType().GetField(fieldName, System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + Assert.NotNull(field); + var value = field!.GetValue(instance); + Assert.NotNull(value); + return (T)value!; + } + + [global::System.Diagnostics.CodeAnalysis.SuppressMessage( + "Reliability", + "CA2000:Dispose objects before losing scope", + Justification = "UDP transport ownership transfers to ZeroTierDataplaneRuntime, which is disposed by the caller.")] + private static ZeroTierDataplaneRuntime CreateRuntime(IPAddress localManagedIpV4A, IPAddress localManagedIpV4B) + => new( + udp: new ZeroTierUdpTransport(localPort: 0, enableIpv6: false), + rootNodeId: new NodeId(0x1111111111), + rootEndpoint: new IPEndPoint(IPAddress.Loopback, 9999), + rootKey: new byte[48], + rootProtocolVersion: 12, + localIdentity: ZeroTierTestIdentities.CreateFastIdentity(0x2222222222), + networkId: 1, + localManagedIpsV4: new[] { localManagedIpV4A, localManagedIpV4B }, + localManagedIpsV6: Array.Empty(), + inlineCom: new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); +} diff --git a/ZTSharp.Tests/ZeroTierUdpTransportTests.cs b/ZTSharp.Tests/ZeroTierUdpTransportTests.cs index 3795f18..8d637b9 100644 --- a/ZTSharp.Tests/ZeroTierUdpTransportTests.cs +++ b/ZTSharp.Tests/ZeroTierUdpTransportTests.cs @@ -28,4 +28,19 @@ public async Task CanSendAndReceiveLoopbackDatagrams() var receivedPong = await a.ReceiveAsync(cts.Token); Assert.True(receivedPong.Payload.AsSpan().SequenceEqual(pong)); } + + [Fact] + public async Task DisposeAsync_CanBeCalledConcurrently_WithoutThrowing() + { + var transport = new ZeroTierUdpTransport(localPort: 0, enableIpv6: false); + + var tasks = Enumerable.Range(0, 20) + .Select(_ => transport.DisposeAsync().AsTask()) + .ToArray(); + + await Task.WhenAll(tasks); + + await Assert.ThrowsAsync(() => + transport.SendAsync(new IPEndPoint(IPAddress.Loopback, 1), new byte[] { 0x00 })); + } } diff --git a/ZTSharp.Tests/ZtTcpListenerTests.cs b/ZTSharp.Tests/ZtTcpListenerTests.cs index 223d9d6..f7bfba1 100644 --- a/ZTSharp.Tests/ZtTcpListenerTests.cs +++ b/ZTSharp.Tests/ZtTcpListenerTests.cs @@ -6,6 +6,14 @@ namespace ZTSharp.Tests; public sealed class ZtTcpListenerTests { + [Fact] + public async Task DisposeAsync_IsIdempotent() + { + var listener = new ZtTcpListener(IPAddress.Loopback, 0); + await listener.DisposeAsync(); + await listener.DisposeAsync(); + } + [Fact] public async Task TcpListener_EchoesPayloadOffline() { diff --git a/ZTSharp/EventLoop.cs b/ZTSharp/EventLoop.cs index c33abfa..6a50c13 100644 --- a/ZTSharp/EventLoop.cs +++ b/ZTSharp/EventLoop.cs @@ -287,9 +287,10 @@ public void Dispose() private void ThrowIfDisposed() { - if (_fault is not null) + var fault = Volatile.Read(ref _fault); + if (fault is not null) { - throw new InvalidOperationException("Event loop is faulted.", _fault); + throw new InvalidOperationException("Event loop is faulted.", fault); } ObjectDisposedException.ThrowIf(_disposed, this); diff --git a/ZTSharp/FileStateStore.cs b/ZTSharp/FileStateStore.cs index ef0967c..1a30938 100644 --- a/ZTSharp/FileStateStore.cs +++ b/ZTSharp/FileStateStore.cs @@ -6,6 +6,8 @@ namespace ZTSharp; /// public sealed class FileStateStore : IStateStore { + private const long MaxReadBytes = 64L * 1024 * 1024; + private readonly string _rootPath; private readonly StringComparison _pathComparison; private readonly string _rootPathPrefix; @@ -13,17 +15,18 @@ public sealed class FileStateStore : IStateStore public FileStateStore(string rootPath) { - ArgumentException.ThrowIfNullOrWhiteSpace(rootPath); + if (string.IsNullOrEmpty(rootPath)) + { + throw new ArgumentException("Root path must not be null or empty.", nameof(rootPath)); + } + _rootPath = Path.GetFullPath(rootPath); _rootPathTrimmed = Path.TrimEndingDirectorySeparator(_rootPath); _rootPathPrefix = _rootPathTrimmed + Path.DirectorySeparatorChar; _pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + ThrowIfRootPathIsReparsePoint(); Directory.CreateDirectory(_rootPath); - - if (IsReparsePoint(_rootPathTrimmed)) - { - throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); - } + ThrowIfRootPathIsReparsePoint(); } public Task ExistsAsync(string key, CancellationToken cancellationToken = default) @@ -70,12 +73,14 @@ public async Task WriteAsync(string key, ReadOnlyMemory value, Cancellatio } var path = GetPhysicalPathForNormalizedKey(normalized, key); - await Internal.AtomicFile.WriteAllBytesAsync(path, value, cancellationToken).ConfigureAwait(false); - + EnsureParentDirectoryExistsNoReparse(path); if (string.Equals(normalized, Internal.NodeStoreKeys.IdentitySecretKey, StringComparison.OrdinalIgnoreCase)) { - Internal.SecretFilePermissions.TryHardenSecretFile(path); + await Internal.AtomicFile.WriteSecretBytesAsync(path, value, cancellationToken).ConfigureAwait(false); + return; } + + await Internal.AtomicFile.WriteAllBytesAsync(path, value, cancellationToken).ConfigureAwait(false); } public Task DeleteAsync(string key, CancellationToken cancellationToken = default) @@ -127,8 +132,8 @@ public Task> ListAsync(string prefix = "", CancellationTok if (virtualPrefix.Length != 0 && StateStorePlanetAliases.TryGetCanonicalAliasKey(virtualPrefix, out var canonicalAliasPrefix)) { - var planetPath = Path.Combine(_rootPath, StateStorePlanetAliases.PlanetKey); - var rootsPath = Path.Combine(_rootPath, StateStorePlanetAliases.RootsKey); + var planetPath = GetPhysicalPathForNormalizedKey(StateStorePlanetAliases.PlanetKey, StateStorePlanetAliases.PlanetKey); + var rootsPath = GetPhysicalPathForNormalizedKey(StateStorePlanetAliases.RootsKey, StateStorePlanetAliases.RootsKey); if (File.Exists(planetPath) || File.Exists(rootsPath)) { return Task.FromResult>([canonicalAliasPrefix]); @@ -248,8 +253,41 @@ private static async Task ReadAllBytesWithSharingAsync(string path, Canc bufferSize: 16 * 1024, options: FileOptions.Asynchronous | FileOptions.SequentialScan); - using var memory = stream.Length <= int.MaxValue ? new MemoryStream((int)stream.Length) : new MemoryStream(); - await stream.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); + if (stream.Length > MaxReadBytes) + { + throw new IOException($"State file exceeds maximum supported size of {MaxReadBytes} bytes."); + } + + var initialCapacity = stream.Length <= int.MaxValue + ? (int)Math.Min(stream.Length, MaxReadBytes) + : 0; + + using var memory = initialCapacity > 0 ? new MemoryStream(initialCapacity) : new MemoryStream(); + var buffer = new byte[16 * 1024]; + long totalRead = 0; + while (true) + { + var read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (read <= 0) + { + break; + } + + var remaining = MaxReadBytes - totalRead; + if (read > remaining) + { + if (remaining > 0) + { + await memory.WriteAsync(buffer.AsMemory(0, (int)remaining), cancellationToken).ConfigureAwait(false); + } + + throw new IOException($"State file exceeds maximum supported size of {MaxReadBytes} bytes."); + } + + await memory.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + totalRead += read; + } + return memory.ToArray(); } @@ -297,6 +335,8 @@ private bool IsUnderRoot(string fullPath) private void ThrowIfPathTraversesReparsePoint(string fullPath) { + ThrowIfRootPathIsReparsePoint(); + if (string.Equals(fullPath, _rootPathTrimmed, _pathComparison)) { return; @@ -318,11 +358,81 @@ private void ThrowIfPathTraversesReparsePoint(string fullPath) for (var i = 0; i < parts.Length; i++) { current = Path.Combine(current, parts[i]); - if (!File.Exists(current) && !Directory.Exists(current)) + if (!TryGetAttributes(current, out var attributes)) { return; } + if ((attributes & FileAttributes.ReparsePoint) != 0) + { + throw new InvalidOperationException("State path traversal via symlink/junction/reparse point is not allowed."); + } + } + } + + private void ThrowIfRootPathIsReparsePoint() + { + if (IsReparsePoint(_rootPathTrimmed)) + { + throw new InvalidOperationException("State root path must not be a symlink/junction/reparse point."); + } + + var current = Path.GetDirectoryName(_rootPathTrimmed); + while (!string.IsNullOrEmpty(current)) + { + if (IsReparsePoint(current)) + { + throw new InvalidOperationException("State root path must not be under a symlink/junction/reparse point."); + } + + var parent = Path.GetDirectoryName(current); + if (parent is null || string.Equals(parent, current, _pathComparison)) + { + break; + } + + current = parent; + } + } + + private void EnsureParentDirectoryExistsNoReparse(string fullPath) + { + ThrowIfRootPathIsReparsePoint(); + + var directory = Path.GetDirectoryName(fullPath); + if (string.IsNullOrWhiteSpace(directory) || string.Equals(directory, _rootPathTrimmed, _pathComparison)) + { + return; + } + + directory = Path.GetFullPath(directory); + if (!IsUnderRoot(directory)) + { + throw new ArgumentException("Path is not under state root.", nameof(fullPath)); + } + + var relative = Path.GetRelativePath(_rootPathTrimmed, directory); + if (relative == "." || relative.Length == 0) + { + return; + } + + var current = _rootPathTrimmed; + var parts = relative.Split(new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < parts.Length; i++) + { + current = Path.Combine(current, parts[i]); + if (Directory.Exists(current)) + { + if (IsReparsePoint(current)) + { + throw new InvalidOperationException("State path traversal via symlink/junction/reparse point is not allowed."); + } + + continue; + } + + Directory.CreateDirectory(current); if (IsReparsePoint(current)) { throw new InvalidOperationException("State path traversal via symlink/junction/reparse point is not allowed."); @@ -346,4 +456,22 @@ private static bool IsReparsePoint(string path) } } + private static bool TryGetAttributes(string path, out FileAttributes attributes) + { + try + { + attributes = File.GetAttributes(path); + return true; + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + + attributes = default; + return false; + } + } diff --git a/ZTSharp/Http/OverlayHttpMessageHandler.cs b/ZTSharp/Http/OverlayHttpMessageHandler.cs index 8338939..1f6ca83 100644 --- a/ZTSharp/Http/OverlayHttpMessageHandler.cs +++ b/ZTSharp/Http/OverlayHttpMessageHandler.cs @@ -36,6 +36,11 @@ public OverlayHttpMessageHandler(Node node, ulong networkId, OverlayHttpMessageH "LocalPortEnd must be in the range 1..65535 and greater than or equal to LocalPortStart."); } + if (_options.LocalPortAllocationTimeout != Timeout.InfiniteTimeSpan && _options.LocalPortAllocationTimeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(options), "LocalPortAllocationTimeout must be positive or Timeout.InfiniteTimeSpan."); + } + var sockets = new SocketsHttpHandler { UseProxy = false @@ -68,14 +73,43 @@ private async ValueTask ConnectOverlayAsync(SocketsHttpConnectionContext } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - await client.DisposeAsync().ConfigureAwait(false); - ReleaseLocalPort(localPort); + try + { + await client.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose is best-effort; preserve cancellation semantics and always release the reserved port. + catch +#pragma warning restore CA1031 + { + } + finally + { + ReleaseLocalPort(localPort); + } + throw; } - catch (Exception ex) when (ex is not HttpRequestException) + catch (Exception ex) { - await client.DisposeAsync().ConfigureAwait(false); - ReleaseLocalPort(localPort); + try + { + await client.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose is best-effort; preserve original exception and always release the reserved port. + catch +#pragma warning restore CA1031 + { + } + finally + { + ReleaseLocalPort(localPort); + } + + if (ex is HttpRequestException) + { + throw; + } + throw new HttpRequestException( $"Overlay connect to '{host}:{endpoint.Port}' failed.", ex); @@ -87,12 +121,23 @@ private async Task AllocateReservedLocalPortAsync(CancellationToken cancell var start = _options.LocalPortStart; var end = _options.LocalPortEnd; var range = end - start + 1; + var allocationTimeout = _options.LocalPortAllocationTimeout; + var timeoutMs = allocationTimeout == Timeout.InfiniteTimeSpan + ? long.MaxValue + : (long)Math.Min(long.MaxValue, Math.Ceiling(allocationTimeout.TotalMilliseconds)); + var startedMs = Environment.TickCount64; var backoffMs = 1; while (true) { cancellationToken.ThrowIfCancellationRequested(); + if (timeoutMs != long.MaxValue && unchecked(Environment.TickCount64 - startedMs) > timeoutMs) + { + throw new HttpRequestException( + $"Overlay TCP local port allocation exhausted (range {start}..{end}) after {allocationTimeout}."); + } + for (var i = 0; i < range; i++) { var candidate = AllocateLocalPort(); @@ -102,6 +147,18 @@ private async Task AllocateReservedLocalPortAsync(CancellationToken cancell } } + if (timeoutMs != long.MaxValue) + { + var elapsed = unchecked(Environment.TickCount64 - startedMs); + var remaining = timeoutMs - elapsed; + if (remaining <= 0) + { + continue; + } + + backoffMs = (int)Math.Min(backoffMs, remaining); + } + await Task.Delay(backoffMs, cancellationToken).ConfigureAwait(false); backoffMs = Math.Min(backoffMs * 2, 50); } diff --git a/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs b/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs index 17d0da4..52b833f 100644 --- a/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs +++ b/ZTSharp/Http/OverlayHttpMessageHandlerOptions.cs @@ -18,5 +18,11 @@ public sealed class OverlayHttpMessageHandlerOptions public int LocalPortStart { get; init; } = 49152; public int LocalPortEnd { get; init; } = 65535; + + /// + /// Maximum time to wait for a reserved local overlay TCP port within ... + /// Use to wait indefinitely (not recommended unless requests are always cancellable). + /// + public TimeSpan LocalPortAllocationTimeout { get; init; } = TimeSpan.FromSeconds(30); } diff --git a/ZTSharp/Internal/ActiveTaskSet.cs b/ZTSharp/Internal/ActiveTaskSet.cs index 21aa3e9..0150ab4 100644 --- a/ZTSharp/Internal/ActiveTaskSet.cs +++ b/ZTSharp/Internal/ActiveTaskSet.cs @@ -30,10 +30,10 @@ public async Task WaitAsync(CancellationToken cancellationToken = default) return; } - var snapshot = new List(_tasks.Count); - foreach (var task in _tasks.Values) + var snapshot = new List>(_tasks.Count); + foreach (var pair in _tasks) { - snapshot.Add(task); + snapshot.Add(pair); } if (snapshot.Count == 0) @@ -44,12 +44,27 @@ public async Task WaitAsync(CancellationToken cancellationToken = default) try { - await Task.WhenAll(snapshot).WaitAsync(cancellationToken).ConfigureAwait(false); + var tasks = new Task[snapshot.Count]; + for (var i = 0; i < snapshot.Count; i++) + { + tasks[i] = snapshot[i].Value; + } + + await Task.WhenAll(tasks).WaitAsync(cancellationToken).ConfigureAwait(false); } #pragma warning disable CA1031 // ActiveTaskSet is primarily used for shutdown coordination; faults are observed but not surfaced here. catch #pragma warning restore CA1031 { + foreach (var pair in snapshot) + { + if (pair.Value.IsCompleted) + { + _tasks.TryRemove(pair.Key, out _); + } + } + + await Task.Yield(); } } } diff --git a/ZTSharp/Internal/AtomicFile.cs b/ZTSharp/Internal/AtomicFile.cs index f1f9feb..90083f7 100644 --- a/ZTSharp/Internal/AtomicFile.cs +++ b/ZTSharp/Internal/AtomicFile.cs @@ -2,6 +2,8 @@ namespace ZTSharp.Internal; internal static class AtomicFile { + private const UnixFileMode SecretFileMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + public static void WriteAllBytes(string path, ReadOnlySpan bytes) { ArgumentException.ThrowIfNullOrWhiteSpace(path); @@ -83,6 +85,17 @@ public static void WriteAllBytes(string path, ReadOnlySpan bytes) } } + public static void WriteSecretBytes(string path, ReadOnlySpan bytes) + { + if (OperatingSystem.IsWindows()) + { + WriteAllBytes(path, bytes); + return; + } + + WriteAllBytesCore(path, bytes, SecretFileMode); + } + public static async Task WriteAllBytesAsync( string path, ReadOnlyMemory bytes, @@ -169,4 +182,258 @@ public static async Task WriteAllBytesAsync( } } } + + public static Task WriteSecretBytesAsync( + string path, + ReadOnlyMemory bytes, + CancellationToken cancellationToken = default) + { + if (OperatingSystem.IsWindows()) + { + return WriteAllBytesAsync(path, bytes, cancellationToken); + } + + return WriteAllBytesCoreAsync(path, bytes, SecretFileMode, cancellationToken); + } + + private static void WriteAllBytesCore(string path, ReadOnlySpan bytes, UnixFileMode unixCreateMode) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(directory)) + { + Directory.CreateDirectory(directory); + } + + var tmpPath = $"{path}.tmp.{Guid.NewGuid():N}"; + try + { + using (var stream = CreateSecretWriteStream(tmpPath, unixCreateMode, asynchronous: false)) + { + stream.Write(bytes); + stream.Flush(flushToDisk: true); + } + + const int maxAttempts = 10; + Exception? lastException = null; + var moved = false; + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + try + { + File.Move(tmpPath, path, overwrite: true); + moved = true; + break; + } + catch (IOException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + Thread.Sleep(TimeSpan.FromMilliseconds(10 * (attempt + 1))); + } + catch (UnauthorizedAccessException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + Thread.Sleep(TimeSpan.FromMilliseconds(10 * (attempt + 1))); + } + catch (IOException ex) + { + lastException = ex; + break; + } + catch (UnauthorizedAccessException ex) + { + lastException = ex; + break; + } + } + + if (!moved) + { + throw new IOException($"Atomic replace failed after {maxAttempts} attempts: '{path}'.", lastException); + } + } + finally + { + try + { + if (File.Exists(tmpPath)) + { + File.Delete(tmpPath); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + } + + private static async Task WriteAllBytesCoreAsync( + string path, + ReadOnlyMemory bytes, + UnixFileMode unixCreateMode, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(directory)) + { + Directory.CreateDirectory(directory); + } + + var tmpPath = $"{path}.tmp.{Guid.NewGuid():N}"; + try + { + using (var stream = CreateSecretWriteStream(tmpPath, unixCreateMode, asynchronous: true)) + { + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); +#pragma warning disable CA1849 + stream.Flush(flushToDisk: true); +#pragma warning restore CA1849 + } + + const int maxAttempts = 10; + Exception? lastException = null; + var moved = false; + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + try + { + File.Move(tmpPath, path, overwrite: true); + moved = true; + break; + } + catch (IOException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + await Task.Delay(TimeSpan.FromMilliseconds(10 * (attempt + 1)), cancellationToken).ConfigureAwait(false); + } + catch (UnauthorizedAccessException ex) when (attempt < maxAttempts - 1) + { + lastException = ex; + await Task.Delay(TimeSpan.FromMilliseconds(10 * (attempt + 1)), cancellationToken).ConfigureAwait(false); + } + catch (IOException ex) + { + lastException = ex; + break; + } + catch (UnauthorizedAccessException ex) + { + lastException = ex; + break; + } + } + + if (!moved) + { + throw new IOException($"Atomic replace failed after {maxAttempts} attempts: '{path}'.", lastException); + } + } + finally + { + try + { + if (File.Exists(tmpPath)) + { + File.Delete(tmpPath); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + } + + private static FileStream CreateSecretWriteStream(string path, UnixFileMode unixCreateMode, bool asynchronous) + { + if (!OperatingSystem.IsWindows()) + { + try + { + var options = new FileStreamOptions + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = 16 * 1024, + Options = asynchronous ? FileOptions.Asynchronous : FileOptions.None, + UnixCreateMode = unixCreateMode + }; + + return new FileStream(path, options); + } + catch (PlatformNotSupportedException) + { + } + catch (NotSupportedException) + { + } + } + + var fallback = new FileStream( + path, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 16 * 1024, + options: asynchronous ? FileOptions.Asynchronous : FileOptions.None); + + try + { + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(path, unixCreateMode); + } + } + catch (PlatformNotSupportedException) + { + fallback.Dispose(); + TryDeleteFallback(path); + throw new IOException("Failed to set secret file permissions."); + } + catch (NotSupportedException) + { + fallback.Dispose(); + TryDeleteFallback(path); + throw new IOException("Failed to set secret file permissions."); + } + catch (IOException ex) + { + fallback.Dispose(); + throw new IOException("Failed to set secret file permissions.", ex); + } + catch (UnauthorizedAccessException ex) + { + fallback.Dispose(); + throw new IOException("Failed to set secret file permissions.", ex); + } + + return fallback; + } + + private static void TryDeleteFallback(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } } diff --git a/ZTSharp/Internal/BoundedFileIO.cs b/ZTSharp/Internal/BoundedFileIO.cs index 8a16112..47a8b0a 100644 --- a/ZTSharp/Internal/BoundedFileIO.cs +++ b/ZTSharp/Internal/BoundedFileIO.cs @@ -24,13 +24,22 @@ public static bool TryReadAllBytes(string path, int maxBytes, out byte[] bytes) bufferSize: 16 * 1024, options: FileOptions.SequentialScan); - if (stream.Length <= 0 || stream.Length > maxBytes || stream.Length > int.MaxValue) + long length; + try + { + length = stream.Length; + } + catch (NotSupportedException) { return false; } - var length = (int)stream.Length; - var buffer = new byte[length]; + if (length <= 0 || length > maxBytes || length > int.MaxValue) + { + return false; + } + + var buffer = new byte[(int)length]; var totalRead = 0; while (totalRead < buffer.Length) @@ -55,6 +64,10 @@ public static bool TryReadAllBytes(string path, int maxBytes, out byte[] bytes) { return false; } + catch (NotSupportedException) + { + return false; + } } public static bool TryReadAllText(string path, int maxBytes, Encoding encoding, out string text) @@ -67,7 +80,9 @@ public static bool TryReadAllText(string path, int maxBytes, Encoding encoding, return false; } - text = encoding.GetString(bytes); + using var stream = new MemoryStream(bytes, writable: false); + using var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true); + text = reader.ReadToEnd(); return true; } } diff --git a/ZTSharp/Internal/NodeCore.cs b/ZTSharp/Internal/NodeCore.cs index 9e47a0a..5f9207a 100644 --- a/ZTSharp/Internal/NodeCore.cs +++ b/ZTSharp/Internal/NodeCore.cs @@ -66,6 +66,8 @@ public NodeCore( public IPEndPoint? LocalTransportEndpoint => _transportService.GetLocalTransportEndpoint(); + internal INodeTransport TransportForTests => _transport; + public NodeId NodeId => _runtime.NodeId; public bool IsRunning => _runtime.State == NodeState.Running; diff --git a/ZTSharp/Internal/NodeLifecycleService.cs b/ZTSharp/Internal/NodeLifecycleService.cs index 10dbe27..8c0de5c 100644 --- a/ZTSharp/Internal/NodeLifecycleService.cs +++ b/ZTSharp/Internal/NodeLifecycleService.cs @@ -1,11 +1,16 @@ using Microsoft.Extensions.Logging; using ZTSharp.Transport; +using System.Diagnostics.CodeAnalysis; namespace ZTSharp.Internal; internal sealed class NodeLifecycleService : IAsyncDisposable { private readonly NodeRuntimeState _runtime; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be safe during concurrent lifecycle operations; disposing this lock can throw on in-flight Release calls.")] private readonly SemaphoreSlim _stateLock; private readonly CancellationTokenSource _nodeCts; private readonly INodeTransport _transport; @@ -17,6 +22,8 @@ internal sealed class NodeLifecycleService : IAsyncDisposable private readonly NodeTransportService _transportService; private readonly bool _ownsTransport; + private int _disposeState; + public NodeLifecycleService( NodeRuntimeState runtime, SemaphoreSlim stateLock, @@ -197,44 +204,129 @@ public async Task ExecuteWhileRunningAsync(Func public async ValueTask DisposeAsync() { - if (_runtime.Disposed) + if (Interlocked.Exchange(ref _disposeState, 1) != 0) { return; } - await _nodeCts.CancelAsync().ConfigureAwait(false); - using var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + try { - await StopAsync(shutdownCts.Token).ConfigureAwait(false); + // Best-effort: block new operations promptly, but don't wedge forever if some operation is stuck. + await _stateLock.WaitAsync(shutdownCts.Token).ConfigureAwait(false); + try + { + if (_runtime.Disposed) + { + return; + } + + _runtime.Disposed = true; + } + finally + { + _stateLock.Release(); + } } catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) { + // Worst case: a concurrent lifecycle operation is wedged under the state lock. Mark the node as disposed + // anyway so future operations fail fast. + _runtime.Disposed = true; } try { - await _networkService.LeaveAllNetworksAsync(shutdownCts.Token).ConfigureAwait(false); + await _nodeCts.CancelAsync().ConfigureAwait(false); } - catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) + catch (ObjectDisposedException) { } - _runtime.Disposed = true; + try + { + await _stateLock.WaitAsync(shutdownCts.Token).ConfigureAwait(false); + try + { + if (_runtime.State is not NodeState.Stopped and not NodeState.Faulted) + { + _runtime.State = NodeState.Stopping; + _events.Publish(EventCode.NodeStopping, DateTimeOffset.UtcNow); - if (_ownsTransport && _transport is IAsyncDisposable asyncTransport) + await _networkService.UnregisterAllNetworksAsync(shutdownCts.Token).ConfigureAwait(false); + await _transport.FlushAsync(shutdownCts.Token).ConfigureAwait(false); + await _store.FlushAsync(shutdownCts.Token).ConfigureAwait(false); + _runtime.State = NodeState.Stopped; + _events.Publish(EventCode.NodeStopped, DateTimeOffset.UtcNow); + } + } + finally + { + _stateLock.Release(); + } + } + catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) + { + } + catch (ObjectDisposedException) + { + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { + _runtime.State = NodeState.Faulted; +#pragma warning disable CA1848 + _logger.LogError(ex, "Failed to stop node during disposal"); +#pragma warning restore CA1848 + _events.Publish(EventCode.NodeFaulted, DateTimeOffset.UtcNow, message: ex.Message, error: ex); + } + + try + { + await _networkService.LeaveAllNetworksAsync(shutdownCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) + { + } + catch (ObjectDisposedException) { - await asyncTransport.DisposeAsync().ConfigureAwait(false); } - _stateLock.Dispose(); - _events.Complete(); - _nodeCts.Dispose(); + try + { + if (_ownsTransport && _transport is IAsyncDisposable asyncTransport) + { + using var transportDisposeCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + try + { + await asyncTransport + .DisposeAsync() + .AsTask() + .WaitAsync(transportDisposeCts.Token) + .ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested) + { + } + catch (OperationCanceledException) when (transportDisposeCts.IsCancellationRequested) + { + } + } + } + finally + { + _events.Complete(); + _nodeCts.Dispose(); + } } private void EnsureNotDisposed() { - ObjectDisposedException.ThrowIf(_runtime.Disposed, nameof(Node)); + ObjectDisposedException.ThrowIf(_runtime.Disposed || Volatile.Read(ref _disposeState) != 0, nameof(Node)); } } diff --git a/ZTSharp/Internal/NodeNetworkService.cs b/ZTSharp/Internal/NodeNetworkService.cs index 2f06ca1..7e8b870 100644 --- a/ZTSharp/Internal/NodeNetworkService.cs +++ b/ZTSharp/Internal/NodeNetworkService.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Net; using System.Text.Json; @@ -7,6 +8,10 @@ namespace ZTSharp.Internal; +[SuppressMessage( + "Design", + "CA1001:Types that own disposable fields should be disposable", + Justification = "This service is owned by NodeCore/NodeLifecycleService; the async gate is intentionally not disposed to keep shutdown idempotent and avoid races with in-flight leave operations.")] internal sealed class NodeNetworkService { private readonly IStateStore _store; @@ -16,6 +21,7 @@ internal sealed class NodeNetworkService private readonly ConcurrentDictionary _joinedNetworks = new(); private readonly ConcurrentDictionary _networkRegistrations = new(); + private readonly SemaphoreSlim _leaveGate = new(1, 1); private readonly NetworkIdReadOnlyCollection _joinedNetworkIds; public NodeNetworkService(IStateStore store, INodeTransport transport, NodeEventStream events, NodePeerService peerService) @@ -40,6 +46,12 @@ public async Task JoinNetworkAsync( Func, CancellationToken, Task> onFrameReceived, CancellationToken cancellationToken) { + // Joining a network is idempotent: avoid leaking registrations/subscribers and keep the transport state consistent. + if (_networkRegistrations.ContainsKey(networkId)) + { + return; + } + _events.Publish(EventCode.NetworkJoinRequested, DateTimeOffset.UtcNow, networkId); Guid registration = default; @@ -59,9 +71,22 @@ public async Task JoinNetworkAsync( JsonContext.Default.NetworkState); await _store.WriteAsync(key, payload, cancellationToken).ConfigureAwait(false); + try + { + await _peerService.RecoverPeersAsync(networkId, _transport, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } +#pragma warning disable CA1031 // Peer recovery is best-effort during join. + catch +#pragma warning restore CA1031 + { + } + _joinedNetworks[networkId] = new NetworkInfo(networkId, now); _networkRegistrations[networkId] = registration; - await _peerService.RecoverPeersAsync(networkId, _transport, cancellationToken).ConfigureAwait(false); _events.Publish(EventCode.NetworkJoined, DateTimeOffset.UtcNow, networkId); } @@ -71,7 +96,16 @@ public async Task JoinNetworkAsync( { try { - await _transport.LeaveNetworkAsync(networkId, registration, CancellationToken.None).ConfigureAwait(false); + await _leaveGate.WaitAsync(CancellationToken.None).ConfigureAwait(false); + try + { + await _transport.LeaveNetworkAsync(networkId, registration, CancellationToken.None).ConfigureAwait(false); + _networkRegistrations.TryRemove(new KeyValuePair(networkId, registration)); + } + finally + { + _leaveGate.Release(); + } } catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException) { @@ -85,17 +119,25 @@ public async Task JoinNetworkAsync( public async Task LeaveNetworkAsync(ulong networkId, CancellationToken cancellationToken) { var key = BuildNetworkFileKey(networkId); - if (_networkRegistrations.TryGetValue(networkId, out var registration)) + await _leaveGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - await _transport.LeaveNetworkAsync(networkId, registration, cancellationToken).ConfigureAwait(false); - _networkRegistrations.TryRemove(networkId, out _); - } + if (_networkRegistrations.TryGetValue(networkId, out var registration)) + { + await _transport.LeaveNetworkAsync(networkId, registration, cancellationToken).ConfigureAwait(false); + _networkRegistrations.TryRemove(networkId, out _); + } - var removed = _joinedNetworks.TryRemove(networkId, out _); - if (removed) + var removed = _joinedNetworks.TryRemove(networkId, out _); + if (removed) + { + await _store.DeleteAsync(key, cancellationToken).ConfigureAwait(false); + _events.Publish(EventCode.NetworkLeft, DateTimeOffset.UtcNow, networkId); + } + } + finally { - await _store.DeleteAsync(key, cancellationToken).ConfigureAwait(false); - _events.Publish(EventCode.NetworkLeft, DateTimeOffset.UtcNow, networkId); + _leaveGate.Release(); } } @@ -192,14 +234,32 @@ public async Task RecoverNetworksAsync( foreach (var network in _joinedNetworks.Keys) { + if (_networkRegistrations.ContainsKey(network)) + { + continue; + } + var registration = await _transport.JoinNetworkAsync( network, localNodeId, onFrameReceived, localEndpoint, cancellationToken).ConfigureAwait(false); + try + { + await _peerService.RecoverPeersAsync(network, _transport, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } +#pragma warning disable CA1031 // Peer recovery is best-effort during recovery. + catch +#pragma warning restore CA1031 + { + } + _networkRegistrations[network] = registration; - await _peerService.RecoverPeersAsync(network, _transport, cancellationToken).ConfigureAwait(false); } } @@ -213,7 +273,19 @@ public async Task UnregisterAllNetworksAsync(CancellationToken cancellationToken { foreach (var kv in _networkRegistrations) { - await _transport.LeaveNetworkAsync(kv.Key, kv.Value, cancellationToken).ConfigureAwait(false); + await _leaveGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_networkRegistrations.TryGetValue(kv.Key, out var registration)) + { + await _transport.LeaveNetworkAsync(kv.Key, registration, cancellationToken).ConfigureAwait(false); + _networkRegistrations.TryRemove(kv.Key, out _); + } + } + finally + { + _leaveGate.Release(); + } } _networkRegistrations.Clear(); diff --git a/ZTSharp/Internal/NodeRuntimeState.cs b/ZTSharp/Internal/NodeRuntimeState.cs index e2c3e42..5814d78 100644 --- a/ZTSharp/Internal/NodeRuntimeState.cs +++ b/ZTSharp/Internal/NodeRuntimeState.cs @@ -6,6 +6,12 @@ internal sealed class NodeRuntimeState public NodeId NodeId { get; set; } - public bool Disposed { get; set; } + private int _disposed; + + public bool Disposed + { + get => Volatile.Read(ref _disposed) != 0; + set => Volatile.Write(ref _disposed, value ? 1 : 0); + } } diff --git a/ZTSharp/Internal/SecretFilePermissions.cs b/ZTSharp/Internal/SecretFilePermissions.cs deleted file mode 100644 index 75b6904..0000000 --- a/ZTSharp/Internal/SecretFilePermissions.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace ZTSharp.Internal; - -internal static class SecretFilePermissions -{ - public static void TryHardenSecretFile(string path) - { - ArgumentException.ThrowIfNullOrWhiteSpace(path); - - if (OperatingSystem.IsWindows()) - { - return; - } - - try - { - File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); - } - catch (PlatformNotSupportedException) - { - } - catch (IOException) - { - } - catch (UnauthorizedAccessException) - { - } - } -} - diff --git a/ZTSharp/Node.cs b/ZTSharp/Node.cs index d14e21b..f5a923b 100644 --- a/ZTSharp/Node.cs +++ b/ZTSharp/Node.cs @@ -32,6 +32,8 @@ public Node(NodeOptions options) public IPEndPoint? LocalTransportEndpoint => _core.LocalTransportEndpoint; + internal Transport.INodeTransport TransportForTests => _core.TransportForTests; + public NodeId NodeId => _core.NodeId; public bool IsRunning => _core.IsRunning; diff --git a/ZTSharp/Sockets/OverlayTcpClient.cs b/ZTSharp/Sockets/OverlayTcpClient.cs index c60ea0a..cc564ca 100644 --- a/ZTSharp/Sockets/OverlayTcpClient.cs +++ b/ZTSharp/Sockets/OverlayTcpClient.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Sockets; @@ -13,7 +14,16 @@ public sealed class OverlayTcpClient : IAsyncDisposable private const int MaxDataPerFrame = 1024; private readonly OverlayTcpIncomingBuffer _incoming; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); + + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _sendLock = new(1, 1); private readonly Node _node; private readonly ulong _networkId; @@ -125,13 +135,25 @@ internal async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationTok throw new IOException("Remote has closed the connection."); } + if (_incoming.RemoteFinReceived) + { + throw new IOException("Remote has closed the connection."); + } + await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_incoming.RemoteClosed || _incoming.RemoteFinReceived) + { + throw new IOException("Remote has closed the connection."); + } + var remaining = buffer; while (!remaining.IsEmpty) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); var chunk = remaining.Slice(0, Math.Min(remaining.Length, MaxDataPerFrame)); remaining = remaining.Slice(chunk.Length); @@ -174,8 +196,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); - _sendLock.Dispose(); } } @@ -213,7 +233,7 @@ private void OnFrameReceived(in RawFrame frame) if (type == OverlayTcpFrameCodec.FrameType.Fin) { - _incoming.MarkRemoteClosed(); + _incoming.MarkRemoteFinReceived(); return; } diff --git a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs index d9fa117..c1c91ca 100644 --- a/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs +++ b/ZTSharp/Sockets/OverlayTcpIncomingBuffer.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Threading; using System.Threading.Channels; namespace ZTSharp.Sockets; @@ -7,6 +8,8 @@ internal sealed class OverlayTcpIncomingBuffer { private const int MaxQueuedSegments = 1024; private const int MaxSegmentLength = 1024; + private const int FinProbeGracePeriodMs = 50; + private const int FinLateDataGracePeriodMs = 200; private readonly Channel> _incoming = Channel.CreateBounded>(new BoundedChannelOptions(MaxQueuedSegments) { @@ -14,16 +17,22 @@ internal sealed class OverlayTcpIncomingBuffer SingleWriter = false, SingleReader = true }); + private readonly TaskCompletionSource _finArrived = new(TaskCreationOptions.RunContinuationsAsynchronously); private ReadOnlyMemory _currentSegment; private int _currentSegmentOffset; - private bool _remoteClosed; + private int _remoteFinReceived; + private long _remoteFinReceivedAtMs; + private int _remoteClosed; + private long _lastActivityMs; private IOException? _fault; - public bool RemoteClosed => _remoteClosed; + public bool RemoteClosed => Volatile.Read(ref _remoteClosed) != 0; + + public bool RemoteFinReceived => Volatile.Read(ref _remoteFinReceived) != 0; public bool TryWrite(ReadOnlyMemory segment) { - if (_fault is not null) + if (Volatile.Read(ref _fault) is not null) { return false; } @@ -39,18 +48,54 @@ public bool TryWrite(ReadOnlyMemory segment) return false; } + if (Volatile.Read(ref _remoteClosed) != 0) + { + return false; + } + if (_incoming.Writer.TryWrite(segment)) { + Volatile.Write(ref _lastActivityMs, Environment.TickCount64); return true; } + if (Volatile.Read(ref _remoteClosed) != 0) + { + return false; + } + Fault(new IOException("Overlay TCP receive buffer overflowed; closing connection to avoid silent data loss.")); return false; } + public void MarkRemoteFinReceived() + { + if (Interlocked.CompareExchange(ref _remoteFinReceived, 1, 0) == 0) + { + var finReceivedAtMs = Environment.TickCount64; + Volatile.Write(ref _remoteFinReceivedAtMs, finReceivedAtMs); + + while (true) + { + var lastActivityMs = Volatile.Read(ref _lastActivityMs); + if (lastActivityMs >= finReceivedAtMs) + { + break; + } + + if (Interlocked.CompareExchange(ref _lastActivityMs, finReceivedAtMs, lastActivityMs) == lastActivityMs) + { + break; + } + } + + _finArrived.TrySetResult(true); + } + } + public void MarkRemoteClosed() { - _remoteClosed = true; + Volatile.Write(ref _remoteClosed, 1); _incoming.Writer.TryComplete(); } @@ -60,32 +105,117 @@ public void Complete() public async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (_fault is not null) + var fault = Volatile.Read(ref _fault); + if (fault is not null) { - throw _fault; + throw fault; } if (_currentSegment.Length == 0 || _currentSegmentOffset >= _currentSegment.Length) { + if (_currentSegmentOffset >= _currentSegment.Length) + { + _currentSegment = default; + _currentSegmentOffset = 0; + } + while (true) { - if (_incoming.Reader.TryRead(out var segment)) + if (_incoming.Reader.TryRead(out var nextSegment)) { - _currentSegment = segment; + _currentSegment = nextSegment; _currentSegmentOffset = 0; break; } - if (_remoteClosed) + if (Volatile.Read(ref _remoteClosed) != 0) { + fault = Volatile.Read(ref _fault); + if (fault is not null) + { + throw fault; + } + + if (_incoming.Reader.Completion.IsFaulted && + _incoming.Reader.Completion.Exception?.InnerException is IOException ioException) + { + throw ioException; + } + return 0; } try { - _currentSegment = await _incoming.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); - _currentSegmentOffset = 0; - break; + ReadOnlyMemory segment; + if (Volatile.Read(ref _remoteFinReceived) != 0) + { + var now = Environment.TickCount64; + var finReceivedAtMs = Volatile.Read(ref _remoteFinReceivedAtMs); + var lastActivityMs = Volatile.Read(ref _lastActivityMs); + var last = Math.Max(lastActivityMs, finReceivedAtMs); + var gracePeriodMs = last > finReceivedAtMs ? FinLateDataGracePeriodMs : FinProbeGracePeriodMs; + var elapsedMs = now - last; + if (elapsedMs < 0) + { + elapsedMs = 0; + } + + var remainingGraceMs = gracePeriodMs - elapsedMs; + if (remainingGraceMs <= 0) + { + MarkRemoteClosed(); + return 0; + } + + using var graceCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + remainingGraceMs = Math.Min(remainingGraceMs, gracePeriodMs); + graceCts.CancelAfter(TimeSpan.FromMilliseconds(remainingGraceMs)); + try + { + while (await _incoming.Reader.WaitToReadAsync(graceCts.Token).ConfigureAwait(false)) + { + if (_incoming.Reader.TryRead(out segment)) + { + _currentSegment = segment; + _currentSegmentOffset = 0; + break; + } + } + + if (_currentSegment.Length != 0) + { + break; + } + + MarkRemoteClosed(); + return 0; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && graceCts.IsCancellationRequested) + { + MarkRemoteClosed(); + return 0; + } + } + + var waitToReadTask = _incoming.Reader.WaitToReadAsync(cancellationToken).AsTask(); + var completed = await Task.WhenAny(waitToReadTask, _finArrived.Task).ConfigureAwait(false); + if (completed != waitToReadTask) + { + continue; + } + + if (!await waitToReadTask.ConfigureAwait(false)) + { + throw new ChannelClosedException(); + } + + if (_incoming.Reader.TryRead(out segment)) + { + _currentSegment = segment; + _currentSegmentOffset = 0; + break; + } } catch (ChannelClosedException) { @@ -109,13 +239,12 @@ public async ValueTask ReadAsync(Memory buffer, CancellationToken can private void Fault(IOException exception) { - if (_fault is not null) + if (Interlocked.CompareExchange(ref _fault, exception, null) is not null) { return; } - _fault = exception; - _remoteClosed = true; + Volatile.Write(ref _remoteClosed, 1); _incoming.Writer.TryComplete(exception); } } diff --git a/ZTSharp/Sockets/OverlayTcpListener.cs b/ZTSharp/Sockets/OverlayTcpListener.cs index 8e5c6b2..eef582d 100644 --- a/ZTSharp/Sockets/OverlayTcpListener.cs +++ b/ZTSharp/Sockets/OverlayTcpListener.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Diagnostics.CodeAnalysis; using System.Net.Sockets; using System.Threading.Channels; @@ -12,6 +13,10 @@ public sealed class OverlayTcpListener : IAsyncDisposable private const int HeaderLength = OverlayTcpFrameCodec.HeaderLength; private readonly Channel _acceptQueue; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly Node _node; private readonly ulong _networkId; @@ -32,9 +37,10 @@ public OverlayTcpListener(Node node, ulong networkId, int localPort) _localPort = localPort; _acceptQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 128) { - FullMode = BoundedChannelFullMode.DropWrite, + FullMode = BoundedChannelFullMode.Wait, SingleWriter = false, - SingleReader = true + // DisposeAsync drains queued connections for best-effort cleanup, so we must allow an additional reader. + SingleReader = false }); _node.RawFrameReceived += OnFrameReceived; @@ -56,11 +62,14 @@ public async ValueTask DisposeAsync() _disposed = true; _node.RawFrameReceived -= OnFrameReceived; _acceptQueue.Writer.TryComplete(); + while (_acceptQueue.Reader.TryRead(out var queued)) + { + ObserveBestEffortAsync(queued.DisposeAsync().AsTask()); + } } finally { _disposeLock.Release(); - _disposeLock.Dispose(); } } diff --git a/ZTSharp/Sockets/OverlayTcpPortForwarder.cs b/ZTSharp/Sockets/OverlayTcpPortForwarder.cs index 3f0fb29..25d0813 100644 --- a/ZTSharp/Sockets/OverlayTcpPortForwarder.cs +++ b/ZTSharp/Sockets/OverlayTcpPortForwarder.cs @@ -1,4 +1,5 @@ using System.Net.Sockets; +using System.Diagnostics.CodeAnalysis; using System.Threading.Channels; using ZTSharp.Internal; using SystemTcpClient = System.Net.Sockets.TcpClient; @@ -10,6 +11,10 @@ namespace ZTSharp.Sockets; /// public sealed class OverlayTcpPortForwarder : IAsyncDisposable { + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly CancellationTokenSource _shutdown = new(); private readonly ActiveTaskSet _connectionTasks = new(); @@ -111,7 +116,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); _shutdown.Dispose(); } } diff --git a/ZTSharp/Sockets/ZtTcpListener.cs b/ZTSharp/Sockets/ZtTcpListener.cs index 8e41e54..118844c 100644 --- a/ZTSharp/Sockets/ZtTcpListener.cs +++ b/ZTSharp/Sockets/ZtTcpListener.cs @@ -12,6 +12,7 @@ public sealed class ZtTcpListener : IAsyncDisposable { private readonly SystemTcpListener _listener; private bool _started; + private int _disposeState; public ZtTcpListener(IPAddress address, int port) { @@ -30,6 +31,7 @@ public void Start(int backlog = 100) public async Task AcceptTcpClientAsync(CancellationToken cancellationToken = default) { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this); if (!_started) { _listener.Start(); @@ -42,10 +44,29 @@ public async Task AcceptTcpClientAsync(CancellationToken cancellati return new ZtTcpClient(client); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - await Task.Yield(); - _listener.Stop(); - _listener.Dispose(); + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + { + return ValueTask.CompletedTask; + } + + try + { + _listener.Stop(); + } + catch (ObjectDisposedException) + { + } + + try + { + _listener.Dispose(); + } + catch (ObjectDisposedException) + { + } + + return ValueTask.CompletedTask; } } diff --git a/ZTSharp/Sockets/ZtUdpClient.cs b/ZTSharp/Sockets/ZtUdpClient.cs index bd16561..7108fbf 100644 --- a/ZTSharp/Sockets/ZtUdpClient.cs +++ b/ZTSharp/Sockets/ZtUdpClient.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Buffers; +using System.Diagnostics.CodeAnalysis; using System.Threading.Channels; namespace ZTSharp.Sockets; @@ -16,6 +17,10 @@ public sealed class ZtUdpClient : IAsyncDisposable private const int UdpFrameHeaderV2Length = 14; private readonly Channel _incoming; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly ulong _networkId; private readonly int _localPort; @@ -134,7 +139,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); } } diff --git a/ZTSharp/StateStoreKeySegmentValidation.cs b/ZTSharp/StateStoreKeySegmentValidation.cs index 2b62e8a..d7e1e02 100644 --- a/ZTSharp/StateStoreKeySegmentValidation.cs +++ b/ZTSharp/StateStoreKeySegmentValidation.cs @@ -5,9 +5,12 @@ internal static class StateStoreKeySegmentValidation private static readonly string[] ReservedDeviceNames = [ "CON", + "CONIN$", + "CONOUT$", "PRN", "AUX", "NUL", + "CLOCK$", "COM1", "COM2", "COM3", diff --git a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs index 3af5f7a..f874a02 100644 --- a/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs +++ b/ZTSharp/Transport/Internal/OsUdpPeerRegistry.cs @@ -7,20 +7,26 @@ namespace ZTSharp.Transport.Internal; internal sealed class OsUdpPeerRegistry { - private readonly record struct PeerDirectoryEntry(IPEndPoint Endpoint, long LastSeenTicks); + internal enum PeerEntrySource + { + Manual = 0, + Directory = 1 + } + + internal readonly record struct PeerEntry(IPEndPoint Endpoint, long LastSeenTicks, PeerEntrySource Source); private const int DirectoryMaxNetworks = 256; internal const int DirectoryMaxPeersPerNetwork = 1024; internal static readonly TimeSpan DirectoryPeerTtl = TimeSpan.FromMinutes(5); private static readonly long DirectoryPeerTtlTicks = DirectoryPeerTtl.Ticks; - private static readonly ConcurrentDictionary> s_networkDirectory = new(); + private static readonly ConcurrentDictionary> s_networkDirectory = new(); private readonly bool _enablePeerDiscovery; private readonly Func _normalizeEndpoint; private readonly TimeProvider _timeProvider; - private readonly ConcurrentDictionary> _networkPeers = new(); + private readonly ConcurrentDictionary> _networkPeers = new(); private readonly ConcurrentDictionary _localNodeIds = new(); public OsUdpPeerRegistry(bool enablePeerDiscovery, Func normalizeEndpoint, TimeProvider? timeProvider = null) @@ -50,17 +56,52 @@ public bool TryRemoveLocalNodeIdIfMatch(ulong networkId, ulong expectedNodeId) return true; } - public bool TryGetPeers(ulong networkId, out ConcurrentDictionary peers) - => _networkPeers.TryGetValue(networkId, out peers!); + public bool TryGetPeers(ulong networkId, out ConcurrentDictionary peers) + { + if (!_networkPeers.TryGetValue(networkId, out peers!)) + { + return false; + } + + var nowTicks = GetNowTicks(); + ImportDirectoryPeers(networkId, peers, nowTicks); + EvictExpiredAndTrimNetworkPeers(networkId, peers, nowTicks); + return true; + } public void RemoveNetworkPeers(ulong networkId) => _networkPeers.TryRemove(networkId, out _); + public void RefreshPeerLastSeen(ulong networkId, ulong nodeId) + { + if (!_networkPeers.TryGetValue(networkId, out var peers)) + { + return; + } + + var nowTicks = GetNowTicks(); + while (peers.TryGetValue(nodeId, out var existing)) + { + if (existing.LastSeenTicks >= nowTicks) + { + return; + } + + var updated = existing with { LastSeenTicks = nowTicks }; + if (peers.TryUpdate(nodeId, updated, existing)) + { + return; + } + } + } + public void AddOrUpdatePeer(ulong networkId, ulong nodeId, IPEndPoint endpoint) { + var nowTicks = GetNowTicks(); var normalized = _normalizeEndpoint(endpoint); - var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - peers[nodeId] = normalized; + var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + peers[nodeId] = new PeerEntry(normalized, nowTicks, PeerEntrySource.Manual); + SweepNetworkPeers(nowTicks); } public IEnumerable> RegisterLocalAndGetKnownPeers( @@ -77,11 +118,11 @@ public IEnumerable> RegisterLocalAndGetKnownPeer var nowTicks = GetNowTicks(); var normalizedAdvertisedEndpoint = _normalizeEndpoint(advertisedEndpoint); - var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - discoveredPeers[localNodeId] = new PeerDirectoryEntry(normalizedAdvertisedEndpoint, nowTicks); + var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks, PeerEntrySource.Directory); SweepDirectory(nowTicks); - var localPeers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + var localPeers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); foreach (var peer in discoveredPeers) { if (peer.Key == localNodeId) @@ -89,29 +130,39 @@ public IEnumerable> RegisterLocalAndGetKnownPeer continue; } - localPeers[peer.Key] = peer.Value.Endpoint; + localPeers[peer.Key] = new PeerEntry(peer.Value.Endpoint, nowTicks, PeerEntrySource.Directory); } + PruneDirectoryPeers(localPeers, discoveredPeers, localNodeId); + return discoveredPeers .Where(p => p.Key != localNodeId) .Select(p => new KeyValuePair(p.Key, p.Value.Endpoint)) .ToArray(); } - public void RegisterDiscoveredPeer(ulong networkId, ulong sourceNodeId, IPEndPoint remoteEndpoint) + public void RefreshLocalRegistration(ulong networkId, ulong localNodeId, IPEndPoint advertisedEndpoint) { if (!_enablePeerDiscovery) { return; } + if (!_localNodeIds.TryGetValue(networkId, out var registeredNodeId) || registeredNodeId != localNodeId) + { + return; + } + var nowTicks = GetNowTicks(); - var endpoint = _normalizeEndpoint(remoteEndpoint); - var peers = _networkPeers.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - peers[sourceNodeId] = endpoint; - var directoryPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); - directoryPeers[sourceNodeId] = new PeerDirectoryEntry(endpoint, nowTicks); + var normalizedAdvertisedEndpoint = _normalizeEndpoint(advertisedEndpoint); + var discoveredPeers = s_networkDirectory.GetOrAdd(networkId, _ => new ConcurrentDictionary()); + discoveredPeers[localNodeId] = new PeerEntry(normalizedAdvertisedEndpoint, nowTicks, PeerEntrySource.Directory); SweepDirectory(nowTicks); + + if (_networkPeers.TryGetValue(networkId, out var localPeers)) + { + PruneDirectoryPeers(localPeers, discoveredPeers, localNodeId); + } } public void Cleanup() @@ -123,6 +174,7 @@ public void Cleanup() } SweepDirectory(nowTicks); + SweepNetworkPeers(nowTicks); _networkPeers.Clear(); _localNodeIds.Clear(); } @@ -142,21 +194,21 @@ private static void RemoveFromDirectory(ulong networkId, ulong nodeId) private long GetNowTicks() => _timeProvider.GetUtcNow().UtcDateTime.Ticks; - private static void EvictExpiredAndTrimNetwork(ulong networkId, ConcurrentDictionary discoveredPeers, long nowTicks) + private static void EvictExpiredAndTrimPeers(ConcurrentDictionary peers, long nowTicks) { var cutoffTicks = nowTicks - DirectoryPeerTtlTicks; - foreach (var peer in discoveredPeers) + foreach (var peer in peers) { if (peer.Value.LastSeenTicks < cutoffTicks) { - discoveredPeers.TryRemove(peer.Key, out _); + peers.TryRemove(peer.Key, out _); } } - if (discoveredPeers.Count > DirectoryMaxPeersPerNetwork) + if (peers.Count > DirectoryMaxPeersPerNetwork) { - var removeCount = discoveredPeers.Count - DirectoryMaxPeersPerNetwork; - var toRemove = discoveredPeers + var removeCount = peers.Count - DirectoryMaxPeersPerNetwork; + var toRemove = peers .OrderBy(p => p.Value.LastSeenTicks) .Take(removeCount) .Select(p => p.Key) @@ -164,13 +216,56 @@ private static void EvictExpiredAndTrimNetwork(ulong networkId, ConcurrentDictio foreach (var key in toRemove) { - discoveredPeers.TryRemove(key, out _); + peers.TryRemove(key, out _); } } + } - if (discoveredPeers.IsEmpty) + private void EvictExpiredAndTrimNetworkPeers(ulong networkId, ConcurrentDictionary peers, long nowTicks) + { + EvictExpiredAndTrimPeers(peers, nowTicks); + if (peers.IsEmpty && (!_localNodeIds.TryGetValue(networkId, out var localNodeId) || localNodeId == 0)) { - s_networkDirectory.TryRemove(networkId, out _); + _networkPeers.TryRemove(networkId, out _); + } + } + + private void SweepNetworkPeers(long nowTicks) + { + foreach (var network in _networkPeers) + { + EvictExpiredAndTrimNetworkPeers(network.Key, network.Value, nowTicks); + } + + if (_networkPeers.Count <= DirectoryMaxNetworks) + { + return; + } + + var overflow = _networkPeers.Count - DirectoryMaxNetworks; + var toRemove = _networkPeers + .Where(network => !_localNodeIds.TryGetValue(network.Key, out var localNodeId) || localNodeId == 0) + .Select(network => + { + var lastSeen = 0L; + foreach (var peer in network.Value.Values) + { + if (peer.LastSeenTicks > lastSeen) + { + lastSeen = peer.LastSeenTicks; + } + } + + return (NetworkId: network.Key, LastSeenTicks: lastSeen); + }) + .OrderBy(network => network.LastSeenTicks) + .Take(overflow) + .Select(network => network.NetworkId) + .ToArray(); + + foreach (var networkId in toRemove) + { + _networkPeers.TryRemove(networkId, out _); } } @@ -178,7 +273,11 @@ private static void SweepDirectory(long nowTicks) { foreach (var network in s_networkDirectory) { - EvictExpiredAndTrimNetwork(network.Key, network.Value, nowTicks); + EvictExpiredAndTrimPeers(network.Value, nowTicks); + if (network.Value.IsEmpty) + { + s_networkDirectory.TryRemove(network.Key, out _); + } } if (s_networkDirectory.Count <= DirectoryMaxNetworks) @@ -212,6 +311,70 @@ private static void SweepDirectory(long nowTicks) } } + private static void PruneDirectoryPeers( + ConcurrentDictionary localPeers, + ConcurrentDictionary discoveredPeers, + ulong localNodeId) + { + foreach (var peer in localPeers) + { + if (peer.Value.Source != PeerEntrySource.Directory || + peer.Key == localNodeId || + discoveredPeers.ContainsKey(peer.Key)) + { + continue; + } + + localPeers.TryRemove(peer.Key, out _); + } + } + + private static void RemoveAllDirectoryPeers(ConcurrentDictionary peers) + { + foreach (var peer in peers) + { + if (peer.Value.Source == PeerEntrySource.Directory) + { + peers.TryRemove(peer.Key, out _); + } + } + } + + private void ImportDirectoryPeers(ulong networkId, ConcurrentDictionary peers, long nowTicks) + { + if (!_enablePeerDiscovery) + { + return; + } + + if (!s_networkDirectory.TryGetValue(networkId, out var directoryPeers)) + { + RemoveAllDirectoryPeers(peers); + return; + } + + EvictExpiredAndTrimPeers(directoryPeers, nowTicks); + if (directoryPeers.IsEmpty) + { + s_networkDirectory.TryRemove(networkId, out _); + RemoveAllDirectoryPeers(peers); + return; + } + + _localNodeIds.TryGetValue(networkId, out var localNodeId); + foreach (var peer in directoryPeers) + { + if (peer.Key == localNodeId) + { + continue; + } + + peers[peer.Key] = new PeerEntry(peer.Value.Endpoint, nowTicks, PeerEntrySource.Directory); + } + + PruneDirectoryPeers(peers, directoryPeers, localNodeId); + } + internal static void ClearDirectoryForTests() => s_networkDirectory.Clear(); diff --git a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs index 6d0a29f..9d02700 100644 --- a/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs +++ b/ZTSharp/Transport/Internal/OsUdpReceiveLoop.cs @@ -1,10 +1,23 @@ +using System.Collections.Generic; using System.Net; using System.Net.Sockets; +using System.Threading.Channels; namespace ZTSharp.Transport.Internal; internal sealed class OsUdpReceiveLoop { + private const int MaxHelloResponseCacheEntries = 4096; + private const int MaxHelloResponseEvictionQueueEntries = MaxHelloResponseCacheEntries * 2; + private const int MaxPendingDiscoverySends = 256; + private const long PeerHelloResponseMinIntervalMs = 1000; + + private readonly record struct DiscoverySendRequest( + ulong NetworkId, + ulong LocalNodeId, + IPEndPoint Endpoint, + OsUdpPeerDiscoveryProtocol.FrameType FrameType); + private readonly UdpClient _udp; private readonly Func> _receiveAsync; private readonly bool _enablePeerDiscovery; @@ -12,6 +25,11 @@ internal sealed class OsUdpReceiveLoop private readonly Func, CancellationToken, Task> _dispatchFrameAsync; private readonly Func _sendDiscoveryFrameAsync; private readonly Action? _log; + private readonly Dictionary<(ulong NetworkId, ulong NodeId), long> _helloResponseLastSentMs = new(); + private readonly Queue<((ulong NetworkId, ulong NodeId) Key, long LastSentMs)> _helloResponseEvictionQueue = new(); + private readonly Channel _discoverySendQueue; + + internal Action? DatagramReceivedForTests { get; set; } public OsUdpReceiveLoop( UdpClient udp, @@ -34,94 +52,206 @@ public OsUdpReceiveLoop( _dispatchFrameAsync = dispatchFrameAsync; _sendDiscoveryFrameAsync = sendDiscoveryFrameAsync; _log = log; + + _discoverySendQueue = Channel.CreateBounded(new BoundedChannelOptions(MaxPendingDiscoverySends) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = true, + SingleWriter = true + }); } public async Task RunAsync(CancellationToken cancellationToken) { - while (!cancellationToken.IsCancellationRequested) + var discoverySendLoop = _enablePeerDiscovery + ? RunDiscoverySendLoopAsync(cancellationToken) + : Task.CompletedTask; + + try { - UdpReceiveResult result; - try - { - result = await _receiveAsync(cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) + while (!cancellationToken.IsCancellationRequested) { - return; - } - catch (ObjectDisposedException) - { - return; - } - catch (SocketException ex) when (ex.SocketErrorCode == SocketError.ConnectionReset) - { - continue; - } - catch (SocketException ex) - { - _log?.Invoke($"OS UDP receive failed (SocketException {ex.SocketErrorCode}: {ex.Message})."); - continue; - } - catch (InvalidOperationException ex) - { - _log?.Invoke($"OS UDP receive failed (InvalidOperationException: {ex.Message})."); - continue; - } + UdpReceiveResult result; + try + { + result = await _receiveAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + catch (ObjectDisposedException) + { + return; + } + catch (SocketException ex) when (ex.SocketErrorCode == SocketError.ConnectionReset) + { + continue; + } + catch (SocketException ex) + { + _log?.Invoke($"OS UDP receive failed (SocketException {ex.SocketErrorCode}: {ex.Message})."); + continue; + } + catch (InvalidOperationException ex) + { + _log?.Invoke($"OS UDP receive failed (InvalidOperationException: {ex.Message})."); + continue; + } - if (!NodeFrameCodec.TryDecode(result.Buffer.AsMemory(), out var networkId, out var sourceNodeId, out var payload)) - { - continue; - } + var normalizedRemoteEndpoint = UdpEndpointNormalization.Normalize(result.RemoteEndPoint); + DatagramReceivedForTests?.Invoke(normalizedRemoteEndpoint); - var normalizedRemoteEndpoint = UdpEndpointNormalization.Normalize(result.RemoteEndPoint); - if (_enablePeerDiscovery && OsUdpPeerDiscoveryProtocol.TryParsePayload(payload.Span, networkId, out var controlFrameType, out var discoveredNodeId)) - { - if (discoveredNodeId != 0 && discoveredNodeId == sourceNodeId) + if (!NodeFrameCodec.TryDecode(result.Buffer.AsMemory(), out var networkId, out var sourceNodeId, out var payload)) + { + continue; + } + + if (_enablePeerDiscovery && + _peers.TryGetLocalNodeId(networkId, out var localNodeId) && + localNodeId != 0 && + OsUdpPeerDiscoveryProtocol.TryParsePayload(payload.Span, networkId, out var controlFrameType, out var discoveredNodeId)) { - _peers.RegisterDiscoveredPeer(networkId, discoveredNodeId, normalizedRemoteEndpoint); - if (_peers.TryGetLocalNodeId(networkId, out var localNodeId) && localNodeId != discoveredNodeId) + if (discoveredNodeId != 0 && discoveredNodeId == sourceNodeId) { - if (controlFrameType == OsUdpPeerDiscoveryProtocol.FrameType.PeerHello) + if (localNodeId != discoveredNodeId && + controlFrameType == OsUdpPeerDiscoveryProtocol.FrameType.PeerHello && + IPAddress.IsLoopback(normalizedRemoteEndpoint.Address) && + ShouldSendHelloResponse(networkId, discoveredNodeId)) { - _ = SendDiscoveryFrameSafeAsync( + QueueDiscoverySend( networkId, localNodeId, normalizedRemoteEndpoint, - OsUdpPeerDiscoveryProtocol.FrameType.PeerHelloResponse, - cancellationToken); + OsUdpPeerDiscoveryProtocol.FrameType.PeerHelloResponse); } } + + continue; } - continue; - } + if (sourceNodeId == 0 || + !_peers.TryGetPeers(networkId, out var peers) || + !peers.TryGetValue(sourceNodeId, out var expectedEntry)) + { + continue; + } + + var expectedEndpoint = expectedEntry.Endpoint; + if (!expectedEndpoint.Equals(normalizedRemoteEndpoint)) + { + continue; + } - if (sourceNodeId != 0 && - _peers.TryGetPeers(networkId, out var peers) && - peers.TryGetValue(sourceNodeId, out var expectedEndpoint) && - expectedEndpoint.Equals(normalizedRemoteEndpoint)) + try + { + _peers.RefreshPeerLastSeen(networkId, sourceNodeId); + await _dispatchFrameAsync(sourceNodeId, networkId, payload, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } +#pragma warning disable CA1031 // Receive loop must survive dispatch failures. + catch (Exception) +#pragma warning restore CA1031 + { + } + } + } + finally + { + _discoverySendQueue.Writer.TryComplete(); + try { - // Valid authenticated-by-endpoint peer. + await discoverySendLoop.ConfigureAwait(false); } - else + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - continue; } + } + } - try + private bool ShouldSendHelloResponse(ulong networkId, ulong remoteNodeId) + { + var key = (networkId, remoteNodeId); + var nowMs = Environment.TickCount64; + if (_helloResponseLastSentMs.TryGetValue(key, out var lastMs)) + { + if (unchecked(nowMs - lastMs) < PeerHelloResponseMinIntervalMs) { - await _dispatchFrameAsync(sourceNodeId, networkId, payload, cancellationToken).ConfigureAwait(false); + return false; } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + } + + _helloResponseLastSentMs[key] = nowMs; + _helloResponseEvictionQueue.Enqueue((key, nowMs)); + TrimHelloResponseEvictionQueueIfNeeded(); + + if (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) + { + TrimHelloResponseCache(); + } + + return true; + } + + private void TrimHelloResponseEvictionQueueIfNeeded() + { + while (_helloResponseEvictionQueue.Count > MaxHelloResponseEvictionQueueEntries) + { + _helloResponseEvictionQueue.Dequeue(); + } + } + + private void TrimHelloResponseCache() + { + while (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries && _helloResponseEvictionQueue.Count > 0) + { + var (key, lastSentMs) = _helloResponseEvictionQueue.Dequeue(); + if (_helloResponseLastSentMs.TryGetValue(key, out var currentLastSentMs) && currentLastSentMs == lastSentMs) { - return; + _helloResponseLastSentMs.Remove(key); } -#pragma warning disable CA1031 // Receive loop must survive dispatch failures. - catch (Exception) -#pragma warning restore CA1031 + } + + while (_helloResponseLastSentMs.Count > MaxHelloResponseCacheEntries) + { + using var enumerator = _helloResponseLastSentMs.Keys.GetEnumerator(); + if (!enumerator.MoveNext()) + { + break; + } + + _helloResponseLastSentMs.Remove(enumerator.Current); + } + } + + private void QueueDiscoverySend( + ulong networkId, + ulong localNodeId, + IPEndPoint endpoint, + OsUdpPeerDiscoveryProtocol.FrameType frameType) + => _discoverySendQueue.Writer.TryWrite(new DiscoverySendRequest(networkId, localNodeId, endpoint, frameType)); + + private async Task RunDiscoverySendLoopAsync(CancellationToken cancellationToken) + { + try + { + await foreach (var request in _discoverySendQueue.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { + await SendDiscoveryFrameSafeAsync( + request.NetworkId, + request.LocalNodeId, + request.Endpoint, + request.FrameType, + cancellationToken) + .ConfigureAwait(false); } } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } } private async Task SendDiscoveryFrameSafeAsync( diff --git a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs index 58c2f69..f5a4aa5 100644 --- a/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs +++ b/ZTSharp/Transport/Internal/OsUdpSocketFactory.cs @@ -8,9 +8,11 @@ internal static class OsUdpSocketFactory { private const int WindowsSioUdpConnReset = unchecked((int)0x9800000C); - public static UdpClient Create(int localPort, bool enableIpv6, Action? log = null) + public static UdpClient Create(int localPort, bool enableIpv6, Action? log = null, IPAddress? localBindAddress = null) { - var udp = CreateSocketCore(localPort, enableIpv6); + var udp = localBindAddress is null + ? CreateSocketCore(localPort, enableIpv6) + : CreateBoundSocket(localBindAddress, localPort); TryDisableWindowsUdpConnReset(udp, log); return udp; } @@ -41,12 +43,23 @@ internal static UdpClient CreateSocketCore( return createUdp4Bound(localPort); } + Exception? lastException = null; try { return createUdp6DualModeBound(localPort); } catch (Exception ex) when (ex is SocketException or PlatformNotSupportedException or NotSupportedException) { + lastException = ex; + } + + try + { + return createUdp4Bound(localPort); + } + catch (Exception ex) when (ex is SocketException or PlatformNotSupportedException or NotSupportedException) + { + lastException = ex; } try @@ -55,31 +68,78 @@ internal static UdpClient CreateSocketCore( } catch (Exception ex) when (ex is SocketException or PlatformNotSupportedException or NotSupportedException) { + lastException = ex; } - return createUdp4Bound(localPort); + throw lastException!; + } + + private static UdpClient CreateBoundSocket(IPAddress localBindAddress, int localPort) + { + ArgumentNullException.ThrowIfNull(localBindAddress); + + return localBindAddress.AddressFamily switch + { + AddressFamily.InterNetwork => CreateUdp4Bound(new IPEndPoint(localBindAddress, localPort)), + AddressFamily.InterNetworkV6 => CreateUdp6OnlyBound(new IPEndPoint(localBindAddress, localPort)), + _ => throw new NotSupportedException($"Unsupported UDP bind address family: {localBindAddress.AddressFamily}.") + }; } private static UdpClient CreateUdp4Bound(int localPort) + => CreateUdp4Bound(new IPEndPoint(IPAddress.Any, localPort)); + + private static UdpClient CreateUdp4Bound(IPEndPoint localEndPoint) { var udp4 = new UdpClient(AddressFamily.InterNetwork); - udp4.Client.Bind(new IPEndPoint(IPAddress.Any, localPort)); - return udp4; + try + { + udp4.Client.Bind(localEndPoint); + return udp4; + } + catch + { + udp4.Dispose(); + throw; + } } private static UdpClient CreateUdp6DualModeBound(int localPort) + => CreateUdp6DualModeBound(new IPEndPoint(IPAddress.IPv6Any, localPort)); + + private static UdpClient CreateUdp6DualModeBound(IPEndPoint localEndPoint) { var udp6 = new UdpClient(AddressFamily.InterNetworkV6); - udp6.Client.DualMode = true; - udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); - return udp6; + try + { + udp6.Client.DualMode = true; + udp6.Client.Bind(localEndPoint); + return udp6; + } + catch + { + udp6.Dispose(); + throw; + } } private static UdpClient CreateUdp6OnlyBound(int localPort) + => CreateUdp6OnlyBound(new IPEndPoint(IPAddress.IPv6Any, localPort)); + + private static UdpClient CreateUdp6OnlyBound(IPEndPoint localEndPoint) { var udp6 = new UdpClient(AddressFamily.InterNetworkV6); - udp6.Client.Bind(new IPEndPoint(IPAddress.IPv6Any, localPort)); - return udp6; + try + { + udp6.Client.DualMode = false; + udp6.Client.Bind(localEndPoint); + return udp6; + } + catch + { + udp6.Dispose(); + throw; + } } private static void TryDisableWindowsUdpConnReset(UdpClient udp, Action? log) diff --git a/ZTSharp/Transport/OsUdpNodeTransport.cs b/ZTSharp/Transport/OsUdpNodeTransport.cs index 5030596..c149c86 100644 --- a/ZTSharp/Transport/OsUdpNodeTransport.cs +++ b/ZTSharp/Transport/OsUdpNodeTransport.cs @@ -2,6 +2,7 @@ using System.Buffers; using System.Net; using System.Net.Sockets; +using System.Diagnostics.CodeAnalysis; using ZTSharp.Transport.Internal; namespace ZTSharp.Transport; @@ -11,18 +12,31 @@ namespace ZTSharp.Transport; /// internal sealed class OsUdpNodeTransport : INodeTransport, IAsyncDisposable { + private static readonly TimeSpan PeerDiscoveryRefreshInterval = TimeSpan.FromSeconds(90); + private static readonly TimeSpan ShutdownTimeout = TimeSpan.FromSeconds(1); + private sealed record Subscriber( ulong NodeId, Func, CancellationToken, Task> OnFrameReceived); private readonly UdpClient _udp; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _gate = new(1, 1); private readonly ConcurrentDictionary> _networkSubscribers = new(); + private readonly ConcurrentDictionary _advertisedEndpoints = new(); private readonly OsUdpPeerRegistry _peers; private readonly OsUdpReceiveLoop _receiver; private readonly CancellationTokenSource _receiverCts = new(); private readonly Task _receiverLoop; private readonly bool _enablePeerDiscovery; + private readonly CancellationTokenSource _peerRefreshCts = new(); + private readonly Task? _peerRefreshLoop; + + private int _disposeState; + private volatile bool _disposed; public OsUdpNodeTransport(int localPort = 0, bool enableIpv6 = true, bool enablePeerDiscovery = true) { @@ -38,12 +52,18 @@ public OsUdpNodeTransport(int localPort = 0, bool enableIpv6 = true, bool enable SendDiscoveryFrameAsync); _receiverLoop = Task.Run(() => _receiver.RunAsync(_receiverCts.Token)); + + if (enablePeerDiscovery) + { + _peerRefreshLoop = Task.Run(() => RefreshLocalDiscoveryEntriesAsync(_peerRefreshCts.Token)); + } } public IPEndPoint LocalEndpoint { get { + ObjectDisposedException.ThrowIf(_disposed, this); return UdpEndpointNormalization.Normalize((IPEndPoint)_udp.Client.LocalEndPoint!); } } @@ -58,11 +78,13 @@ public async Task JoinNetworkAsync( ArgumentOutOfRangeException.ThrowIfZero(nodeId); ArgumentNullException.ThrowIfNull(onFrameReceived); cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); var registrationId = Guid.NewGuid(); var advertisedEndpoint = localEndpoint is null ? UdpEndpointNormalization.NormalizeForAdvertisement(LocalEndpoint) : UdpEndpointNormalization.NormalizeForAdvertisement(localEndpoint); + _advertisedEndpoints[networkId] = advertisedEndpoint; var subscribers = _networkSubscribers.GetOrAdd( networkId, _ => new ConcurrentDictionary()); @@ -84,26 +106,30 @@ await SendDiscoveryFrameAsync( public async Task LeaveNetworkAsync(ulong networkId, Guid registrationId, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - if (_networkSubscribers.TryGetValue(networkId, out var subscribers) && - subscribers.TryGetValue(registrationId, out var localSubscriber)) - { - _ = _peers.TryRemoveLocalNodeIdIfMatch(networkId, localSubscriber.NodeId); - } - - _peers.RemoveNetworkPeers(networkId); - if (!_networkSubscribers.TryGetValue(networkId, out var networkSubscribers)) - { - return; - } + ObjectDisposedException.ThrowIf(_disposed, this); await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { - networkSubscribers.TryRemove(registrationId, out _); - if (networkSubscribers.IsEmpty) + if (!_networkSubscribers.TryGetValue(networkId, out var networkSubscribers)) + { + return; + } + + if (!networkSubscribers.TryRemove(registrationId, out var localSubscriber)) + { + return; + } + + if (!networkSubscribers.IsEmpty) { - _networkSubscribers.TryRemove(networkId, out _); + return; } + + _networkSubscribers.TryRemove(networkId, out _); + _advertisedEndpoints.TryRemove(networkId, out _); + _ = _peers.TryRemoveLocalNodeIdIfMatch(networkId, localSubscriber.NodeId); + _peers.RemoveNetworkPeers(networkId); } finally { @@ -118,6 +144,7 @@ public async Task SendFrameAsync( CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); if (!_peers.TryGetPeers(networkId, out var peers)) { return; @@ -147,8 +174,9 @@ public async Task SendFrameAsync( try { await _udp - .SendAsync(frame, peer.Value, cancellationToken) + .SendAsync(frame, peer.Value.Endpoint, cancellationToken) .ConfigureAwait(false); + _peers.RefreshPeerLastSeen(networkId, peer.Key); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -169,6 +197,7 @@ await _udp public Task FlushAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); return Task.CompletedTask; } @@ -176,6 +205,7 @@ public async ValueTask AddPeerAsync(ulong networkId, ulong nodeId, IPEndPoint en { ArgumentOutOfRangeException.ThrowIfZero(nodeId); ArgumentNullException.ThrowIfNull(endpoint); + ObjectDisposedException.ThrowIf(_disposed, this); UdpEndpointNormalization.ValidateRemoteEndpoint(endpoint, nameof(endpoint)); var remoteEndpoint = UdpEndpointNormalization.Normalize(endpoint); @@ -208,19 +238,85 @@ await SendDiscoveryFrameAsync( public async ValueTask DisposeAsync() { - await _receiverCts.CancelAsync().ConfigureAwait(false); + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + { + return; + } + + _disposed = true; try { - await _receiverLoop.ConfigureAwait(false); + await _receiverCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + + try + { + await _peerRefreshCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + + try + { + await _receiverLoop.WaitAsync(ShutdownTimeout).ConfigureAwait(false); + } + catch (TimeoutException) + { } catch (OperationCanceledException) when (_receiverCts.IsCancellationRequested) { } + catch (ObjectDisposedException) + { + } + + if (_peerRefreshLoop is not null) + { + try + { + await _peerRefreshLoop.WaitAsync(ShutdownTimeout).ConfigureAwait(false); + } + catch (TimeoutException) + { + } + catch (OperationCanceledException) when (_peerRefreshCts.IsCancellationRequested) + { + } + catch (ObjectDisposedException) + { + } + } + + try + { + _udp.Dispose(); + } + catch (ObjectDisposedException) + { + } - _udp.Dispose(); _peers.Cleanup(); _receiverCts.Dispose(); - _gate.Dispose(); + _peerRefreshCts.Dispose(); + } + + private async Task RefreshLocalDiscoveryEntriesAsync(CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(PeerDiscoveryRefreshInterval); + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var entry in _advertisedEndpoints) + { + if (_peers.TryGetLocalNodeId(entry.Key, out var localNodeId) && localNodeId != 0) + { + _peers.RefreshLocalRegistration(entry.Key, localNodeId, entry.Value); + } + } + } } private async Task DispatchFrameAsync(ulong sourceNodeId, ulong networkId, ReadOnlyMemory payload, CancellationToken cancellationToken) @@ -278,5 +374,7 @@ await _udp } } + internal void SetDatagramObserverForTests(Action? observer) + => _receiver.DatagramReceivedForTests = observer; } diff --git a/ZTSharp/VirtualNetworkInterface.cs b/ZTSharp/VirtualNetworkInterface.cs index 8004f4e..02a3c86 100644 --- a/ZTSharp/VirtualNetworkInterface.cs +++ b/ZTSharp/VirtualNetworkInterface.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Buffers.Binary; +using System.Diagnostics.CodeAnalysis; using System.Threading.Channels; namespace ZTSharp; @@ -14,6 +15,10 @@ public sealed class VirtualNetworkInterface : IAsyncDisposable private const int HeaderLength = 1 + 1 + sizeof(ulong); private readonly Channel _incoming; + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw on subsequent/overlapping DisposeAsync calls.")] private readonly SemaphoreSlim _disposeLock = new(1, 1); private readonly Node _node; private readonly ulong _networkId; @@ -57,7 +62,6 @@ public async ValueTask DisposeAsync() finally { _disposeLock.Release(); - _disposeLock.Dispose(); } } @@ -114,4 +118,3 @@ private void OnFrameReceived(in RawFrame frame) DateTimeOffset.UtcNow)); } } - diff --git a/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs b/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs index 36ecbcc..4ad3a6d 100644 --- a/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs +++ b/ZTSharp/ZeroTier/Http/ZeroTierHttpMessageHandler.cs @@ -8,7 +8,6 @@ namespace ZTSharp.ZeroTier.Http; public sealed class ZeroTierHttpMessageHandler : DelegatingHandler { private readonly ZeroTier.ZeroTierSocket _socket; - private static readonly TimeSpan DefaultPerAddressConnectTimeout = TimeSpan.FromSeconds(2); public ZeroTierHttpMessageHandler(ZeroTier.ZeroTierSocket socket) { @@ -35,7 +34,7 @@ private async ValueTask ConnectAsync(SocketsHttpConnectionContext contex endpoint, new[] { ip }, connectAsync: (ep, ct) => _socket.ConnectTcpAsync(ep, ct), - perAddressConnectTimeout: DefaultPerAddressConnectTimeout, + perAddressConnectTimeout: _socket.SuggestedHttpConnectTimeout, cancellationToken) .ConfigureAwait(false); } @@ -68,7 +67,7 @@ private async ValueTask ConnectAsync(SocketsHttpConnectionContext contex endpoint, addresses, connectAsync: (ep, ct) => _socket.ConnectTcpAsync(ep, ct), - perAddressConnectTimeout: DefaultPerAddressConnectTimeout, + perAddressConnectTimeout: _socket.SuggestedHttpConnectTimeout, cancellationToken) .ConfigureAwait(false); } diff --git a/ZTSharp/ZeroTier/Internal/ManagedIpToNodeIdCache.cs b/ZTSharp/ZeroTier/Internal/ManagedIpToNodeIdCache.cs index 166a420..c152911 100644 --- a/ZTSharp/ZeroTier/Internal/ManagedIpToNodeIdCache.cs +++ b/ZTSharp/ZeroTier/Internal/ManagedIpToNodeIdCache.cs @@ -77,7 +77,6 @@ private void Set(IPAddress managedIp, NodeId nodeId, bool isAuthoritative) { if (IsExpired(existing)) { - EnqueueForEviction(managedIp); return new Entry(nodeId, expiresAt, isAuthoritative); } @@ -86,7 +85,6 @@ private void Set(IPAddress managedIp, NodeId nodeId, bool isAuthoritative) return existing; } - EnqueueForEviction(managedIp); return new Entry(nodeId, expiresAt, isAuthoritative); }); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs index d2d71c2..bf24a0c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneIpHandler.cs @@ -58,18 +58,20 @@ public async ValueTask HandleIpv6PacketAsync(NodeId peerNodeId, ReadOnlyMemory? _handleAuthenticatedPeer; + private readonly Action? _handleHelloOk; + private readonly Func? _handleUnknownDirectPathAsync; public ZeroTierDataplanePeerDatagramProcessor( NodeId localNodeId, @@ -26,7 +32,11 @@ public ZeroTierDataplanePeerDatagramProcessor( ZeroTierExternalSurfaceAddressTracker surfaceAddresses, ZeroTierPeerQosManager peerQos, ZeroTierPeerPathNegotiationManager peerNegotiation, - bool multipathEnabled) + ZeroTierInboundDatagramDiagnostics diagnostics, + bool multipathEnabled, + Action? handleAuthenticatedPeer = null, + Action? handleHelloOk = null, + Func? handleUnknownDirectPathAsync = null) { ArgumentNullException.ThrowIfNull(peerSecurity); ArgumentNullException.ThrowIfNull(peerPackets); @@ -35,6 +45,7 @@ public ZeroTierDataplanePeerDatagramProcessor( ArgumentNullException.ThrowIfNull(surfaceAddresses); ArgumentNullException.ThrowIfNull(peerQos); ArgumentNullException.ThrowIfNull(peerNegotiation); + ArgumentNullException.ThrowIfNull(diagnostics); _localNodeId = localNodeId; _peerSecurity = peerSecurity; @@ -44,7 +55,11 @@ public ZeroTierDataplanePeerDatagramProcessor( _surfaceAddresses = surfaceAddresses; _peerQos = peerQos; _peerNegotiation = peerNegotiation; + _diagnostics = diagnostics; _multipathEnabled = multipathEnabled; + _handleAuthenticatedPeer = handleAuthenticatedPeer; + _handleHelloOk = handleHelloOk; + _handleUnknownDirectPathAsync = handleUnknownDirectPathAsync; } public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken cancellationToken) @@ -66,23 +81,63 @@ public async Task ProcessAsync(ZeroTierUdpDatagram datagram, CancellationToken c if (decoded.Header.CipherSuite == 0 && decoded.Header.Verb == ZeroTierVerb.Hello) { - await _peerSecurity - .HandleHelloAsync(peerNodeId, decoded.Header.PacketId, packetBytes, datagram.RemoteEndPoint, cancellationToken) + var inboundHello = await _peerSecurity + .HandleHelloAsync( + peerNodeId, + datagram.LocalSocketId, + decoded.Header.PacketId, + packetBytes, + datagram.RemoteEndPoint, + cancellationToken) .ConfigureAwait(false); + + var observedNewInboundHop0Path = false; + if (inboundHello is { } hello && decoded.Header.HopCount == 0) + { + observedNewInboundHop0Path = !_peerPaths.ObserveKnownHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + + if (hello.ReportedLocalSurfaceAddress is { } reportedSurface) + { + _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, reportedSurface); + } + + if (ZeroTierTrace.Enabled) + { + var surfaceText = hello.ReportedLocalSurfaceAddress?.ToString() ?? ""; + ZeroTierTrace.WriteLine( + $"[zerotier] RX HELLO direct: peer={peerNodeId} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint} reportedLocalSurface={surfaceText}."); + } + } + + if (inboundHello is not null) + { + _handleAuthenticatedPeer?.Invoke(peerNodeId); + } + + if (observedNewInboundHop0Path && _handleUnknownDirectPathAsync is not null) + { + await _handleUnknownDirectPathAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) + .ConfigureAwait(false); + } + return; } if (!_peerSecurity.TryGetPeerKey(peerNodeId, out var key)) { + _diagnostics.TracePeerKeyMissing(peerNodeId, datagram, decoded.Header.HopCount); _peerSecurity.EnsurePeerKeyAsync(peerNodeId); return; } if (!ZeroTierPacketCrypto.Dearmor(packetBytes, key)) { + _diagnostics.TraceDearmorFailure(peerNodeId, datagram, decoded.Header.HopCount); return; } + _handleAuthenticatedPeer?.Invoke(peerNodeId); + if ((packetBytes[ZeroTierPacketHeader.IndexVerb] & ZeroTierPacketHeader.VerbFlagCompressed) != 0) { if (!ZeroTierPacketCompression.TryUncompress(packetBytes, out var uncompressed)) @@ -93,21 +148,50 @@ await _peerSecurity packetBytes = uncompressed; } + if (ZeroTierTrace.Enabled && _traceDecryptedRemaining > 0) + { + _traceDecryptedRemaining--; + var decryptedVerb = (ZeroTierVerb)(packetBytes[ZeroTierPacketHeader.IndexVerb] & 0x1F); + ZeroTierTrace.WriteLine( + $"[zerotier] RX peer decrypted: src={peerNodeId} hop={decoded.Header.HopCount} verb={decryptedVerb} socket={datagram.LocalSocketId} via {datagram.RemoteEndPoint}."); + } + + var observedNewHop0Path = false; + var hasKnownHop0Path = false; + if (_multipathEnabled) { + var verb = (ZeroTierVerb)(packetBytes[ZeroTierPacketHeader.IndexVerb] & 0x1F); + if (decoded.Header.HopCount == 0) { - _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + if (verb == ZeroTierVerb.Ok) + { + observedNewHop0Path = _peerPaths.ObserveHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + } + else + { + hasKnownHop0Path = _peerPaths.ObserveKnownHop0(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint); + } + await _peerEcho .TrySendEchoProbeAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, key, cancellationToken) .ConfigureAwait(false); } - var verb = (ZeroTierVerb)(packetBytes[ZeroTierPacketHeader.IndexVerb] & 0x1F); + if (decoded.Header.HopCount == 0 && + verb != ZeroTierVerb.Ok && + !hasKnownHop0Path && + _handleUnknownDirectPathAsync is not null) + { + await _handleUnknownDirectPathAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) + .ConfigureAwait(false); + } + var payload = packetBytes.AsMemory(ZeroTierPacketHeader.IndexPayload); var payloadSpan = payload.Span; - if (decoded.Header.HopCount == 0 && verb != ZeroTierVerb.QosMeasurement) + if (decoded.Header.HopCount == 0 && verb is not (ZeroTierVerb.QosMeasurement or ZeroTierVerb.Ack)) { _peerQos.RecordIncomingPacket(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, decoded.Header.PacketId); } @@ -154,7 +238,17 @@ await _peerEcho if (inReVerb == ZeroTierVerb.Echo) { var inRePacketId = BinaryPrimitives.ReadUInt64BigEndian(payloadSpan.Slice(1, 8)); - _peerEcho.HandleEchoOk(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, inRePacketId, payloadSpan.Slice(1 + 8)); + var matchedPending = _peerEcho.HandleEchoOk( + peerNodeId, + datagram.LocalSocketId, + datagram.RemoteEndPoint, + inRePacketId, + payloadSpan.Slice(1 + 8)); + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] RX OK(ECHO): peer={peerNodeId} via={datagram.RemoteEndPoint} hop={decoded.Header.HopCount} inRe=0x{inRePacketId:x16} localSocket={datagram.LocalSocketId} matched={matchedPending}."); + } return; } @@ -162,12 +256,29 @@ await _peerEcho { if (ZeroTierHelloOkParser.TryParseDecryptedOkHello(packetBytes, out var ok)) { - _peerSecurity.ObservePeerProtocolVersion(peerNodeId, ok.RemoteProtocolVersion); - _peerEcho.ObserveHelloOkRtt(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, ok.TimestampEcho); - - if (ok.ExternalSurfaceAddress is { } surface) + _peerSecurity.ObservePeerVersion( + peerNodeId, + ok.RemoteProtocolVersion, + ok.RemoteMajorVersion, + ok.RemoteMinorVersion, + ok.RemoteRevision); + if (_handleHelloOk is not null) { - _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, surface); + _handleHelloOk( + peerNodeId, + datagram.LocalSocketId, + datagram.RemoteEndPoint, + decoded.Header.HopCount, + ok); + } + else + { + _peerEcho.ObserveHelloOkRtt(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, ok.TimestampEcho); + + if (decoded.Header.HopCount == 0 && ok.ExternalSurfaceAddress is { } surface) + { + _surfaceAddresses.Observe(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, surface); + } } } @@ -176,6 +287,8 @@ await _peerEcho } } - await _peerPackets.HandleAsync(peerNodeId, packetBytes, cancellationToken).ConfigureAwait(false); + await _peerPackets + .HandleAsync(peerNodeId, datagram.LocalSocketId, datagram.RemoteEndPoint, packetBytes, cancellationToken) + .ConfigureAwait(false); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs index 5ea91f2..ca814d0 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerPacketHandler.cs @@ -9,13 +9,13 @@ internal sealed class ZeroTierDataplanePeerPacketHandler private readonly ulong _networkId; private readonly ZeroTierMac _localMac; private readonly ZeroTierDataplaneIpHandler _ip; - private readonly Func, CancellationToken, ValueTask>? _handleControlAsync; + private readonly Func, CancellationToken, ValueTask>? _handleControlAsync; public ZeroTierDataplanePeerPacketHandler( ulong networkId, ZeroTierMac localMac, ZeroTierDataplaneIpHandler ip, - Func, CancellationToken, ValueTask>? handleControlAsync = null) + Func, CancellationToken, ValueTask>? handleControlAsync = null) { ArgumentNullException.ThrowIfNull(ip); _networkId = networkId; @@ -24,7 +24,12 @@ public ZeroTierDataplanePeerPacketHandler( _handleControlAsync = handleControlAsync; } - public async ValueTask HandleAsync(NodeId peerNodeId, byte[] packetBytes, CancellationToken cancellationToken) + public async ValueTask HandleAsync( + NodeId peerNodeId, + int receivedLocalSocketId, + IPEndPoint receivedVia, + byte[] packetBytes, + CancellationToken cancellationToken) { if (packetBytes.Length <= ZeroTierPacketHeader.IndexVerb) { @@ -40,7 +45,7 @@ public async ValueTask HandleAsync(NodeId peerNodeId, byte[] packetBytes, Cancel { if (_handleControlAsync is not null) { - await _handleControlAsync(peerNodeId, verb, payload, cancellationToken).ConfigureAwait(false); + await _handleControlAsync(peerNodeId, verb, receivedLocalSocketId, receivedVia, payload, cancellationToken).ConfigureAwait(false); } return; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs index 4566238..9a73593 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplanePeerSecurity.cs @@ -8,6 +8,20 @@ namespace ZTSharp.ZeroTier.Internal; +internal readonly record struct ZeroTierInboundHelloPayload( + ulong Timestamp, + byte PeerProtocolVersion, + IPEndPoint? ReportedLocalSurfaceAddress); + +internal readonly record struct ZeroTierPeerVersion( + byte ProtocolVersion, + byte MajorVersion, + byte MinorVersion, + ushort Revision) +{ + public static readonly ZeroTierPeerVersion None; +} + internal sealed class ZeroTierDataplanePeerSecurity : IDisposable { private const int HelloPayloadMinLength = 13 + (5 + 1 + ZeroTierIdentity.PublicKeyLength + 1); @@ -22,7 +36,7 @@ internal sealed class ZeroTierDataplanePeerSecurity : IDisposable private readonly ConcurrentDictionary _peerKeys = new(); private readonly ConcurrentDictionary> _inflightKeys = new(); - private readonly ConcurrentDictionary _peerProtocolVersions = new(); + private readonly ConcurrentDictionary _peerVersions = new(); private readonly CancellationTokenSource _cts = new(); private int _trimPeerKeysInProgress; @@ -47,22 +61,43 @@ public ZeroTierDataplanePeerSecurity( } public byte GetPeerProtocolVersionOrDefault(NodeId peerNodeId) - => _peerProtocolVersions.TryGetValue(peerNodeId, out var version) ? version : (byte)0; + => GetPeerVersionOrDefault(peerNodeId).ProtocolVersion; - internal void ObservePeerProtocolVersion(NodeId peerNodeId, byte peerProtocolVersion) + public ZeroTierPeerVersion GetPeerVersionOrDefault(NodeId peerNodeId) + => _peerVersions.TryGetValue(peerNodeId, out var version) ? version : ZeroTierPeerVersion.None; + + internal void ObservePeerVersion(NodeId peerNodeId, byte peerProtocolVersion, byte peerMajorVersion, byte peerMinorVersion, ushort peerRevision) { peerProtocolVersion = peerProtocolVersion <= ZeroTierHelloClient.AdvertisedProtocolVersion ? peerProtocolVersion : ZeroTierHelloClient.AdvertisedProtocolVersion; - _peerProtocolVersions[peerNodeId] = peerProtocolVersion; + _peerVersions[peerNodeId] = new ZeroTierPeerVersion( + peerProtocolVersion, + peerMajorVersion, + peerMinorVersion, + peerRevision); + } + + internal void PrimePeerForTests( + NodeId peerNodeId, + byte[] sharedKey, + byte peerProtocolVersion, + byte peerMajorVersion, + byte peerMinorVersion, + ushort peerRevision) + { + ArgumentNullException.ThrowIfNull(sharedKey); + + CachePeerKey(peerNodeId, sharedKey.ToArray(), Environment.TickCount64); + ObservePeerVersion(peerNodeId, peerProtocolVersion, peerMajorVersion, peerMinorVersion, peerRevision); } public bool TryGetPeerKey(NodeId peerNodeId, out byte[] key) { key = Array.Empty(); - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; if (!TryGetPeerKeyEntry(peerNodeId, nowMs, out var entry)) { return false; @@ -79,7 +114,7 @@ public bool TryGetPeerKey(NodeId peerNodeId, out byte[] key) public void EnsurePeerKeyAsync(NodeId peerNodeId) { - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; if (TryGetPeerKeyEntry(peerNodeId, nowMs, out _)) { return; @@ -92,7 +127,7 @@ public async Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken c { cancellationToken.ThrowIfCancellationRequested(); - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; if (TryGetPeerKeyEntry(peerNodeId, nowMs, out var existing)) { if (existing.Key is not null) @@ -107,8 +142,9 @@ public async Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken c return await task.WaitAsync(cancellationToken).ConfigureAwait(false); } - public async ValueTask HandleHelloAsync( + public async ValueTask HandleHelloAsync( NodeId peerNodeId, + int localSocketId, ulong helloPacketId, byte[] packetBytes, IPEndPoint remoteEndPoint, @@ -118,21 +154,24 @@ public async ValueTask HandleHelloAsync( if (packetBytes.Length > ZeroTierProtocolLimits.MaxPacketBytes) { - return; + return null; } if (packetBytes.Length < ZeroTierPacketHeader.Length + HelloPayloadMinLength) { - return; + return null; } var payload = packetBytes.AsSpan(ZeroTierPacketHeader.IndexPayload); if (payload.Length < HelloPayloadMinLength) { - return; + return null; } var helloTimestamp = BinaryPrimitives.ReadUInt64BigEndian(payload.Slice(5, 8)); + var peerMajorVersion = payload[1]; + var peerMinorVersion = payload[2]; + var peerRevision = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(3, 2)); ZeroTierIdentity identity; try @@ -141,12 +180,20 @@ public async ValueTask HandleHelloAsync( } catch (FormatException) { - return; + return null; } if (identity.NodeId != peerNodeId) { - return; + return null; + } + + var identityLength = ZeroTierIdentityCodec.GetSerializedLength(identity, includePrivate: false); + IPEndPoint? reportedLocalSurface = null; + if (payload.Length > 13 + identityLength && + ZeroTierInetAddressCodec.TryDeserialize(payload.Slice(13 + identityLength), out var parsedSurface, out _)) + { + reportedLocalSurface = parsedSurface; } var sharedKey = new byte[48]; @@ -156,7 +203,7 @@ public async ValueTask HandleHelloAsync( } catch (CryptographicException) { - return; + return null; } if (!ZeroTierPacketCrypto.Dearmor(packetBytes, sharedKey)) @@ -166,17 +213,18 @@ public async ValueTask HandleHelloAsync( ZeroTierTrace.WriteLine($"[zerotier] Drop: failed to authenticate HELLO from {peerNodeId} via {remoteEndPoint}."); } - return; + return null; } if (!identity.LocallyValidate()) { - return; + return null; } - CachePeerKey(peerNodeId, sharedKey, nowMs: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + CachePeerKey(peerNodeId, sharedKey, nowMs: Environment.TickCount64); var peerProtocolVersion = payload[0]; - ObservePeerProtocolVersion(peerNodeId, peerProtocolVersion); + ObservePeerVersion(peerNodeId, peerProtocolVersion, peerMajorVersion, peerMinorVersion, peerRevision); + var reportedRemoteSurface = NormalizeExternalSurfaceForPeer(remoteEndPoint, peerProtocolVersion); var okPacket = ZeroTierHelloOkPacketBuilder.BuildPacket( packetId: ZeroTierPacketIdGenerator.GeneratePacketId(), @@ -184,16 +232,21 @@ public async ValueTask HandleHelloAsync( source: _localNodeId, inRePacketId: helloPacketId, helloTimestampEcho: helloTimestamp, - externalSurfaceAddress: remoteEndPoint, + externalSurfaceAddress: reportedRemoteSurface, sharedKey: ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, peerProtocolVersion)); try { - await _udp.SendAsync(remoteEndPoint, okPacket, cancellationToken).ConfigureAwait(false); + await _udp.SendAsync(localSocketId, remoteEndPoint, okPacket, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is SocketException or ObjectDisposedException) { } + + return new ZeroTierInboundHelloPayload( + Timestamp: helloTimestamp, + PeerProtocolVersion: peerProtocolVersion, + ReportedLocalSurfaceAddress: reportedLocalSurface); } public void Dispose() @@ -202,6 +255,23 @@ public void Dispose() _cts.Dispose(); } + internal NodeId[] GetTrustedPeersSnapshot() + { + var nowMs = Environment.TickCount64; + var peers = new List(); + foreach (var (peerNodeId, entry) in _peerKeys) + { + if (entry.Key is null || entry.ExpiresAtUnixMs <= nowMs) + { + continue; + } + + peers.Add(peerNodeId); + } + + return peers.ToArray(); + } + private Task StartPeerKeyFetch(NodeId peerNodeId) { var task = FetchAndCachePeerKeyAsync(peerNodeId); @@ -211,7 +281,7 @@ private Task StartPeerKeyFetch(NodeId peerNodeId) private async Task FetchAndCachePeerKeyAsync(NodeId peerNodeId) { - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nowMs = Environment.TickCount64; try { var identity = await _rootClient @@ -323,6 +393,7 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) if (entry.ExpiresAtUnixMs <= nowMs) { _peerKeys.TryRemove(peerNodeId, out _); + _peerVersions.TryRemove(peerNodeId, out _); } } @@ -341,6 +412,7 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) } _peerKeys.TryRemove(peerNodeId, out _); + _peerVersions.TryRemove(peerNodeId, out _); toRemove--; } } @@ -350,5 +422,15 @@ private void TrimPeerKeyCacheIfNeeded(long nowMs) } } + private static IPEndPoint NormalizeExternalSurfaceForPeer(IPEndPoint remoteEndPoint, byte peerProtocolVersion) + { + if (peerProtocolVersion >= 5 || remoteEndPoint.Port == 0) + { + return remoteEndPoint; + } + + return new IPEndPoint(remoteEndPoint.Address, 0); + } + private readonly record struct PeerKeyCacheEntry(byte[]? Key, long ExpiresAtUnixMs); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRouteRegistry.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRouteRegistry.cs index 5a71cc0..70250e5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRouteRegistry.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRouteRegistry.cs @@ -249,6 +249,7 @@ public bool TryRegisterTcpListener( Func, CancellationToken, Task> onSyn) { ArgumentNullException.ThrowIfNull(localAddress); + ArgumentNullException.ThrowIfNull(onSyn); return localAddress.AddressFamily switch { @@ -330,6 +331,21 @@ public bool TryGetTcpSynHandler( ushort localPort, out Func, CancellationToken, Task> handler) { + ArgumentNullException.ThrowIfNull(destinationIp); + if (addressFamily == AddressFamily.InterNetwork && + destinationIp.AddressFamily == AddressFamily.InterNetworkV6 && + destinationIp.IsIPv4MappedToIPv6) + { + destinationIp = destinationIp.MapToIPv4(); + } + + if (destinationIp.AddressFamily != addressFamily) + { + throw new ArgumentException( + "Destination IP address family must match the provided address family.", + nameof(destinationIp)); + } + if (addressFamily == AddressFamily.InterNetwork) { if (_tcpListenersV4.TryGetValue(localPort, out var registrations) && registrations.TryGet(destinationIp, out var existing)) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs index 21f6071..70f8172 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntime.cs @@ -1,68 +1,106 @@ using System.Buffers.Binary; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; -using System.Net; -using System.Net.Sockets; -using System.Threading.Channels; -using ZTSharp.Transport.Internal; -using ZTSharp.ZeroTier.Net; -using ZTSharp.ZeroTier.Protocol; -using ZTSharp.ZeroTier.Transport; - -namespace ZTSharp.ZeroTier.Internal; - +using System.Net; +using System.Net.Sockets; +using System.Threading.Channels; +using ZTSharp.Transport.Internal; +using ZTSharp.ZeroTier.Net; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + internal sealed class ZeroTierDataplaneRuntime : IAsyncDisposable { + private const long DirectEndpointManagerTtlMs = 600_000; + private const long NetworkCredentialsBootstrapIntervalMs = 5_000; + private const long DirectHelloMinIntervalMs = 1_000; + private const long DirectBootstrapRootHelloIntervalMs = 5_000; + private const long DirectBootstrapPathPushIntervalMs = 15_000; + private const long DirectBootstrapPathPushIntervalHavePathMs = 120_000; + private const long DirectBootstrapHintProbeIntervalMs = 1_000; + private const int DirectBootstrapHintProbeBudget = 2; + private const int DirectHintMaintenanceProbeBudget = 2; + private const long DirectHintFullHelloIntervalMs = 60_000; + private const long DirectOnlyHintHelloIntervalMs = 5_000; + private const long DirectOnlyRendezvousGraceMs = 3_000; + private const long DirectOnlyPinnedRendezvousSettleMs = 1_000; + private const long DirectOnlyPinnedRendezvousVersionGraceMs = 750; + private const long DirectOnlyPinnedRendezvousRelayQuietMs = 2_000; + private readonly IZeroTierUdpTransport _udp; - private readonly NodeId _rootNodeId; - private readonly IPEndPoint _rootEndpoint; - private readonly byte[] _rootKey; - private readonly byte _rootProtocolVersion; - private readonly ZeroTierIdentity _localIdentity; - private readonly ulong _networkId; + private readonly NodeId _rootNodeId; + private readonly IPEndPoint _rootEndpoint; + private readonly byte[] _rootKey; + private readonly byte _rootProtocolVersion; + private readonly ZeroTierIdentity _localIdentity; + private readonly ulong _networkId; private readonly byte[] _inlineCom; + private readonly ulong _planetId; + private readonly ulong _planetTimestamp; private readonly IPAddress[] _localManagedIpsV4; private readonly byte[][] _localManagedIpsV4Bytes; private readonly IPAddress[] _localManagedIpsV6; + private readonly IPEndPoint[] _localDirectPathAdvertisements; private readonly ZeroTierMac _localMac; private readonly ConcurrentDictionary _directEndpoints = new(); - private readonly ZeroTierPeerPhysicalPathTracker _peerPaths; - private readonly ZeroTierPeerEchoManager _peerEcho; + private readonly ConcurrentDictionary _directEndpointLastUsedMs = new(); + private long _lastDirectEndpointCleanupMs; + private readonly ZeroTierPeerPhysicalPathTracker _peerPaths; + private readonly ZeroTierPeerEchoManager _peerEcho; private readonly ZeroTierExternalSurfaceAddressTracker _surfaceAddresses; private readonly ZeroTierPeerQosManager _peerQos; private readonly ZeroTierPeerPathNegotiationManager _peerNegotiation; private readonly ZeroTierPeerBondPolicyEngine _bondEngine; + private readonly ZeroTierPeerRootSocketAffinity _peerRootSocketAffinity; + private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; + private readonly ZeroTierStickyHintMaintenanceSelector _stickyHintMaintenanceSelector; + private readonly ZeroTierDirectEndpointPolicy _directEndpointPolicy; + private readonly ZeroTierDirectPathSocketAdmissibility _directPathSocketAdmissibility; + private readonly ZeroTierLocalDirectPathAdvertisementPlanner _localDirectAdvertisementPlanner; + private readonly ZeroTierLocalDirectPathAdvertisementSource _localDirectPathAdvertisementsSource; private readonly ZeroTierMultipathOptions _multipath; - + private readonly ConcurrentDictionary _authenticatedPeers = new(); + private readonly ConcurrentDictionary _directBootstrapTasks = new(); + private readonly ConcurrentDictionary _directBootstrapStartedMs = new(); + private readonly ConcurrentDictionary _pinnedRendezvousObservedMs = new(); + private readonly ConcurrentDictionary _lastNetworkCredentialsBootstrapMs = new(); + private readonly ConcurrentDictionary _lastDirectPathPushSentMs = new(); + private readonly ConcurrentDictionary<(NodeId PeerNodeId, int LocalSocketId, string Endpoint), long> _lastDirectHelloSentMs = new(); + private readonly ConcurrentDictionary _pendingHelloProbes = new(); + private long _lastPendingHelloCleanupMs; + private readonly Channel _peerQueue = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) { FullMode = BoundedChannelFullMode.Wait, - SingleReader = true, + SingleReader = false, SingleWriter = true }); - private long _peerQueueDropCount; - private readonly CancellationTokenSource _cts = new(); - private readonly Task _dispatcherLoop; - private readonly Task _peerLoop; - private readonly Task? _multipathMaintenanceLoop; - - private readonly ZeroTierDataplaneRootClient _rootClient; - private readonly ZeroTierDataplanePeerSecurity _peerSecurity; - private readonly ManagedIpToNodeIdCache _managedIpToNodeId = new(); - private readonly ZeroTierDataplaneRouteRegistry _routes; - private readonly ZeroTierDataplanePeerPacketHandler _peerPackets; - private readonly ZeroTierDataplanePeerDatagramProcessor _peerDatagrams; - private readonly ZeroTierDataplaneRxLoops _rxLoops; - - private bool _disposed; - - public ZeroTierDataplaneRuntime( - IZeroTierUdpTransport udp, - NodeId rootNodeId, - IPEndPoint rootEndpoint, - byte[] rootKey, - byte rootProtocolVersion, - ZeroTierIdentity localIdentity, + private long _peerQueueDropCount; + private readonly CancellationTokenSource _cts = new(); + private readonly Task _dispatcherLoop; + private readonly Task _peerLoop; + private readonly Task? _multipathMaintenanceLoop; + + private readonly ZeroTierDataplaneRootClient _rootClient; + private readonly ZeroTierDataplanePeerSecurity _peerSecurity; + private readonly ManagedIpToNodeIdCache _managedIpToNodeId = new(); + private readonly ZeroTierDataplaneRouteRegistry _routes; + private readonly ZeroTierDataplanePeerPacketHandler _peerPackets; + private readonly ZeroTierDataplanePeerDatagramProcessor _peerDatagrams; + private readonly ZeroTierDataplaneRxLoops _rxLoops; + + private bool _disposed; + + public ZeroTierDataplaneRuntime( + IZeroTierUdpTransport udp, + NodeId rootNodeId, + IPEndPoint rootEndpoint, + byte[] rootKey, + byte rootProtocolVersion, + ZeroTierIdentity localIdentity, ulong networkId, IReadOnlyList localManagedIpsV4, IReadOnlyList localManagedIpsV6, @@ -78,102 +116,126 @@ public ZeroTierDataplaneRuntime( localManagedIpsV4, localManagedIpsV6, inlineCom, - multipath: new ZeroTierMultipathOptions()) + multipath: new ZeroTierMultipathOptions(), + planetId: 0, + planetTimestamp: 0, + localExternalSurfaceAddress: null) { } - - public ZeroTierDataplaneRuntime( - IZeroTierUdpTransport udp, - NodeId rootNodeId, - IPEndPoint rootEndpoint, - byte[] rootKey, - byte rootProtocolVersion, - ZeroTierIdentity localIdentity, + + public ZeroTierDataplaneRuntime( + IZeroTierUdpTransport udp, + NodeId rootNodeId, + IPEndPoint rootEndpoint, + byte[] rootKey, + byte rootProtocolVersion, + ZeroTierIdentity localIdentity, ulong networkId, IReadOnlyList localManagedIpsV4, IReadOnlyList localManagedIpsV6, byte[] inlineCom, - ZeroTierMultipathOptions multipath) + ZeroTierMultipathOptions multipath, + ulong planetId = 0, + ulong planetTimestamp = 0, + IPEndPoint? localExternalSurfaceAddress = null, + IReadOnlyList? initialExternalSurfaceObservations = null) { - ArgumentNullException.ThrowIfNull(udp); - ArgumentNullException.ThrowIfNull(rootEndpoint); - ArgumentNullException.ThrowIfNull(rootKey); - ArgumentNullException.ThrowIfNull(localIdentity); - ArgumentNullException.ThrowIfNull(localManagedIpsV6); - ArgumentNullException.ThrowIfNull(inlineCom); - ArgumentNullException.ThrowIfNull(multipath); - - // Some local test harnesses pass wildcard-bound socket endpoints (0.0.0.0/::) as a "reachable" root endpoint. - // Normalize those to loopback to keep the dataplane's remote root endpoint concrete. - rootEndpoint = UdpEndpointNormalization.NormalizeForAdvertisement(rootEndpoint); - - if (localManagedIpsV4.Count == 0 && localManagedIpsV6.Count == 0) - { - throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "At least one managed IP (IPv4 or IPv6) is required."); - } - - for (var i = 0; i < localManagedIpsV4.Count; i++) - { - if (localManagedIpsV4[i].AddressFamily != AddressFamily.InterNetwork) - { - throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "All IPv4 managed IPs must be IPv4."); - } - } - - for (var i = 0; i < localManagedIpsV6.Count; i++) - { - if (localManagedIpsV6[i].AddressFamily != AddressFamily.InterNetworkV6) - { - throw new ArgumentOutOfRangeException(nameof(localManagedIpsV6), "All IPv6 managed IPs must be IPv6."); - } - } - - _udp = udp; - _rootNodeId = rootNodeId; - _rootEndpoint = rootEndpoint; - _rootKey = rootKey; - _rootProtocolVersion = rootProtocolVersion; - _localIdentity = localIdentity; + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(rootEndpoint); + ArgumentNullException.ThrowIfNull(rootKey); + ArgumentNullException.ThrowIfNull(localIdentity); + ArgumentNullException.ThrowIfNull(localManagedIpsV6); + ArgumentNullException.ThrowIfNull(inlineCom); + ArgumentNullException.ThrowIfNull(multipath); + + // Some local test harnesses pass wildcard-bound socket endpoints (0.0.0.0/::) as a "reachable" root endpoint. + // Normalize those to loopback to keep the dataplane's remote root endpoint concrete. + rootEndpoint = UdpEndpointNormalization.NormalizeForAdvertisement(rootEndpoint); + + if (localManagedIpsV4.Count == 0 && localManagedIpsV6.Count == 0) + { + throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "At least one managed IP (IPv4 or IPv6) is required."); + } + + for (var i = 0; i < localManagedIpsV4.Count; i++) + { + if (localManagedIpsV4[i].AddressFamily != AddressFamily.InterNetwork) + { + throw new ArgumentOutOfRangeException(nameof(localManagedIpsV4), "All IPv4 managed IPs must be IPv4."); + } + } + + for (var i = 0; i < localManagedIpsV6.Count; i++) + { + if (localManagedIpsV6[i].AddressFamily != AddressFamily.InterNetworkV6) + { + throw new ArgumentOutOfRangeException(nameof(localManagedIpsV6), "All IPv6 managed IPs must be IPv6."); + } + } + + _udp = udp; + _rootNodeId = rootNodeId; + _rootEndpoint = rootEndpoint; + _rootKey = rootKey; + _rootProtocolVersion = rootProtocolVersion; + _localIdentity = localIdentity; _networkId = networkId; _inlineCom = inlineCom; + _planetId = planetId; + _planetTimestamp = planetTimestamp; _multipath = multipath; - _localManagedIpsV4 = localManagedIpsV4.Count == 0 ? Array.Empty() : localManagedIpsV4.ToArray(); + _localManagedIpsV4 = localManagedIpsV4.Count == 0 ? Array.Empty() : localManagedIpsV4.ToArray(); _localManagedIpsV4Bytes = _localManagedIpsV4.Length == 0 ? Array.Empty() : _localManagedIpsV4.Select(ip => ip.GetAddressBytes()).ToArray(); _localManagedIpsV6 = localManagedIpsV6.Count == 0 ? Array.Empty() : localManagedIpsV6.ToArray(); + _localDirectPathAdvertisements = BuildLocalDirectPathAdvertisements(udp, localExternalSurfaceAddress); _localMac = ZeroTierMac.FromAddress(localIdentity.NodeId, networkId); - - _routes = new ZeroTierDataplaneRouteRegistry(this); - _rootClient = new ZeroTierDataplaneRootClient( - udp, - rootNodeId, - rootEndpoint, - rootKey, - rootProtocolVersion, - localIdentity.NodeId, - networkId, - inlineCom); + + _routes = new ZeroTierDataplaneRouteRegistry(this); + _rootClient = new ZeroTierDataplaneRootClient( + udp, + rootNodeId, + rootEndpoint, + rootKey, + rootProtocolVersion, + localIdentity.NodeId, + networkId, + inlineCom); _peerSecurity = new ZeroTierDataplanePeerSecurity(udp, _rootClient, localIdentity); _peerPaths = new ZeroTierPeerPhysicalPathTracker(ttl: TimeSpan.FromSeconds(30)); _peerEcho = new ZeroTierPeerEchoManager(udp, localIdentity.NodeId, _peerSecurity.GetPeerProtocolVersionOrDefault); _surfaceAddresses = new ZeroTierExternalSurfaceAddressTracker(ttl: TimeSpan.FromMinutes(30)); + _localDirectPathAdvertisementsSource = new ZeroTierLocalDirectPathAdvertisementSource(); + SeedInitialExternalSurfaceObservations(initialExternalSurfaceObservations); _peerQos = new ZeroTierPeerQosManager(); _peerNegotiation = new ZeroTierPeerPathNegotiationManager(); _bondEngine = new ZeroTierPeerBondPolicyEngine(GetPathLatencyMsOrNull, GetRemoteUtilityOrZero); + _peerRootSocketAffinity = new ZeroTierPeerRootSocketAffinity(rootEndpoint); + _directEndpointPolicy = new ZeroTierDirectEndpointPolicy(); + _directPathSocketAdmissibility = new ZeroTierDirectPathSocketAdmissibility( + _directEndpointPolicy, + new ZeroTierDirectRouteResolver()); + _localDirectAdvertisementPlanner = new ZeroTierLocalDirectPathAdvertisementPlanner(_directEndpointPolicy); + _directHintPlanner = new ZeroTierDirectHintPathPlanner( + udp, + GetOrCreateDirectEndpointManager, + _directPathSocketAdmissibility); + _stickyHintMaintenanceSelector = new ZeroTierStickyHintMaintenanceSelector(_directHintPlanner); + var inboundDiagnostics = new ZeroTierInboundDatagramDiagnostics(localIdentity.NodeId, rootEndpoint); var icmpv6 = new ZeroTierDataplaneIcmpv6Handler(this, _localMac, _localManagedIpsV6, _managedIpToNodeId); - var ip = new ZeroTierDataplaneIpHandler( - sender: this, - routes: _routes, - managedIpToNodeId: _managedIpToNodeId, - icmpv6: icmpv6, - networkId: _networkId, - localMac: _localMac, - localManagedIpsV4: _localManagedIpsV4, - localManagedIpsV4Bytes: _localManagedIpsV4Bytes, - localManagedIpsV6: _localManagedIpsV6); - _peerPackets = new ZeroTierDataplanePeerPacketHandler(_networkId, _localMac, ip, handleControlAsync: HandlePeerControlPacketAsync); + var ip = new ZeroTierDataplaneIpHandler( + sender: this, + routes: _routes, + managedIpToNodeId: _managedIpToNodeId, + icmpv6: icmpv6, + networkId: _networkId, + localMac: _localMac, + localManagedIpsV4: _localManagedIpsV4, + localManagedIpsV4Bytes: _localManagedIpsV4Bytes, + localManagedIpsV6: _localManagedIpsV6); + _peerPackets = new ZeroTierDataplanePeerPacketHandler(_networkId, _localMac, ip, handleControlAsync: HandlePeerControlPacketAsync); _peerDatagrams = new ZeroTierDataplanePeerDatagramProcessor( localIdentity.NodeId, _peerSecurity, @@ -183,7 +245,11 @@ public ZeroTierDataplaneRuntime( _surfaceAddresses, _peerQos, _peerNegotiation, - multipath.Enabled); + inboundDiagnostics, + multipath.Enabled, + HandleAuthenticatedPeer, + HandlePeerHelloOk, + HandleUnknownDirectPathAsync); _rxLoops = new ZeroTierDataplaneRxLoops( _udp, _rootNodeId, @@ -192,137 +258,248 @@ public ZeroTierDataplaneRuntime( _localIdentity.NodeId, _rootClient, _peerDatagrams, + inboundDiagnostics, + acceptDirectPeerDatagrams: multipath.Enabled, handleRootControlAsync: HandleRootControlPacketAsync, onPeerQueueDrop: () => { Interlocked.Increment(ref _peerQueueDropCount); if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine("[zerotier] Drop: peer queue is full."); - } - }); - - _dispatcherLoop = Task.Run(() => _rxLoops.DispatcherLoopAsync(_peerQueue.Writer, _cts.Token), CancellationToken.None); + { + ZeroTierTrace.WriteLine("[zerotier] Drop: peer queue is full."); + } + }); + + _dispatcherLoop = Task.Run(() => _rxLoops.DispatcherLoopAsync(_peerQueue, _cts.Token), CancellationToken.None); _peerLoop = Task.Run(() => _rxLoops.PeerLoopAsync(_peerQueue.Reader, _cts.Token), CancellationToken.None); _multipathMaintenanceLoop = multipath.Enabled ? Task.Run(() => MultipathMaintenanceLoopAsync(_cts.Token), CancellationToken.None) : null; } - public NodeId NodeId => _localIdentity.NodeId; - - public IPEndPoint LocalUdp => _udp.LocalSockets[0].LocalEndpoint; - - public long PeerQueueDropCount => Interlocked.Read(ref _peerQueueDropCount); - - public IZeroTierRoutedIpLink RegisterTcpRoute(NodeId peerNodeId, IPEndPoint localEndpoint, IPEndPoint remoteEndpoint) + private static IPEndPoint[] BuildLocalDirectPathAdvertisements( + IZeroTierUdpTransport udp, + IPEndPoint? localExternalSurfaceAddress) { - ObjectDisposedException.ThrowIf(_disposed, this); - return _routes.RegisterTcpRoute(peerNodeId, localEndpoint, remoteEndpoint); - } - - public void UnregisterRoute(ZeroTierTcpRouteKey routeKey) - => _routes.UnregisterRoute(routeKey); - - public void UnregisterRoute(ZeroTierTcpRouteKeyV6 routeKey) - => _routes.UnregisterRoute(routeKey); - - public bool TryRegisterTcpListener( - IPAddress localAddress, - ushort localPort, - Func, CancellationToken, Task> onSyn) - => _routes.TryRegisterTcpListener(localAddress, localPort, onSyn); - - public void UnregisterTcpListener(IPAddress localAddress, ushort localPort) - => _routes.UnregisterTcpListener(localAddress, localPort); + var endpoints = new List(); + if (localExternalSurfaceAddress is not null && localExternalSurfaceAddress.Port != 0) + { + endpoints.Add(UdpEndpointNormalization.Normalize(localExternalSurfaceAddress)); + } - public bool TryRegisterUdpPort(AddressFamily addressFamily, ushort localPort, ChannelWriter handler) - => _routes.TryRegisterUdpPort(addressFamily, localPort, handler); + var localSockets = udp.LocalSockets; + for (var i = 0; i < localSockets.Count; i++) + { + var endpoint = UdpEndpointNormalization.Normalize(localSockets[i].LocalEndpoint); + if (endpoint.Port == 0 || + endpoint.Address.Equals(IPAddress.Any) || + endpoint.Address.Equals(IPAddress.IPv6Any) || + IPAddress.IsLoopback(endpoint.Address)) + { + continue; + } - public void UnregisterUdpPort(AddressFamily addressFamily, ushort localPort) - => _routes.UnregisterUdpPort(addressFamily, localPort); + endpoints.Add(endpoint); + } - public async Task ResolveNodeIdAsync(IPAddress managedIp, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(managedIp); - cancellationToken.ThrowIfCancellationRequested(); - ObjectDisposedException.ThrowIf(_disposed, this); - return await _rootClient.ResolveNodeIdAsync(managedIp, _managedIpToNodeId, cancellationToken).ConfigureAwait(false); + return endpoints + .Distinct() + .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) + .ToArray(); } - public async ValueTask SendIpv4Async(NodeId peerNodeId, ReadOnlyMemory ipv4Packet, CancellationToken cancellationToken) + private IPEndPoint[] GetLocalDirectPathAdvertisements() { - cancellationToken.ThrowIfCancellationRequested(); - ObjectDisposedException.ThrowIf(_disposed, this); - - var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); - var packet = ZeroTierExtFramePacketBuilder.BuildPacket( - packetId, - destination: peerNodeId, - source: _localIdentity.NodeId, - networkId: _networkId, - inlineCom: _inlineCom, - to: remoteMac, - from: _localMac, - etherType: ZeroTierFrameCodec.EtherTypeIpv4, - frame: ipv4Packet.Span, - sharedKey: key, - remoteProtocolVersion: peerProtocolVersion); + var endpoints = new List(_localDirectPathAdvertisements); + var localSockets = _udp.LocalSockets; + endpoints.AddRange(_localDirectPathAdvertisementsSource.GetSnapshot(localSockets)); + if (localSockets.Count == 0) + { + endpoints.AddRange(_surfaceAddresses.GetSnapshot(localSocketId: 0)); + } + else + { + for (var i = 0; i < localSockets.Count; i++) + { + endpoints.AddRange(_surfaceAddresses.GetSnapshot(localSockets[i].Id)); + } + } - var flowId = _multipath.Enabled ? ZeroTierFlowId.Derive(ipv4Packet.Span) : 0; - await SendToPeerAsync(peerNodeId, packet, flowId, cancellationToken).ConfigureAwait(false); + var expanded = ZeroTierPredictedPublicSurfaceGenerator.Expand(endpoints); + return ZeroTierDirectEndpointSelection.Normalize( + expanded.Where(static endpoint => endpoint.Port != 0).Select(UdpEndpointNormalization.Normalize), + _rootEndpoint, + maxEndpoints: ZeroTierProtocolLimits.MaxPushedDirectPaths); } - public async ValueTask SendEthernetFrameAsync( - NodeId peerNodeId, - ushort etherType, - ReadOnlyMemory frame, - CancellationToken cancellationToken) + private void SeedInitialExternalSurfaceObservations(IReadOnlyList? observations) { - cancellationToken.ThrowIfCancellationRequested(); - ObjectDisposedException.ThrowIf(_disposed, this); - - var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); - var packet = ZeroTierExtFramePacketBuilder.BuildPacket( - packetId, - destination: peerNodeId, - source: _localIdentity.NodeId, - networkId: _networkId, - inlineCom: _inlineCom, - to: remoteMac, - from: _localMac, - etherType: etherType, - frame: frame.Span, - sharedKey: key, - remoteProtocolVersion: peerProtocolVersion); - - var flowId = (_multipath.Enabled && (etherType == ZeroTierFrameCodec.EtherTypeIpv4 || etherType == ZeroTierFrameCodec.EtherTypeIpv6)) - ? ZeroTierFlowId.Derive(frame.Span) - : 0; - - await SendToPeerAsync(peerNodeId, packet, flowId, cancellationToken).ConfigureAwait(false); - } + if (observations is null || observations.Count == 0) + { + return; + } + + for (var i = 0; i < observations.Count; i++) + { + var observation = observations[i]; + if (observation.SurfaceAddress.Port == 0) + { + continue; + } + + _surfaceAddresses.Observe( + _rootNodeId, + observation.LocalSocketId, + _rootEndpoint, + UdpEndpointNormalization.Normalize(observation.SurfaceAddress)); + } + if (ZeroTierTrace.Enabled) + { + var advertised = GetLocalDirectPathAdvertisements(); + ZeroTierTrace.WriteLine( + $"[zerotier] Seed local direct advertisements from root: {ZeroTierDirectEndpointSelection.Format(advertised)}."); + } + } + + public NodeId NodeId => _localIdentity.NodeId; + + public IPEndPoint LocalUdp => _udp.LocalSockets[0].LocalEndpoint; + + public long PeerQueueDropCount => Interlocked.Read(ref _peerQueueDropCount); + + public IZeroTierRoutedIpLink RegisterTcpRoute(NodeId peerNodeId, IPEndPoint localEndpoint, IPEndPoint remoteEndpoint) + { + ObjectDisposedException.ThrowIf(_disposed, this); + return _routes.RegisterTcpRoute(peerNodeId, localEndpoint, remoteEndpoint); + } + + public void UnregisterRoute(ZeroTierTcpRouteKey routeKey) + => _routes.UnregisterRoute(routeKey); + + public void UnregisterRoute(ZeroTierTcpRouteKeyV6 routeKey) + => _routes.UnregisterRoute(routeKey); + + public bool TryRegisterTcpListener( + IPAddress localAddress, + ushort localPort, + Func, CancellationToken, Task> onSyn) + => _routes.TryRegisterTcpListener(localAddress, localPort, onSyn); + + public void UnregisterTcpListener(IPAddress localAddress, ushort localPort) + => _routes.UnregisterTcpListener(localAddress, localPort); + + public bool TryRegisterUdpPort(AddressFamily addressFamily, ushort localPort, ChannelWriter handler) + => _routes.TryRegisterUdpPort(addressFamily, localPort, handler); + + public void UnregisterUdpPort(AddressFamily addressFamily, ushort localPort) + => _routes.UnregisterUdpPort(addressFamily, localPort); + + public async Task ResolveNodeIdAsync(IPAddress managedIp, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(managedIp); + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); + return await _rootClient.ResolveNodeIdAsync(managedIp, _managedIpToNodeId, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask SendIpv4Async(NodeId peerNodeId, ReadOnlyMemory ipv4Packet, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); + + var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var packet = ZeroTierExtFramePacketBuilder.BuildPacket( + packetId, + destination: peerNodeId, + source: _localIdentity.NodeId, + networkId: _networkId, + inlineCom: _inlineCom, + to: remoteMac, + from: _localMac, + etherType: ZeroTierFrameCodec.EtherTypeIpv4, + frame: ipv4Packet.Span, + sharedKey: key, + remoteProtocolVersion: peerProtocolVersion); + + var flowId = _multipath.Enabled ? ZeroTierFlowId.Derive(ipv4Packet.Span) : 0; + var preferHintedDirectFanout = ZeroTierTcpHandshakeClassifier.IsInitialSyn(ipv4Packet.Span); + await SendToPeerAsync(peerNodeId, packet, flowId, preferHintedDirectFanout, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask SendEthernetFrameAsync( + NodeId peerNodeId, + ushort etherType, + ReadOnlyMemory frame, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); + + var key = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var remoteMac = ZeroTierMac.FromAddress(peerNodeId, _networkId); + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var packet = ZeroTierExtFramePacketBuilder.BuildPacket( + packetId, + destination: peerNodeId, + source: _localIdentity.NodeId, + networkId: _networkId, + inlineCom: _inlineCom, + to: remoteMac, + from: _localMac, + etherType: etherType, + frame: frame.Span, + sharedKey: key, + remoteProtocolVersion: peerProtocolVersion); + + var flowId = (_multipath.Enabled && (etherType == ZeroTierFrameCodec.EtherTypeIpv4 || etherType == ZeroTierFrameCodec.EtherTypeIpv6)) + ? ZeroTierFlowId.Derive(frame.Span) + : 0; + + var preferHintedDirectFanout = + (etherType == ZeroTierFrameCodec.EtherTypeIpv4 || etherType == ZeroTierFrameCodec.EtherTypeIpv6) && + ZeroTierTcpHandshakeClassifier.IsInitialSyn(frame.Span); + await SendToPeerAsync(peerNodeId, packet, flowId, preferHintedDirectFanout, cancellationToken).ConfigureAwait(false); + } + private async ValueTask SendToPeerAsync( NodeId peerNodeId, ReadOnlyMemory packet, uint flowId, + bool preferHintedDirectFanout, CancellationToken cancellationToken) { - cancellationToken.ThrowIfCancellationRequested(); - + cancellationToken.ThrowIfCancellationRequested(); + if (!_multipath.Enabled) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "multipath direct paths are disabled"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; } + var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); + var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; + + if (preferHintedDirectFanout && + !_multipath.AllowRootRelayFallback && + !HasConfirmedDirectPath(peerNodeId)) + { + await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + } + else if (preferHintedDirectFanout && + !HasConfirmedDirectPath(peerNodeId) && + await TrySendDirectOnlyHintedPayloadAsync(peerNodeId, packet, flowId, parsedOk ? parsed.PacketId : null, shouldRecord, cancellationToken).ConfigureAwait(false)) + { + return; + } + + await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + if (_multipath.BondPolicy == ZeroTierBondPolicy.Broadcast) { await SendToPeerBroadcastAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); @@ -331,40 +508,52 @@ private async ValueTask SendToPeerAsync( if (!TrySelectDirectPath(peerNodeId, flowId, out var direct)) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "no direct peer path is available"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; } var confirmed = _peerEcho.TryGetLastRttMs(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, out _); - if (_multipath.WarmupDuplicateToRoot && !confirmed) + if (_multipath.WarmupDuplicateToRoot && _multipath.AllowRootRelayFallback && !confirmed) { - var (packetId, verb) = TryGetPacketIdAndVerb(packet, out var parsed) ? parsed : default; - if (verb != ZeroTierVerb.QosMeasurement) + if (shouldRecord) { - _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId); + _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); } + Task directSend = Task.CompletedTask; + Task rootSend = Task.CompletedTask; try { - await Task - .WhenAll( - _udp.SendAsync(direct.LocalSocketId, direct.RemoteEndPoint, packet, cancellationToken), - _udp.SendAsync(_rootEndpoint, packet, cancellationToken)) - .ConfigureAwait(false); + directSend = _udp.SendAsync(direct.LocalSocketId, direct.RemoteEndPoint, packet, cancellationToken); + rootSend = _udp.SendAsync(_rootEndpoint, packet, cancellationToken); + await Task.WhenAll(directSend, rootSend).ConfigureAwait(false); } - catch (SocketException) + catch (Exception ex) when (ex is SocketException || (ex is AggregateException ae && ae.InnerExceptions.All(static inner => inner is SocketException))) { - _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId); - await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + // Warm-up duplicates to root. If the direct send fails, forget any QoS state for it and + // ensure a root send happens (if it hasn't already). + // If only the root send fails, keep QoS state intact since the direct send may still succeed. + var directOk = directSend.IsCompletedSuccessfully; + var rootOk = rootSend.IsCompletedSuccessfully; + + if (!directOk && shouldRecord) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); + } + + if (!directOk && !rootOk) + { + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } } return; } - var (packetId2, verb2) = TryGetPacketIdAndVerb(packet, out var parsed2) ? parsed2 : default; - if (verb2 != ZeroTierVerb.QosMeasurement) + if (shouldRecord) { - _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId2); + _peerQos.RecordOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); } try @@ -373,53 +562,61 @@ await Task } catch (SocketException) { - _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, packetId2); + if (shouldRecord) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, direct.LocalSocketId, direct.RemoteEndPoint, parsed.PacketId); + } + + EnsureRelayAllowedForPayload(peerNodeId, reason: $"direct send to {direct.RemoteEndPoint} failed"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); } } - + private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemory packet, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + await EnsureDirectPathAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + var observed = _peerPaths.GetSnapshot(peerNodeId); var hinted = observed.Length == 0 ? GetOrCreateDirectEndpointManager(peerNodeId).Endpoints : Array.Empty(); - + if (observed.Length == 0 && hinted.Length == 0) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "no direct peer paths are available for broadcast"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; } - - var parsed = TryGetPacketIdAndVerb(packet, out var pv) ? pv : default; - var shouldRecord = parsed.Verb != ZeroTierVerb.QosMeasurement; + + var parsedOk = TryGetPacketIdAndVerb(packet, out var parsed); + var shouldRecord = parsedOk && parsed.Verb != ZeroTierVerb.QosMeasurement; var anyConfirmed = false; var directSuccess = 0; if (observed.Length > 0) - { - var broadcast = ZeroTierPeerBondPolicyEngine.GetBroadcastPaths(observed); - for (var i = 0; i < broadcast.Length; i++) - { - var path = broadcast[i]; - if (_peerEcho.TryGetLastRttMs(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out _)) - { - anyConfirmed = true; - } - + { + var broadcast = ZeroTierPeerBondPolicyEngine.GetBroadcastPaths(observed); + for (var i = 0; i < broadcast.Length; i++) + { + var path = broadcast[i]; + if (_peerEcho.TryGetLastRttMs(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out _)) + { + anyConfirmed = true; + } + if (shouldRecord) { _peerQos.RecordOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); } - - try - { - await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); - directSuccess++; - } - catch (SocketException) - { + + try + { + await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); + directSuccess++; + } + catch (SocketException) + { if (shouldRecord) { _peerQos.ForgetOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, parsed.PacketId); @@ -429,20 +626,31 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo } else { + var localSockets = _udp.LocalSockets; for (var i = 0; i < hinted.Length; i++) { + var localSocketId = localSockets.Count == 0 ? 0 : localSockets[i % localSockets.Count].Id; + if (shouldRecord) + { + _peerQos.RecordOutgoingPacket(peerNodeId, localSocketId, hinted[i], parsed.PacketId); + } + try { - await _udp.SendAsync(localSocketId: 0, hinted[i], packet, cancellationToken).ConfigureAwait(false); + await _udp.SendAsync(localSocketId, hinted[i], packet, cancellationToken).ConfigureAwait(false); directSuccess++; } catch (SocketException) { + if (shouldRecord) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, localSocketId, hinted[i], parsed.PacketId); + } } } } - - if (_multipath.WarmupDuplicateToRoot && !anyConfirmed) + + if (_multipath.WarmupDuplicateToRoot && _multipath.AllowRootRelayFallback && !anyConfirmed) { await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); return; @@ -450,321 +658,2268 @@ private async ValueTask SendToPeerBroadcastAsync(NodeId peerNodeId, ReadOnlyMemo if (directSuccess == 0) { + EnsureRelayAllowedForPayload(peerNodeId, reason: "all direct broadcast sends failed"); await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); } } - private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSelectedPeerPath selected) + private bool TryGetDirectOnlyHintedPayloadFanout( + NodeId peerNodeId, + uint flowId, + out ZeroTierSelectedPeerPath[] fanoutPaths) { - var observed = _peerPaths.GetSnapshot(peerNodeId); - if (observed.Length > 0) + fanoutPaths = Array.Empty(); + if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) { - return _bondEngine.TrySelectSinglePath(peerNodeId, observed, flowId, _multipath.BondPolicy, out selected); + return false; } - var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; - if (hinted.Length > 0) + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var hinted = directEndpoints.Endpoints; + if (hinted.Length == 0) { - var index = hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length); - selected = new ZeroTierSelectedPeerPath(LocalSocketId: 0, hinted[index]); - return true; + return false; } - selected = default; - return false; - } + if (ShouldWaitForPinnedRendezvousBeforeHintedPayload(peerNodeId)) + { + return false; + } - private int? GetPathLatencyMsOrNull(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) - { - if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) + var restrictToPinnedRendezvous = ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId); + var selectedEndpoints = restrictToPinnedRendezvous + ? SelectPinnedRendezvousEndpoints(peerNodeId, hinted) + : hinted; + var unique = new HashSet(); + var candidates = new List(selectedEndpoints.Length * 2); + for (var i = 0; i < selectedEndpoints.Length; i++) { - return rttMs; + var socketIds = GetDirectOnlyHintedPayloadSocketIds( + peerNodeId, + selectedEndpoints[i], + restrictToPinnedRendezvous); + for (var s = 0; s < socketIds.Length; s++) + { + var candidate = new ZeroTierSelectedPeerPath(socketIds[s], selectedEndpoints[i]); + if (!unique.Add(new ZeroTierPeerPhysicalPathKey(candidate.LocalSocketId, candidate.RemoteEndPoint))) + { + continue; + } + + candidates.Add(candidate); + } } - if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) + if (candidates.Count == 0) { - return (int)Math.Min((long)latencyMs * 2, int.MaxValue); + return false; } - return null; + fanoutPaths = candidates.ToArray(); + return true; } - private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) - => _peerNegotiation.TryGetRemoteUtility(peerNodeId, localSocketId, remoteEndPoint, out var util) ? util : (short)0; + private IPEndPoint[] SelectPinnedRendezvousEndpoints(NodeId peerNodeId, IPEndPoint[] hinted) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var pinned = hinted + .Where(directEndpoints.IsPinnedRendezvousEndpoint) + .ToArray(); + return pinned.Length != 0 ? pinned : hinted; + } + + private bool ShouldAllowAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var endpointIsPublic = ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint); + var hasPinnedPublicSibling = endpointIsPublic && directEndpoints.Endpoints.Any(candidate => + directEndpoints.IsPinnedRendezvousEndpoint(candidate) && + ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidate) && + candidate.Address.Equals(endpoint.Address)); + var hasUsableSocketPath = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint).Length != 0; + + return ZeroTierPinnedRendezvousAlternateHintPolicy.ShouldProbe( + endpointIsPublic, + hasPinnedPublicSibling, + hasUsableSocketPath); + } - private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) + private int[] GetDirectOnlyHintedPayloadSocketIds( + NodeId peerNodeId, + IPEndPoint endpoint, + bool restrictToPinnedRendezvous) { - while (!cancellationToken.IsCancellationRequested) + if (restrictToPinnedRendezvous) { - try - { - await RunMultipathMaintenanceOnceAsync(cancellationToken).ConfigureAwait(false); - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (directEndpoints.IsPinnedRendezvousEndpoint(endpoint)) { - return; + return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); } -#pragma warning disable CA1031 // Maintenance loop must survive per-iteration faults. - catch (Exception ex) -#pragma warning restore CA1031 + + var pinnedEndpoints = directEndpoints.Endpoints; + if (pinnedEndpoints.Length != 0 && directEndpoints.IsPinnedRendezvousEndpoint(pinnedEndpoints[0])) { - ZeroTierTrace.WriteLine($"[zerotier] Multipath maintenance fault: {ex.GetType().Name}: {ex.Message}"); + return GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoints[0]); } } + + var preferredSocketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); + if (preferredSocketIds.Length != 0) + { + return [preferredSocketIds[0]]; + } + + return _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true); } - private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellationToken) + private async ValueTask TrySendDirectOnlyHintedPayloadAsync( + NodeId peerNodeId, + ReadOnlyMemory packet, + uint flowId, + ulong? packetId, + bool shouldRecord, + CancellationToken cancellationToken) { - _bondEngine.MaintenanceTick(); - - var peers = _peerPaths.GetPeersSnapshot(); - if (peers.Length == 0) + if (_multipath.AllowRootRelayFallback || HasConfirmedDirectPath(peerNodeId)) { - return; + return false; } - for (var i = 0; i < peers.Length; i++) - { - var peerNodeId = peers[i]; - if (!_peerSecurity.TryGetPeerKey(peerNodeId, out var key)) - { - continue; - } + await EnsureDirectBootstrapStartedAsync(peerNodeId, cancellationToken).ConfigureAwait(false); - var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); - var paths = _peerPaths.GetSnapshot(peerNodeId); - if (paths.Length == 0) - { - continue; - } + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); - for (var p = 0; p < paths.Length; p++) + if (TryGetDirectOnlyHintedPayloadFanout(peerNodeId, flowId, out var fanoutPaths)) { - var path = paths[p]; - - await _peerEcho - .TrySendEchoProbeAsync(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, key, cancellationToken) - .ConfigureAwait(false); + await PrimeDirectOnlyHintedPayloadPathsAsync(peerNodeId, fanoutPaths, cancellationToken).ConfigureAwait(false); - if (!_peerQos.TryBuildOutboundPayload(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out var qosPayload)) - { - continue; - } - - await SendPeerControlAsync( + var directSuccess = await SendDirectOnlyHintedPayloadFanoutAsync( peerNodeId, - path.LocalSocketId, - path.RemoteEndPoint, - ZeroTierVerb.QosMeasurement, - qosPayload, - key, - peerProtocolVersion, + fanoutPaths, + packet, + packetId, + shouldRecord, cancellationToken) .ConfigureAwait(false); - } + if (directSuccess > 0) + { + return true; + } - if (_multipath.BondPolicy != ZeroTierBondPolicy.ActiveBackup || paths.Length < 2) - { - continue; + EnsureRelayAllowedForPayload(peerNodeId, reason: "direct-only hinted payload fanout failed"); } - var bestPathIndex = -1; - var bestScore = int.MinValue; - var secondScore = int.MinValue; - - for (var p = 0; p < paths.Length; p++) + if (_directBootstrapTasks.TryGetValue(peerNodeId, out var bootstrapTask)) { - var path = paths[p]; - var score = ComputePathQualityScore(peerNodeId, path.LocalSocketId, path.RemoteEndPoint); - - if (score > bestScore) - { - secondScore = bestScore; - bestScore = score; - bestPathIndex = p; - } - else if (score > secondScore) + if (bootstrapTask.IsCompleted) { - secondScore = score; + return false; } } - - if (bestPathIndex < 0) + else { - continue; + return false; } - var bestPath = paths[bestPathIndex]; - if (!_peerNegotiation.TryMarkSent(peerNodeId, bestPath.LocalSocketId, bestPath.RemoteEndPoint)) - { - continue; - } + await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken).ConfigureAwait(false); + } + } - var delta = bestScore - Math.Max(secondScore, 0); - var utility = (short)Math.Clamp(delta, short.MinValue, short.MaxValue); - var payload = new byte[2]; - BinaryPrimitives.WriteInt16BigEndian(payload, utility); + private async ValueTask PrimeDirectOnlyHintedPayloadPathsAsync( + NodeId peerNodeId, + ZeroTierSelectedPeerPath[] fanoutPaths, + CancellationToken cancellationToken) + { + var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + await TrySendPeriodicDirectPathPushAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); - await SendPeerControlAsync( + for (var i = 0; i < fanoutPaths.Length; i++) + { + var path = fanoutPaths[i]; + var forceFullHello = ShouldForceFullHelloForDirectOnlyPayloadPrime(peerNodeId, path.RemoteEndPoint); + await SendDirectBootstrapProbeAsync( peerNodeId, - bestPath.LocalSocketId, - bestPath.RemoteEndPoint, - ZeroTierVerb.PathNegotiationRequest, - payload, - key, - peerProtocolVersion, + path.LocalSocketId, + path.RemoteEndPoint, + sharedKey, + forceFullHello, + helloMinIntervalMs: 0, cancellationToken) .ConfigureAwait(false); } } - private int ComputePathQualityScore(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + private async ValueTask SendDirectOnlyHintedPayloadFanoutAsync( + NodeId peerNodeId, + ZeroTierSelectedPeerPath[] fanoutPaths, + ReadOnlyMemory packet, + ulong? packetId, + bool shouldRecord, + CancellationToken cancellationToken) { - if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] TX direct-only payload fanout: peer={peerNodeId} paths={string.Join(", ", fanoutPaths.Select(static path => $"{path.RemoteEndPoint}@{path.LocalSocketId}"))}."); + } + + var directSuccess = 0; + for (var i = 0; i < fanoutPaths.Length; i++) { - return 32767 - Math.Clamp(rttMs, 0, 32767); + var path = fanoutPaths[i]; + if (shouldRecord && packetId is ulong outboundPacketId) + { + _peerQos.RecordOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, outboundPacketId); + } + + try + { + await _udp.SendAsync(path.LocalSocketId, path.RemoteEndPoint, packet, cancellationToken).ConfigureAwait(false); + directSuccess++; + } + catch (SocketException) + { + if (shouldRecord && packetId is ulong failedPacketId) + { + _peerQos.ForgetOutgoingPacket(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, failedPacketId); + } + } } - if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) + if (directSuccess > 0 && ZeroTierTrace.Enabled) { - var estRtt = (int)Math.Min((long)latencyMs * 2, 32767L); - return 32767 - estRtt; + ZeroTierTrace.WriteLine( + $"[zerotier] Direct-only payload fanout sent: peer={peerNodeId} successes={directSuccess}/{fanoutPaths.Length}."); } - return 0; + return directSuccess; } - private async ValueTask SendPeerControlAsync( - NodeId peerNodeId, - int localSocketId, - IPEndPoint remoteEndPoint, - ZeroTierVerb verb, - ReadOnlyMemory payload, - byte[] sharedKey, - byte remoteProtocolVersion, - CancellationToken cancellationToken) + private void EnsureRelayAllowedForPayload(NodeId peerNodeId, string reason) { - cancellationToken.ThrowIfCancellationRequested(); - - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); - var header = new ZeroTierPacketHeader( - PacketId: packetId, - Destination: peerNodeId, - Source: _localIdentity.NodeId, - Flags: 0, - Mac: 0, - VerbRaw: (byte)verb); - - var packet = ZeroTierPacketCodec.Encode(header, payload.Span); - ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), encryptPayload: false); - await _udp.SendAsync(localSocketId, remoteEndPoint, packet, cancellationToken).ConfigureAwait(false); + if (_multipath.AllowRootRelayFallback) + { + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Drop payload relay fallback: peer={peerNodeId} reason={reason}."); + } + + throw new InvalidOperationException( + $"No direct ZeroTier payload path is available for peer {peerNodeId}; relay fallback is disabled ({reason})."); + } + + private async ValueTask EnsureDirectPathAsync(NodeId peerNodeId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_multipath.Enabled || HasSufficientDirectPath(peerNodeId)) + { + return; + } + + var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + if (HasSufficientDirectPath(peerNodeId)) + { + return; + } + + var bootstrapTask = _directBootstrapTasks.GetOrAdd( + peerNodeId, + id => BootstrapDirectPathCoreAsync(id, (byte[])sharedKey.Clone(), _cts.Token)); + + try + { + await bootstrapTask.WaitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + if (bootstrapTask.IsCompleted) + { + _directBootstrapTasks.TryRemove(new KeyValuePair(peerNodeId, bootstrapTask)); + } + } + + if (!HasSufficientDirectPath(peerNodeId)) + { + EnsureRelayAllowedForPayload( + peerNodeId, + reason: _multipath.AllowRootRelayFallback + ? "direct bootstrap did not discover a peer path" + : "direct bootstrap did not discover a usable peer path"); + } + + if (ZeroTierTrace.Enabled) + { + var observed = _peerPaths.GetSnapshot(peerNodeId); + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + ZeroTierTrace.WriteLine( + $"[zerotier] Direct bootstrap result: peer={peerNodeId} confirmed={HasConfirmedDirectPath(peerNodeId)} observed={observed.Length} hinted={ZeroTierDirectEndpointSelection.Format(hinted)} relayAllowed={_multipath.AllowRootRelayFallback}."); + } + } + + private async ValueTask EnsureDirectBootstrapStartedAsync(NodeId peerNodeId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_multipath.Enabled || HasSufficientDirectPath(peerNodeId)) + { + return; + } + + var sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + if (HasSufficientDirectPath(peerNodeId)) + { + return; + } + + _directBootstrapStartedMs.TryAdd(peerNodeId, Environment.TickCount64); + _ = _directBootstrapTasks.GetOrAdd( + peerNodeId, + id => BootstrapDirectPathCoreAsync(id, (byte[])sharedKey.Clone(), _cts.Token)); + } + + private async Task BootstrapDirectPathCoreAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var bootstrapTimeout = _multipath.AllowRootRelayFallback + ? TimeSpan.FromSeconds(5) + : TimeSpan.FromSeconds(75); + var deadline = Environment.TickCount64 + (long)bootstrapTimeout.TotalMilliseconds; + var now = Environment.TickCount64; + var nextRootHelloAt = now; + var nextDirectPathPushAt = now; + var nextHintProbeAt = now; + + while (!HasSufficientDirectPath(peerNodeId)) + { + cancellationToken.ThrowIfCancellationRequested(); + + now = Environment.TickCount64; + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var hinted = ZeroTierPinnedRendezvousSiblingPortPredictor.Expand( + directEndpoints.Endpoints, + directEndpoints.IsPinnedRendezvousEndpoint); + var suppressRelayedRootControl = ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(peerNodeId, now); + var suppressRelayedRootMaintenance = ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved(peerNodeId); + if (unchecked(now - nextRootHelloAt) >= 0 && + !suppressRelayedRootMaintenance && + !suppressRelayedRootControl && + ZeroTierDirectBootstrapControlPolicy.ShouldRefreshRelayedRootControl( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId))) + { + await SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + await SendNetworkCredentialsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + nextRootHelloAt = now + DirectBootstrapRootHelloIntervalMs; + } + + if (unchecked(now - nextDirectPathPushAt) >= 0) + { + if (!suppressRelayedRootControl && !suppressRelayedRootMaintenance) + { + await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + } + + nextDirectPathPushAt = now + GetDirectPathPushBootstrapIntervalMs(peerNodeId); + } + + if (hinted.Length > 0 && unchecked(now - nextHintProbeAt) >= 0) + { + if (ShouldDelayDirectOnlyHintedControlUntilRendezvous(peerNodeId, hinted[0]) || + ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(peerNodeId)) + { + nextHintProbeAt = now + 250; + } + else + { + await ProbeHintedDirectEndpointsAsync(peerNodeId, hinted, sharedKey, cancellationToken).ConfigureAwait(false); + nextHintProbeAt = now + DirectBootstrapHintProbeIntervalMs; + } + } + + if (unchecked(Environment.TickCount64 - deadline) >= 0) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken).ConfigureAwait(false); + } + } + + private long GetDirectPathPushBootstrapIntervalMs(NodeId peerNodeId) + => HasConfirmedDirectPath(peerNodeId) + ? DirectBootstrapPathPushIntervalHavePathMs + : DirectBootstrapPathPushIntervalMs; + + private async Task TrySendPeriodicDirectPathPushAsync( + NodeId peerNodeId, + byte[] sharedKey, + CancellationToken cancellationToken) + { + var nowMs = Environment.TickCount64; + var minIntervalMs = GetDirectPathPushBootstrapIntervalMs(peerNodeId); + while (true) + { + if (_lastDirectPathPushSentMs.TryGetValue(peerNodeId, out var lastSent) && + unchecked(nowMs - lastSent) < minIntervalMs) + { + return; + } + + if (_lastDirectPathPushSentMs.TryAdd(peerNodeId, nowMs) || + _lastDirectPathPushSentMs.TryUpdate(peerNodeId, nowMs, lastSent)) + { + break; + } + } + + await SendPushDirectPathsViaRootAsync(peerNodeId, sharedKey, cancellationToken).ConfigureAwait(false); + } + + private async Task ProbeHintedDirectEndpointsAsync( + NodeId peerNodeId, + IPEndPoint[] hinted, + byte[] sharedKey, + CancellationToken cancellationToken) + { + if (hinted.Length == 0) + { + return; + } + + var endpointsToProbe = ShouldUseStickyHintedPathSelection(peerNodeId) + ? SelectStickyHintedEndpointsToProbe(peerNodeId, hinted) + : _directHintPlanner.TakeNextHintedEndpoints(peerNodeId, hinted, DirectBootstrapHintProbeBudget); + for (var i = 0; i < endpointsToProbe.Length; i++) + { + var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId, endpointsToProbe[i]); + var minIntervalMs = forceFullHello + ? DirectOnlyHintHelloIntervalMs + : DirectHelloMinIntervalMs; + var localSocketIds = ShouldUseStickyHintedPathSelection(peerNodeId) + ? GetStickyDirectHintProbeSocketIds(peerNodeId, endpointsToProbe[i], forceFullHello) + : _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpointsToProbe[i], + includeFallbackLocalSockets: true); + + RefreshPinnedRendezvousHolePunch(peerNodeId, endpointsToProbe[i], localSocketIds); + + if (forceFullHello && localSocketIds.Length > 1) + { + await SendHelloPacketAcrossSocketsAsync( + localSocketIds, + peerNodeId, + endpointsToProbe[i], + endpointsToProbe[i], + sharedKey, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + continue; + } + + for (var s = 0; s < localSocketIds.Length; s++) + { + await SendDirectBootstrapProbeAsync( + peerNodeId, + localSocketIds[s], + endpointsToProbe[i], + sharedKey, + forceFullHello, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + } + } + } + + private bool HasDirectPathCandidate(NodeId peerNodeId) + => _peerPaths.GetSnapshot(peerNodeId).Length != 0 || + GetOrCreateDirectEndpointManager(peerNodeId).Endpoints.Length != 0; + + private bool HasSufficientDirectPath(NodeId peerNodeId) + => _multipath.AllowRootRelayFallback + ? HasDirectPathCandidate(peerNodeId) + : HasConfirmedDirectPath(peerNodeId); + + private bool HasConfirmedDirectPath(NodeId peerNodeId) + { + if (_peerPaths.GetSnapshot(peerNodeId).Length != 0) + { + return true; + } + + var endpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (endpoints.Length == 0) + { + return false; + } + + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + for (var i = 0; i < endpoints.Length; i++) + { + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId: 0, endpoints[i], out _)) + { + return true; + } + } + + return false; + } + + for (var i = 0; i < endpoints.Length; i++) + { + for (var s = 0; s < localSockets.Count; s++) + { + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSockets[s].Id, endpoints[i], out _)) + { + return true; + } + } + } + + return false; + } + + private async Task SendHelloViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + if (ShouldUsePeerRootSocketAffinity(peerNodeId) && + _peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) + { + await SendHelloViaRootSocketAsync( + peerNodeId, + sharedKey, + rootSocketId, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + await SendHelloPacketAsync(localSocketId: 0, peerNodeId, physicalDestination: null, _rootEndpoint, sharedKey, cancellationToken) + .ConfigureAwait(false); + return; + } + + for (var i = 0; i < localSockets.Count; i++) + { + await SendHelloViaRootSocketAsync( + peerNodeId, + sharedKey, + localSockets[i].Id, + cancellationToken) + .ConfigureAwait(false); + } + } + + private Task SendHelloViaRootSocketAsync( + NodeId peerNodeId, + byte[] sharedKey, + int localSocketId, + CancellationToken cancellationToken) + => SendHelloPacketAsync( + localSocketId, + peerNodeId, + physicalDestination: null, + _rootEndpoint, + sharedKey, + cancellationToken); + + private async Task SendPushDirectPathsViaRootAsync( + NodeId peerNodeId, + byte[] sharedKey, + CancellationToken cancellationToken, + bool useFullLocalAdvertisements = false) + { + var advertisements = useFullLocalAdvertisements + ? GetLocalDirectPathAdvertisements() + : GetPeerAwareLocalDirectPathAdvertisements(peerNodeId); + if (advertisements.Length == 0) + { + return; + } + + try + { + var payload = ZeroTierPushDirectPathsCodec.BuildPayload(advertisements); + if (payload.Length <= 2) + { + return; + } + + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var header = new ZeroTierPacketHeader( + PacketId: packetId, + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.PushDirectPaths); + + var packet = ZeroTierPacketCodec.Encode(header, payload); + ZeroTierPacketCrypto.Armor( + packet, + ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), + encryptPayload: true); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] TX PUSH_DIRECT_PATHS via root for {peerNodeId}: {ZeroTierDirectEndpointSelection.Format(advertisements)}."); + } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] PUSH_DIRECT_PATHS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private IPEndPoint[] GetPeerAwareLocalDirectPathAdvertisements(NodeId peerNodeId) + { + var localAdvertisements = GetLocalDirectPathAdvertisements(); + var hintedPeerEndpoints = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + var observedPeerPaths = _peerPaths.GetSnapshot(peerNodeId); + return _localDirectAdvertisementPlanner.SelectForPeer(localAdvertisements, hintedPeerEndpoints, observedPeerPaths); + } + + private async Task SendViaRootSocketsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken) + { + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + await _udp.SendAsync(_rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + return; + } + + for (var i = 0; i < localSockets.Count; i++) + { + await _udp.SendAsync(localSockets[i].Id, _rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + } + + private Task SendViaPeerRootAsync(NodeId peerNodeId, ReadOnlyMemory packet, CancellationToken cancellationToken) + { + if (ShouldUsePeerRootSocketAffinity(peerNodeId) && + _peerRootSocketAffinity.TryGet(peerNodeId, out var localSocketId)) + { + return _udp.SendAsync(localSocketId, _rootEndpoint, packet, cancellationToken); + } + + return SendViaRootSocketsAsync(packet, cancellationToken); + } + + private bool ShouldUsePeerRootSocketAffinity(NodeId peerNodeId) + => _multipath.AllowRootRelayFallback || + HasConfirmedDirectPath(peerNodeId) || + (!_multipath.AllowRootRelayFallback && + GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints); + + private async Task SendNetworkCredentialsViaRootAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken) + { + if (_inlineCom.Length == 0) + { + return; + } + + var nowMs = Environment.TickCount64; + while (true) + { + if (_lastNetworkCredentialsBootstrapMs.TryGetValue(peerNodeId, out var lastSent) && + unchecked(nowMs - lastSent) < NetworkCredentialsBootstrapIntervalMs) + { + return; + } + + if (_lastNetworkCredentialsBootstrapMs.TryAdd(peerNodeId, nowMs) || + _lastNetworkCredentialsBootstrapMs.TryUpdate(peerNodeId, nowMs, lastSent)) + { + break; + } + } + + try + { + var remoteProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var packet = ZeroTierNetworkCredentialsPacketBuilder.BuildPacket( + packetId: ZeroTierPacketIdGenerator.GeneratePacketId(), + destination: peerNodeId, + source: _localIdentity.NodeId, + inlineCom: _inlineCom, + sharedKey: ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion)); + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX NETWORK_CREDENTIALS via root for {peerNodeId}."); + } + + await SendViaPeerRootAsync(peerNodeId, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] NETWORK_CREDENTIALS bootstrap send failed for {peerNodeId}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private async Task SendHelloDirectAsync( + NodeId peerNodeId, + IPEndPoint[] endpoints, + byte[] sharedKey, + CancellationToken cancellationToken) + { + for (var i = 0; i < endpoints.Length; i++) + { + var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoints[i]); + for (var s = 0; s < localSocketIds.Length; s++) + { + await SendHelloPacketAsync( + localSocketIds[s], + peerNodeId, + endpoints[i], + endpoints[i], + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } + } + } + + private async Task SendEchoDirectAsync( + NodeId peerNodeId, + IPEndPoint[] endpoints, + byte[] sharedKey, + CancellationToken cancellationToken) + { + for (var i = 0; i < endpoints.Length; i++) + { + var localSocketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoints[i]); + for (var s = 0; s < localSocketIds.Length; s++) + { + await TrySendEchoDirectProbeAsync(peerNodeId, localSocketIds[s], endpoints[i], sharedKey, cancellationToken) + .ConfigureAwait(false); + } + } + } + + private async ValueTask HandleDirectEndpointHintAsync( + NodeId peerNodeId, + int receivedLocalSocketId, + IPEndPoint endpoint, + bool forceFullHello, + bool useAllEligibleLocalSockets, + CancellationToken cancellationToken) + { + if (!_multipath.Enabled) + { + return; + } + + if (ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (directEndpoints.IsPinnedRendezvousEndpoint(endpoint)) + { + useAllEligibleLocalSockets = false; + } + else if (ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + { + useAllEligibleLocalSockets = !ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint); + } + else + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap skipped: peer={peerNodeId} endpoint={endpoint} mode=alternate-hint payloadPinned=True."); + } + + return; + } + } + else if (ShouldDelayDirectOnlyHintedControlUntilRendezvous(peerNodeId, endpoint)) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap delayed: peer={peerNodeId} endpoint={endpoint} reason=await-rendezvous."); + } + + return; + } + + if (ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(peerNodeId)) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap delayed: peer={peerNodeId} endpoint={endpoint} reason=await-peer-version."); + } + + return; + } + + byte[] sharedKey; + try + { + sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is InvalidOperationException or OperationCanceledException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap skipped: peer={peerNodeId} endpoint={endpoint} reason={ex.GetType().Name}: {ex.Message}"); + } + + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Direct hint bootstrap: peer={peerNodeId} endpoint={endpoint} socket={receivedLocalSocketId} fullHello={forceFullHello} allSockets={useAllEligibleLocalSockets}."); + } + + var localSocketIds = GetDirectHintBootstrapSocketIds( + peerNodeId, + endpoint, + receivedLocalSocketId, + useAllEligibleLocalSockets); + if (forceFullHello && useAllEligibleLocalSockets && localSocketIds.Length > 1) + { + await SendHelloPacketAcrossSocketsAsync( + localSocketIds, + peerNodeId, + endpoint, + endpoint, + sharedKey, + DirectHelloMinIntervalMs, + cancellationToken) + .ConfigureAwait(false); + return; + } + + for (var i = 0; i < localSocketIds.Length; i++) + { + await SendDirectBootstrapProbeAsync( + peerNodeId, + localSocketIds[i], + endpoint, + sharedKey, + forceFullHello, + DirectHelloMinIntervalMs, + cancellationToken) + .ConfigureAwait(false); + } + } + + private bool CanUseEchoForDirectBootstrap(NodeId peerNodeId) + { + var version = _peerSecurity.GetPeerVersionOrDefault(peerNodeId); + return version.ProtocolVersion >= 5 && + !(version.MajorVersion == 1 && version.MinorVersion == 1 && version.Revision == 0); + } + + private int[] GetDirectHintBootstrapSocketIds( + NodeId peerNodeId, + IPEndPoint endpoint, + int receivedLocalSocketId, + bool useAllEligibleLocalSockets) + { + if (useAllEligibleLocalSockets) + { + return _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + } + + if (ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + { + var alternateSocketIds = GetAlternatePinnedRendezvousSocketIds(peerNodeId, endpoint, receivedLocalSocketId); + if (alternateSocketIds.Length != 0) + { + return alternateSocketIds; + } + } + + if (receivedLocalSocketId >= 0) + { + return [receivedLocalSocketId]; + } + + var preferredSocketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); + if (preferredSocketIds.Length != 0) + { + return preferredSocketIds; + } + + var rotatingSocketIds = _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true); + if (rotatingSocketIds.Length != 0) + { + return rotatingSocketIds; + } + + return [receivedLocalSocketId]; + } + + private Task SendDirectBootstrapProbeAsync( + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + byte[] sharedKey, + bool forceFullHello, + long helloMinIntervalMs, + CancellationToken cancellationToken) + => forceFullHello || !CanUseEchoForDirectBootstrap(peerNodeId) + ? SendHelloPacketAsync( + localSocketId, + peerNodeId, + remoteEndPoint, + remoteEndPoint, + sharedKey, + helloMinIntervalMs, + cancellationToken) + : TrySendEchoDirectProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken); + + private async Task TrySendEchoDirectProbeAsync( + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + byte[] sharedKey, + CancellationToken cancellationToken) + { + try + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX ECHO bootstrap to {remoteEndPoint} (socket={localSocketId})."); + } + + await _peerEcho + .TrySendEchoProbeAsync(peerNodeId, localSocketId, remoteEndPoint, sharedKey, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] ECHO bootstrap send failed to {remoteEndPoint}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private async Task SendHelloPacketAsync( + int localSocketId, + NodeId peerNodeId, + IPEndPoint? physicalDestination, + IPEndPoint sendTo, + byte[] sharedKey, + long minIntervalMs, + CancellationToken cancellationToken) + { + if (physicalDestination is not null && !ShouldSendDirectHello(peerNodeId, localSocketId, sendTo, minIntervalMs)) + { + return; + } + + try + { + var reportedSurface = physicalDestination; + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] TX HELLO bootstrap to {sendTo} (socket={localSocketId}, reported={reportedSurface})."); + } + + var packet = ZeroTierHelloPacketBuilder.BuildPacket( + _localIdentity, + peerNodeId, + reportedSurface, + timestamp: (ulong)Environment.TickCount64, + _planetId, + _planetTimestamp, + sharedKey, + ZeroTierHelloClient.AdvertisedProtocolVersion, + ZeroTierHelloClient.AdvertisedMajorVersion, + ZeroTierHelloClient.AdvertisedMinorVersion, + ZeroTierHelloClient.AdvertisedRevision, + out var packetId); + + TrackPendingHello( + packetId, + new PendingHelloProbe( + PeerNodeId: peerNodeId, + LocalSocketId: localSocketId, + SendTo: sendTo, + PhysicalDestination: physicalDestination, + SentAtMs: Environment.TickCount64)); + + await _udp.SendAsync(localSocketId, sendTo, packet, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or InvalidOperationException) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] HELLO bootstrap send failed to {sendTo}: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private Task SendHelloPacketAsync( + int localSocketId, + NodeId peerNodeId, + IPEndPoint? physicalDestination, + IPEndPoint sendTo, + byte[] sharedKey, + CancellationToken cancellationToken) + => SendHelloPacketAsync( + localSocketId, + peerNodeId, + physicalDestination, + sendTo, + sharedKey, + DirectHelloMinIntervalMs, + cancellationToken); + + private async Task SendHelloPacketAcrossSocketsAsync( + int[] localSocketIds, + NodeId peerNodeId, + IPEndPoint? physicalDestination, + IPEndPoint sendTo, + byte[] sharedKey, + long minIntervalMs, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(localSocketIds); + + for (var i = 0; i < localSocketIds.Length; i++) + { + await SendHelloPacketAsync( + localSocketIds[i], + peerNodeId, + physicalDestination, + sendTo, + sharedKey, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + } + } + + private bool ShouldSendDirectHello(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, long minIntervalMs) + { + var endpoint = remoteEndPoint.Address.IsIPv4MappedToIPv6 + ? new IPEndPoint(remoteEndPoint.Address.MapToIPv4(), remoteEndPoint.Port) + : remoteEndPoint; + var key = (peerNodeId, localSocketId, endpoint.ToString()); + var nowMs = Environment.TickCount64; + + while (true) + { + if (_lastDirectHelloSentMs.TryGetValue(key, out var lastSent) && + unchecked(nowMs - lastSent) < minIntervalMs) + { + return false; + } + + if (_lastDirectHelloSentMs.TryAdd(key, nowMs) || + _lastDirectHelloSentMs.TryUpdate(key, nowMs, lastSent)) + { + return true; + } + } + } + + private bool TrySelectDirectPath(NodeId peerNodeId, uint flowId, out ZeroTierSelectedPeerPath selected) + { + var observed = _peerPaths.GetSnapshot(peerNodeId); + if (observed.Length > 0) + { + return _bondEngine.TrySelectSinglePath(peerNodeId, observed, flowId, _multipath.BondPolicy, out selected); + } + + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (TrySelectConfirmedHintedDirectPath(peerNodeId, hinted, out selected)) + { + return true; + } + + if (hinted.Length > 0) + { + var useStickyHintedPath = ShouldUseStickyHintedPathSelection(peerNodeId); + var endpoint = useStickyHintedPath + ? hinted[0] + : hinted[hinted.Length == 1 ? 0 : (int)(flowId % (uint)hinted.Length)]; + var preferredLocalSocketIds = useStickyHintedPath + ? GetStickyHintedLocalSocketIds(peerNodeId, endpoint) + : _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + + if (preferredLocalSocketIds.Length == 0) + { + selected = default; + return false; + } + + var localSocketId = useStickyHintedPath + ? preferredLocalSocketIds[0] + : preferredLocalSocketIds[preferredLocalSocketIds.Length <= 1 + ? 0 + : (int)((flowId / (uint)hinted.Length) % (uint)preferredLocalSocketIds.Length)]; + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Select hinted direct path: peer={peerNodeId} endpoint={endpoint} socket={localSocketId} relayAllowed={_multipath.AllowRootRelayFallback} confirmed={HasConfirmedDirectPath(peerNodeId)} sticky={useStickyHintedPath}."); + } + + selected = new ZeroTierSelectedPeerPath(localSocketId, endpoint); + return true; + } + + selected = default; + return false; + } + + private bool TrySelectConfirmedHintedDirectPath( + NodeId peerNodeId, + IPEndPoint[] hinted, + out ZeroTierSelectedPeerPath selected) + { + selected = default; + + var haveBest = false; + var bestRttMs = int.MaxValue; + for (var i = 0; i < hinted.Length; i++) + { + var socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); + for (var s = 0; s < socketIds.Length; s++) + { + if (!_peerEcho.TryGetLastRttMs(peerNodeId, socketIds[s], hinted[i], out var rttMs)) + { + continue; + } + + if (!haveBest || rttMs < bestRttMs) + { + bestRttMs = rttMs; + selected = new ZeroTierSelectedPeerPath(socketIds[s], hinted[i]); + haveBest = true; + } + } + } + + if (haveBest && ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine( + $"[zerotier] Select confirmed hinted direct path: peer={peerNodeId} endpoint={selected.RemoteEndPoint} socket={selected.LocalSocketId} rttMs={bestRttMs}."); + } + + return haveBest; + } + + private bool ShouldUseStickyHintedPathSelection(NodeId peerNodeId) + => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + + private IPEndPoint[] SelectStickyHintedEndpointsToProbe(NodeId peerNodeId, IPEndPoint[] hinted) + { + if (hinted.Length == 0) + { + return []; + } + + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + return [hinted[0]]; + } + + var selected = new List { hinted[0] }; + for (var i = 1; i < hinted.Length; i++) + { + if (ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, hinted[i])) + { + selected.Add(hinted[i]); + } + } + + return selected.ToArray(); + } + + private bool ShouldWaitForPinnedRendezvousBeforeHintedPayload(NodeId peerNodeId) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var nowMs = Environment.TickCount64; + _ = _directBootstrapStartedMs.TryGetValue(peerNodeId, out var bootstrapStartedAtMs); + _ = _pinnedRendezvousObservedMs.TryGetValue(peerNodeId, out var pinnedRendezvousObservedAtMs); + + return ZeroTierDirectBootstrapControlPolicy.ShouldWaitForDirectOnlyHintedPayload( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + nowMs, + bootstrapStartedAtMs == 0 ? null : bootstrapStartedAtMs, + DirectOnlyRendezvousGraceMs, + pinnedRendezvousObservedAtMs == 0 ? null : pinnedRendezvousObservedAtMs, + DirectOnlyPinnedRendezvousSettleMs); + } + + private bool ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(NodeId peerNodeId) + => ShouldUseStickyHintedPathSelection(peerNodeId) && + GetOrCreateDirectEndpointManager(peerNodeId).HasPinnedRendezvousEndpoints; + + private bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap(NodeId peerNodeId, long nowMs) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + _ = _pinnedRendezvousObservedMs.TryGetValue(peerNodeId, out var pinnedRendezvousObservedAtMs); + + return ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + CanUseEchoForDirectBootstrap(peerNodeId), + nowMs, + pinnedRendezvousObservedAtMs == 0 ? null : pinnedRendezvousObservedAtMs, + DirectOnlyPinnedRendezvousRelayQuietMs); + } + + private bool ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved(NodeId peerNodeId) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + return ZeroTierDirectBootstrapControlPolicy.ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints); + } + + private bool ShouldDelayDirectOnlyHintedControlUntilRendezvous(NodeId peerNodeId, IPEndPoint endpoint) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + return ZeroTierDirectBootstrapControlPolicy.ShouldDelayDirectOnlyHintedControlUntilRendezvous( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + directEndpoints.IsPinnedRendezvousEndpoint(endpoint)); + } + + private bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown(NodeId peerNodeId) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + _ = _pinnedRendezvousObservedMs.TryGetValue(peerNodeId, out var pinnedRendezvousObservedAtMs); + + return ZeroTierDirectBootstrapControlPolicy.ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(peerNodeId), + directEndpoints.HasPinnedRendezvousEndpoints, + CanUseEchoForDirectBootstrap(peerNodeId), + Environment.TickCount64, + pinnedRendezvousObservedAtMs == 0 ? null : pinnedRendezvousObservedAtMs, + DirectOnlyPinnedRendezvousVersionGraceMs); + } + + private int[] GetStickyDirectHintProbeSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) + { + if (ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + { + var alternateSocketIds = GetAlternatePinnedRendezvousSocketIds(peerNodeId, endpoint); + if (alternateSocketIds.Length != 0) + { + return alternateSocketIds; + } + } + + if (ShouldFanOutFullHelloAcrossEligibleSockets(peerNodeId, endpoint, forceFullHello)) + { + return _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + } + + return GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + } + + private bool ShouldFanOutFullHelloAcrossEligibleSockets(NodeId peerNodeId, IPEndPoint endpoint, bool forceFullHello) + { + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId)) + { + return false; + } + + if (!GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint)) + { + return false; + } + + return forceFullHello || !CanUseEchoForDirectBootstrap(peerNodeId); + } + + private int[] GetStickyHintedLocalSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + { + var socketIds = _directHintPlanner.GetPreferredSocketIds(peerNodeId, endpoint); + if (socketIds.Length == 0) + { + socketIds = _directHintPlanner.GetPreferredAndFallbackSocketIds(peerNodeId, endpoint); + } + + if (socketIds.Length == 0) + { + return socketIds; + } + + return [socketIds[0]]; + } + + private int[] GetPinnedRendezvousStickySocketIds(NodeId peerNodeId) + { + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + var pinnedEndpoint = directEndpoints.Endpoints.FirstOrDefault(directEndpoints.IsPinnedRendezvousEndpoint); + if (pinnedEndpoint is null) + { + return []; + } + + var socketIds = GetStickyHintedLocalSocketIds(peerNodeId, pinnedEndpoint); + if (socketIds.Length != 0) + { + return socketIds; + } + + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) + { + return [rootSocketId]; + } + + return []; + } + + private int[] GetAlternatePinnedRendezvousSocketIds(NodeId peerNodeId, IPEndPoint endpoint, int fallbackLocalSocketId = -1) + { + if (ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint)) + { + var pinnedSocketIds = GetPinnedRendezvousStickySocketIds(peerNodeId); + if (pinnedSocketIds.Length != 0) + { + return pinnedSocketIds; + } + } + + if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + if (fallbackLocalSocketId >= 0) + { + return [fallbackLocalSocketId]; + } + + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var observedRootSocketId)) + { + return [observedRootSocketId]; + } + } + + var socketIds = GetStickyHintedLocalSocketIds(peerNodeId, endpoint); + if (socketIds.Length != 0) + { + return socketIds; + } + + if (fallbackLocalSocketId >= 0) + { + return [fallbackLocalSocketId]; + } + + if (_peerRootSocketAffinity.TryGet(peerNodeId, out var rootSocketId)) + { + return [rootSocketId]; + } + + return []; + } + + private bool ShouldUseSingleSocketForAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) + => ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) && + ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint); + + private bool ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(NodeId peerNodeId, IPEndPoint endpoint) + { + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) || + !ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint) || + !ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + return false; + } + + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + return directEndpoints.Endpoints.Any(candidate => + directEndpoints.IsPinnedRendezvousEndpoint(candidate) && + ZeroTierDirectEndpointSelection.IsPublicEndpoint(candidate) && + candidate.Address.Equals(endpoint.Address)); } - private static bool TryGetPacketIdAndVerb(ReadOnlyMemory packet, out (ulong PacketId, ZeroTierVerb Verb) parsed) - { - if (packet.Length < ZeroTierPacketHeader.Length) + private void RefreshPinnedRendezvousHolePunch(NodeId peerNodeId, IPEndPoint endpoint, int[] localSocketIds) + { + if (_multipath.AllowRootRelayFallback || localSocketIds.Length == 0) + { + return; + } + + GetOrCreateDirectEndpointManager(peerNodeId).RefreshPinnedRendezvousHolePunch(endpoint, localSocketIds); + } + + private int? GetPathLatencyMsOrNull(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + { + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) + { + return rttMs; + } + + if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) + { + return (int)Math.Min((long)latencyMs * 2, int.MaxValue); + } + + return null; + } + + private void HandlePeerHelloOk( + NodeId peerNodeId, + int receivedLocalSocketId, + IPEndPoint receivedVia, + byte hopCount, + ZeroTierHelloOkPayload ok) + { + ArgumentNullException.ThrowIfNull(receivedVia); + _peerRootSocketAffinity.Observe(peerNodeId, receivedLocalSocketId, receivedVia); + + var matchedPending = TryTakePendingHello(peerNodeId, ok.InRePacketId, receivedLocalSocketId, receivedVia, out var pending); + var observedLocalSocketId = matchedPending ? pending.LocalSocketId : receivedLocalSocketId; + var trustedSurface = matchedPending && + hopCount == 0 && + pending.PhysicalDestination is { } physicalDestination && + receivedVia.Equals(physicalDestination); + + if (hopCount == 0 && ok.ExternalSurfaceAddress is { } surface) { - parsed = default; - return false; + _surfaceAddresses.Observe(peerNodeId, observedLocalSocketId, receivedVia, surface); } - var span = packet.Span; - var packetId = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(ZeroTierPacketHeader.IndexPacketId, 8)); - var verb = (ZeroTierVerb)(span[ZeroTierPacketHeader.IndexVerb] & 0x1F); - parsed = (packetId, verb); - return true; + if (ZeroTierTrace.Enabled) + { + var surfaceText = ok.ExternalSurfaceAddress?.ToString() ?? ""; + var probeText = matchedPending + ? $"sendTo={pending.SendTo} physical={pending.PhysicalDestination?.ToString() ?? ""}" + : "sendTo= physical="; + ZeroTierTrace.WriteLine( + $"[zerotier] RX OK(HELLO): peer={peerNodeId} via={receivedVia} hop={hopCount} inRe=0x{ok.InRePacketId:x16} localSocket={observedLocalSocketId} surface={surfaceText} matched={matchedPending} trustedSurface={trustedSurface} {probeText}."); + } + + if (!matchedPending) + { + return; + } + + if (matchedPending && + hopCount == 0 && + pending.PhysicalDestination is { } confirmedPhysicalDestination && + receivedVia.Equals(confirmedPhysicalDestination)) + { + _peerEcho.ObserveHelloOkRtt(peerNodeId, pending.LocalSocketId, receivedVia, ok.TimestampEcho); + return; + } + + _peerEcho.ObserveHelloOkRtt(peerNodeId, receivedLocalSocketId, receivedVia, ok.TimestampEcho); } - public async ValueTask DisposeAsync() + private async ValueTask HandleUnknownDirectPathAsync( + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + CancellationToken cancellationToken) { - if (_disposed) + if (!_multipath.Enabled) { return; } - _disposed = true; - _peerQueue.Writer.TryComplete(); - await _cts.CancelAsync().ConfigureAwait(false); + byte[] sharedKey; + try + { + sharedKey = await GetPeerKeyAsync(peerNodeId, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is InvalidOperationException or OperationCanceledException) + { + return; + } - if (_multipathMaintenanceLoop is not null) + if (ZeroTierTrace.Enabled) { - try + ZeroTierTrace.WriteLine($"[zerotier] Confirm unknown hop-0 path: peer={peerNodeId} endpoint={remoteEndPoint} socket={localSocketId}."); + } + + await SendHelloPacketAsync( + localSocketId, + peerNodeId, + remoteEndPoint, + remoteEndPoint, + sharedKey, + cancellationToken) + .ConfigureAwait(false); + } + + private void HandleAuthenticatedPeer(NodeId peerNodeId) + { + _authenticatedPeers[peerNodeId] = 0; + } + + private void TrackPendingHello(ulong packetId, PendingHelloProbe probe) + { + CleanupPendingHellosIfNeeded(Environment.TickCount64); + _pendingHelloProbes[packetId] = new PendingHelloProbeSet([probe]); + } + + private void TrackPendingHello(ulong packetId, PendingHelloProbe[] probes) + { + CleanupPendingHellosIfNeeded(Environment.TickCount64); + _pendingHelloProbes[packetId] = new PendingHelloProbeSet(probes); + } + + private bool TryTakePendingHello( + NodeId peerNodeId, + ulong packetId, + int receivedLocalSocketId, + IPEndPoint receivedVia, + out PendingHelloProbe pending) + { + CleanupPendingHellosIfNeeded(Environment.TickCount64); + if (_pendingHelloProbes.TryRemove(packetId, out var exact) && + exact.TryTakeBest(peerNodeId, receivedLocalSocketId, receivedVia, out pending)) + { + return true; + } + + foreach (var candidate in _pendingHelloProbes) + { + if (!ZeroTierReplyCorrelation.Matches(candidate.Key, packetId)) { - await _multipathMaintenanceLoop.ConfigureAwait(false); + continue; } - catch (OperationCanceledException) when (_cts.IsCancellationRequested) + + if (_pendingHelloProbes.TryRemove(candidate.Key, out var correlated) && + correlated.TryTakeBest(peerNodeId, receivedLocalSocketId, receivedVia, out pending)) { + return true; } } - await _udp.DisposeAsync().ConfigureAwait(false); + pending = default; + return false; + } - try + private void CleanupPendingHellosIfNeeded(long nowMs) + { + var last = Volatile.Read(ref _lastPendingHelloCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 5_000) { - await _dispatcherLoop.ConfigureAwait(false); + return; } - catch (OperationCanceledException) when (_cts.IsCancellationRequested) + + if (Interlocked.CompareExchange(ref _lastPendingHelloCleanupMs, nowMs, last) != last) { + return; } - catch (ChannelClosedException) when (_cts.IsCancellationRequested) + + var cutoff = nowMs - (long)TimeSpan.FromMinutes(2).TotalMilliseconds; + foreach (var pair in _pendingHelloProbes) { + if (pair.Value.SentAtMs <= cutoff) + { + _pendingHelloProbes.TryRemove(pair.Key, out _); + } } + } + + private short GetRemoteUtilityOrZero(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + => _peerNegotiation.TryGetRemoteUtility(peerNodeId, localSocketId, remoteEndPoint, out var util) ? util : (short)0; - try + private bool ShouldFanOutHintedBootstrapAcrossSockets(NodeId peerNodeId) + => !_multipath.AllowRootRelayFallback && !HasConfirmedDirectPath(peerNodeId); + + private bool ShouldPeriodicallyForceHelloOnStickyHintedPath(NodeId peerNodeId, IPEndPoint endpoint) + => ShouldUseStickyHintedPathSelection(peerNodeId) && + CanUseEchoForDirectBootstrap(peerNodeId) && + !ShouldUsePinnedSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint) && + !GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint); + + private bool ShouldForceFullHelloForDirectOnlyPayloadPrime(NodeId peerNodeId, IPEndPoint endpoint) + { + if (!ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId) || + !CanUseEchoForDirectBootstrap(peerNodeId)) + { + return false; + } + + return !GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint(endpoint); + } + + private async Task MultipathMaintenanceLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await RunMultipathMaintenanceOnceAsync(cancellationToken).ConfigureAwait(false); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } +#pragma warning disable CA1031 // Maintenance loop must survive per-iteration faults. + catch (Exception ex) +#pragma warning restore CA1031 + { + ZeroTierTrace.WriteLine($"[zerotier] Multipath maintenance fault: {ex.GetType().Name}: {ex.Message}"); + } + } + } + + private async Task RunMultipathMaintenanceOnceAsync(CancellationToken cancellationToken) + { + var nowMs = Environment.TickCount64; + _bondEngine.MaintenanceTick(); + CleanupDirectEndpointManagers(nowMs); + + var peers = GetPeersForMultipathMaintenance(); + if (peers.Length == 0) + { + return; + } + + for (var i = 0; i < peers.Length; i++) { - await _peerLoop.ConfigureAwait(false); + var peerNodeId = peers[i]; + if (!_peerSecurity.TryGetPeerKey(peerNodeId, out var key)) + { + continue; + } + + if (!ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved(peerNodeId)) + { + await TrySendPeriodicDirectPathPushAsync(peerNodeId, key, cancellationToken).ConfigureAwait(false); + } + + var peerProtocolVersion = _peerSecurity.GetPeerProtocolVersionOrDefault(peerNodeId); + var paths = _peerPaths.GetSnapshot(peerNodeId); + var hintedCandidates = GetHintedCandidatesForMaintenance(peerNodeId, paths); + if (paths.Length == 0 && hintedCandidates.Length == 0) + { + continue; + } + + for (var c = 0; c < hintedCandidates.Length; c++) + { + var candidate = hintedCandidates[c]; + var forceFullHello = ShouldPeriodicallyForceHelloOnStickyHintedPath(peerNodeId, candidate.RemoteEndPoint); + var minIntervalMs = forceFullHello + ? (ShouldUseStickyHintedPathSelection(peerNodeId) + ? DirectOnlyHintHelloIntervalMs + : DirectHintFullHelloIntervalMs) + : DirectHintFullHelloIntervalMs; + var localSocketIds = GetHintedMaintenanceProbeSocketIds(peerNodeId, candidate, forceFullHello); + + RefreshPinnedRendezvousHolePunch(peerNodeId, candidate.RemoteEndPoint, localSocketIds); + + if (forceFullHello && localSocketIds.Length > 1) + { + await SendHelloPacketAcrossSocketsAsync( + localSocketIds, + peerNodeId, + candidate.RemoteEndPoint, + candidate.RemoteEndPoint, + key, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + continue; + } + + for (var s = 0; s < localSocketIds.Length; s++) + { + await SendDirectBootstrapProbeAsync( + peerNodeId, + localSocketIds[s], + candidate.RemoteEndPoint, + key, + forceFullHello, + minIntervalMs, + cancellationToken) + .ConfigureAwait(false); + } + } + + for (var p = 0; p < paths.Length; p++) + { + var path = paths[p]; + + await _peerEcho + .TrySendEchoProbeAsync(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, key, cancellationToken) + .ConfigureAwait(false); + + if (!_peerQos.TryBuildOutboundPayload(peerNodeId, path.LocalSocketId, path.RemoteEndPoint, out var qosPayload)) + { + continue; + } + + await SendPeerControlAsync( + peerNodeId, + path.LocalSocketId, + path.RemoteEndPoint, + ZeroTierVerb.QosMeasurement, + qosPayload, + key, + peerProtocolVersion, + cancellationToken) + .ConfigureAwait(false); + } + + if (_multipath.BondPolicy != ZeroTierBondPolicy.ActiveBackup || paths.Length < 2) + { + continue; + } + + var bestPathIndex = -1; + var bestScore = int.MinValue; + var secondScore = int.MinValue; + + for (var p = 0; p < paths.Length; p++) + { + var path = paths[p]; + var score = ComputePathQualityScore(peerNodeId, path.LocalSocketId, path.RemoteEndPoint); + + if (score > bestScore) + { + secondScore = bestScore; + bestScore = score; + bestPathIndex = p; + } + else if (score > secondScore) + { + secondScore = score; + } + } + + if (bestPathIndex < 0) + { + continue; + } + + var bestPath = paths[bestPathIndex]; + if (!_peerNegotiation.TryMarkSent(peerNodeId, bestPath.LocalSocketId, bestPath.RemoteEndPoint)) + { + continue; + } + + var delta = bestScore - Math.Max(secondScore, 0); + var utility = (short)Math.Clamp(delta, short.MinValue, short.MaxValue); + var payload = new byte[2]; + BinaryPrimitives.WriteInt16BigEndian(payload, utility); + + await SendPeerControlAsync( + peerNodeId, + bestPath.LocalSocketId, + bestPath.RemoteEndPoint, + ZeroTierVerb.PathNegotiationRequest, + payload, + key, + peerProtocolVersion, + cancellationToken) + .ConfigureAwait(false); } - catch (OperationCanceledException) when (_cts.IsCancellationRequested) + } + + private int[] GetHintedMaintenanceProbeSocketIds( + NodeId peerNodeId, + ZeroTierSelectedPeerPath candidate, + bool forceFullHello) + { + if (!ShouldUseStickyHintedPathSelection(peerNodeId)) { + return [candidate.LocalSocketId]; } - catch (ChannelClosedException) when (_cts.IsCancellationRequested) + + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (directEndpoints.IsPinnedRendezvousEndpoint(candidate.RemoteEndPoint)) { + return GetStickyDirectHintProbeSocketIds(peerNodeId, candidate.RemoteEndPoint, forceFullHello); } - _cts.Dispose(); - _peerSecurity.Dispose(); + return [candidate.LocalSocketId]; + } + + private int ComputePathQualityScore(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + { + if (_peerEcho.TryGetLastRttMs(peerNodeId, localSocketId, remoteEndPoint, out var rttMs)) + { + return 32767 - Math.Clamp(rttMs, 0, 32767); + } + + if (_peerQos.TryGetLastLatencyAverageMs(peerNodeId, localSocketId, remoteEndPoint, out var latencyMs)) + { + var estRtt = (int)Math.Min((long)latencyMs * 2, 32767L); + return 32767 - estRtt; + } + + return 0; + } + + private async ValueTask SendPeerControlAsync( + NodeId peerNodeId, + int localSocketId, + IPEndPoint remoteEndPoint, + ZeroTierVerb verb, + ReadOnlyMemory payload, + byte[] sharedKey, + byte remoteProtocolVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); + var header = new ZeroTierPacketHeader( + PacketId: packetId, + Destination: peerNodeId, + Source: _localIdentity.NodeId, + Flags: 0, + Mac: 0, + VerbRaw: (byte)verb); + + var packet = ZeroTierPacketCodec.Encode(header, payload.Span); + ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, remoteProtocolVersion), encryptPayload: false); + await _udp.SendAsync(localSocketId, remoteEndPoint, packet, cancellationToken).ConfigureAwait(false); } - private Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken cancellationToken) - => _peerSecurity.GetPeerKeyAsync(peerNodeId, cancellationToken); + private ZeroTierSelectedPeerPath[] GetHintedCandidatesForMaintenance(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths) + { + if (_multipath.AllowRootRelayFallback || observedPaths.Length != 0 || HasConfirmedDirectPath(peerNodeId)) + { + return _directHintPlanner.GetNextHintedCandidatesForMaintenance( + peerNodeId, + observedPaths, + DirectHintMaintenanceProbeBudget); + } + + var hinted = GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length == 0) + { + return Array.Empty(); + } + + if (ShouldUseStickyHintedPathSelection(peerNodeId)) + { + return _stickyHintMaintenanceSelector.SelectCandidates( + peerNodeId, + hinted, + DirectHintMaintenanceProbeBudget, + ShouldRestrictDirectOnlyHintedTrafficToPinnedRendezvous(peerNodeId), + GetOrCreateDirectEndpointManager(peerNodeId).IsPinnedRendezvousEndpoint, + endpoint => ShouldAllowAlternatePinnedRendezvousHint(peerNodeId, endpoint), + endpoint => GetStickyHintedLocalSocketIds(peerNodeId, endpoint), + endpoint => ShouldUseSingleSocketForAlternatePinnedRendezvousHint(peerNodeId, endpoint) + ? GetAlternatePinnedRendezvousSocketIds(peerNodeId, endpoint) + : _directHintPlanner.GetRotatingSocketIds( + peerNodeId, + endpoint, + includeFallbackLocalSockets: true)); + } + + var selectedEndpoints = _directHintPlanner.TakeNextHintedEndpoints( + peerNodeId, + hinted, + DirectHintMaintenanceProbeBudget); + var candidates = new List(selectedEndpoints.Length); + for (var i = 0; i < selectedEndpoints.Length; i++) + { + var socketIds = GetStickyHintedLocalSocketIds(peerNodeId, selectedEndpoints[i]); + if (socketIds.Length == 0) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(socketIds[0], selectedEndpoints[i])); + } - private ValueTask HandleRootControlPacketAsync( + return candidates.ToArray(); + } + + private static bool TryGetPacketIdAndVerb(ReadOnlyMemory packet, out (ulong PacketId, ZeroTierVerb Verb) parsed) + { + if (packet.Length < ZeroTierPacketHeader.Length) + { + parsed = default; + return false; + } + + var span = packet.Span; + var packetId = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(ZeroTierPacketHeader.IndexPacketId, 8)); + var verb = (ZeroTierVerb)(span[ZeroTierPacketHeader.IndexVerb] & 0x1F); + parsed = (packetId, verb); + return true; + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + _peerQueue.Writer.TryComplete(); + await _cts.CancelAsync().ConfigureAwait(false); + + if (_multipathMaintenanceLoop is not null) + { + try + { + await _multipathMaintenanceLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + } + } + + await _udp.DisposeAsync().ConfigureAwait(false); + + try + { + await _dispatcherLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + } + catch (ChannelClosedException) when (_cts.IsCancellationRequested) + { + } + + try + { + await _peerLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + } + catch (ChannelClosedException) when (_cts.IsCancellationRequested) + { + } + + _cts.Dispose(); + _peerSecurity.Dispose(); + } + + private Task GetPeerKeyAsync(NodeId peerNodeId, CancellationToken cancellationToken) + => _peerSecurity.GetPeerKeyAsync(peerNodeId, cancellationToken); + + private async ValueTask HandleRootControlPacketAsync( ZeroTierVerb verb, ReadOnlyMemory payload, + int receivedLocalSocketId, IPEndPoint receivedVia, CancellationToken cancellationToken) { if (verb != ZeroTierVerb.Rendezvous) { - return ValueTask.CompletedTask; + return; } if (!ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) || rendezvous.With.Value == 0) { - return ValueTask.CompletedTask; + return; + } + + _peerRootSocketAffinity.Observe(rendezvous.With, receivedLocalSocketId, receivedVia); + + var shouldRelayRootControlOnRendezvous = ZeroTierDirectBootstrapControlPolicy.ShouldRelayRootControlOnRendezvous( + _multipath.AllowRootRelayFallback, + HasConfirmedDirectPath(rendezvous.With)); + if (shouldRelayRootControlOnRendezvous && + _peerSecurity.TryGetPeerKey(rendezvous.With, out var sharedKey)) + { + await SendHelloViaRootSocketAsync( + rendezvous.With, + sharedKey, + receivedLocalSocketId, + cancellationToken) + .ConfigureAwait(false); + + await SendPushDirectPathsViaRootAsync( + rendezvous.With, + sharedKey, + cancellationToken, + useFullLocalAdvertisements: true) + .ConfigureAwait(false); } + _pinnedRendezvousObservedMs[rendezvous.With] = Environment.TickCount64; + var directEndpoints = GetOrCreateDirectEndpointManager(rendezvous.With); - return directEndpoints.HandleRendezvousFromRootAsync(payload, receivedVia, cancellationToken); + await directEndpoints.HandleRendezvousFromRootAsync(payload, receivedLocalSocketId, receivedVia, cancellationToken) + .ConfigureAwait(false); } private ValueTask HandlePeerControlPacketAsync( NodeId peerNodeId, ZeroTierVerb verb, + int receivedLocalSocketId, + IPEndPoint receivedVia, ReadOnlyMemory payload, CancellationToken cancellationToken) { - if (verb != ZeroTierVerb.PushDirectPaths) + _peerRootSocketAffinity.Observe(peerNodeId, receivedLocalSocketId, receivedVia); + + var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); + if (verb == ZeroTierVerb.PushDirectPaths) { - return ValueTask.CompletedTask; + return directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, receivedLocalSocketId, cancellationToken); } - var directEndpoints = GetOrCreateDirectEndpointManager(peerNodeId); - return directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, cancellationToken); + return ValueTask.CompletedTask; } private ZeroTierDirectEndpointManager GetOrCreateDirectEndpointManager(NodeId peerNodeId) - => _directEndpoints.GetOrAdd(peerNodeId, id => new ZeroTierDirectEndpointManager(_udp, _rootEndpoint, id)); + { + _directEndpointLastUsedMs[peerNodeId] = Environment.TickCount64; + return _directEndpoints.GetOrAdd( + peerNodeId, + id => new ZeroTierDirectEndpointManager( + _udp, + _rootEndpoint, + id, + HandleDirectEndpointHintAsync, + ShouldAcceptDirectEndpoint)); + } + + private bool ShouldAcceptDirectEndpoint(IPEndPoint endpoint) + { + if (_directEndpointPolicy.ShouldAccept(endpoint)) + { + return true; + } + + var localSockets = _udp.LocalSockets; + for (var i = 0; i < localSockets.Count; i++) + { + if (_directPathSocketAdmissibility.ShouldUsePath(localSockets[i], endpoint)) + { + return true; + } + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Drop unusable hinted endpoint: {endpoint}."); + } + + return false; + } + + private void CleanupDirectEndpointManagers(long nowMs) + { + var last = Volatile.Read(ref _lastDirectEndpointCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 10_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastDirectEndpointCleanupMs, nowMs, last) != last) + { + return; + } + + foreach (var pair in _directEndpointLastUsedMs) + { + if (unchecked(nowMs - pair.Value) > DirectEndpointManagerTtlMs) + { + _directEndpointLastUsedMs.TryRemove(pair.Key, out _); + _directEndpoints.TryRemove(pair.Key, out _); + _directHintPlanner.RemovePeer(pair.Key); + _peerRootSocketAffinity.Remove(pair.Key); + } + } + } + + internal ZeroTierSelectedPeerPath[] GetHintedDirectCandidatesForMaintenance(NodeId peerNodeId) + => _directHintPlanner.GetHintedCandidates(peerNodeId, _peerPaths.GetSnapshot(peerNodeId)); + + internal IPEndPoint[] GetDirectEndpointsForTests(NodeId peerNodeId) + => GetOrCreateDirectEndpointManager(peerNodeId).Endpoints; + + internal Task RunMultipathMaintenanceOnceForTestsAsync(CancellationToken cancellationToken = default) + => RunMultipathMaintenanceOnceAsync(cancellationToken); + + internal NodeId[] GetPeersForMultipathMaintenanceForTests() + => GetPeersForMultipathMaintenance(); + internal IPEndPoint[] GetLocalDirectPathAdvertisementsForTests() + => GetLocalDirectPathAdvertisements(); + + internal Task SendViaRootSocketsForTestsAsync(ReadOnlyMemory packet, CancellationToken cancellationToken = default) + => SendViaRootSocketsAsync(packet, cancellationToken); + + internal bool TrySelectDirectPathForTests(NodeId peerNodeId, uint flowId, out ZeroTierSelectedPeerPath selected) + => TrySelectDirectPath(peerNodeId, flowId, out selected); + + internal void SeedDirectEndpointsForTests(NodeId peerNodeId, params IPEndPoint[] endpoints) + => GetOrCreateDirectEndpointManager(peerNodeId).SeedEndpoints(endpoints); + + internal void ObservePeerRootSocketForTests(NodeId peerNodeId, int localSocketId) + => _peerRootSocketAffinity.Observe(peerNodeId, localSocketId, _rootEndpoint); + + internal Task SendHelloViaRootForTestsAsync(NodeId peerNodeId, byte[] sharedKey, CancellationToken cancellationToken = default) + => SendHelloViaRootAsync(peerNodeId, sharedKey, cancellationToken); + + internal UserSpaceTcpConnectOptions GetTcpConnectOptionsForTests() + => GetSuggestedTcpConnectOptions(); + + internal void PrimePeerForTests( + ZeroTierIdentity peerIdentity, + byte peerProtocolVersion = ZeroTierHelloClient.AdvertisedProtocolVersion, + byte peerMajorVersion = ZeroTierHelloClient.AdvertisedMajorVersion, + byte peerMinorVersion = ZeroTierHelloClient.AdvertisedMinorVersion, + ushort peerRevision = ZeroTierHelloClient.AdvertisedRevision) + { + ArgumentNullException.ThrowIfNull(peerIdentity); + + if (_localIdentity.PrivateKey is null) + { + throw new InvalidOperationException("Local identity private key is required for test priming."); + } + + var sharedKey = new byte[48]; + ZeroTierC25519.Agree(_localIdentity.PrivateKey, peerIdentity.PublicKey, sharedKey); + _peerSecurity.PrimePeerForTests( + peerIdentity.NodeId, + sharedKey, + peerProtocolVersion, + peerMajorVersion, + peerMinorVersion, + peerRevision); + } + + internal UserSpaceTcpConnectOptions GetSuggestedTcpConnectOptions() + => _multipath.Enabled && !_multipath.AllowRootRelayFallback + ? UserSpaceTcpConnectOptions.DirectOnlyMultipath + : UserSpaceTcpConnectOptions.Default; + + private NodeId[] GetPeersForMultipathMaintenance() + { + var peers = _peerPaths.GetPeersSnapshot(); + if (_directEndpoints.IsEmpty && _authenticatedPeers.IsEmpty) + { + var trustedPeers = _peerSecurity.GetTrustedPeersSnapshot(); + if (trustedPeers.Length == 0) + { + return peers; + } + + return peers.Concat(trustedPeers).Distinct().ToArray(); + } + + var set = peers.ToHashSet(); + foreach (var peerNodeId in _peerSecurity.GetTrustedPeersSnapshot()) + { + set.Add(peerNodeId); + } + + foreach (var peerNodeId in _authenticatedPeers.Keys) + { + set.Add(peerNodeId); + } + + foreach (var peerNodeId in _directEndpoints.Keys) + { + set.Add(peerNodeId); + } + + return set.ToArray(); + } + +} + +internal readonly record struct PendingHelloProbe( + NodeId PeerNodeId, + int LocalSocketId, + IPEndPoint SendTo, + IPEndPoint? PhysicalDestination, + long SentAtMs); + +internal readonly record struct PendingHelloProbeSet(PendingHelloProbe[] Probes) +{ + public long SentAtMs => Probes.Length == 0 ? 0 : Probes.Min(static probe => probe.SentAtMs); + + public bool TryTakeBest(NodeId peerNodeId, int receivedLocalSocketId, IPEndPoint receivedVia, out PendingHelloProbe probe) + { + for (var i = 0; i < Probes.Length; i++) + { + if (Probes[i].PeerNodeId == peerNodeId && Probes[i].LocalSocketId == receivedLocalSocketId) + { + probe = Probes[i]; + return true; + } + } + + for (var i = 0; i < Probes.Length; i++) + { + if (Probes[i].PeerNodeId != peerNodeId) + { + continue; + } + + if (Probes[i].PhysicalDestination is { } physicalDestination && physicalDestination.Equals(receivedVia)) + { + probe = Probes[i]; + return true; + } + + if (Probes[i].SendTo.Equals(receivedVia)) + { + probe = Probes[i]; + return true; + } + } + + for (var i = 0; i < Probes.Length; i++) + { + if (Probes[i].PeerNodeId == peerNodeId) + { + probe = Probes[i]; + return true; + } + } + + probe = default; + return false; + } } + + + + + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs index a521fa8..1291d49 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRuntimeFactory.cs @@ -28,12 +28,37 @@ internal static class ZeroTierDataplaneRuntimeFactory ArgumentNullException.ThrowIfNull(inlineCom); ArgumentNullException.ThrowIfNull(multipath); + var requiredSurfaceObservations = Math.Max(udp.LocalSockets.Count, 1); ZeroTierHelloOk helloOk; byte[] rootKey; if (cachedRoot is { } cachedHelloOk && cachedRootKey is not null) { - helloOk = cachedHelloOk; - rootKey = cachedRootKey; + var needsSurfaceRefresh = multipath.Enabled && + cachedHelloOk.ExternalSurfaceObservations.Count < requiredSurfaceObservations; + + if (!needsSurfaceRefresh) + { + helloOk = cachedHelloOk; + rootKey = cachedRootKey; + } + else + { + try + { + helloOk = await ZeroTierHelloClient + .HelloRootsAsync(udp, localIdentity, planet, timeout: TimeSpan.FromSeconds(10), cancellationToken) + .ConfigureAwait(false); + + rootKey = ComputeRootKey(localIdentity, planet, helloOk.RootNodeId); + } +#pragma warning disable CA1031 // Best-effort refresh should fall back to the cached root state. + catch +#pragma warning restore CA1031 + { + helloOk = cachedHelloOk; + rootKey = cachedRootKey; + } + } } else { @@ -44,6 +69,36 @@ internal static class ZeroTierDataplaneRuntimeFactory rootKey = ComputeRootKey(localIdentity, planet, helloOk.RootNodeId); } + if (multipath.Enabled && helloOk.ExternalSurfaceObservations.Count < requiredSurfaceObservations) + { + var refreshedObservations = await ZeroTierHelloClient + .ProbeRootExternalSurfacesAsync( + udp, + localIdentity, + planet, + helloOk.RootNodeId, + helloOk.RootEndpoint, + rootKey, + helloOk.ExternalSurfaceObservations, + timeout: TimeSpan.FromSeconds(10), + cancellationToken) + .ConfigureAwait(false); + + if (refreshedObservations.Count != helloOk.ExternalSurfaceObservations.Count) + { + var primarySurface = refreshedObservations + .OrderBy(static observation => observation.LocalSocketId) + .Select(static observation => observation.SurfaceAddress) + .FirstOrDefault() ?? helloOk.ExternalSurfaceAddress; + + helloOk = helloOk with + { + ExternalSurfaceAddress = primarySurface, + ExternalSurfaceObservations = refreshedObservations + }; + } + } + var runtime = new ZeroTierDataplaneRuntime( udp, rootNodeId: helloOk.RootNodeId, @@ -55,7 +110,11 @@ internal static class ZeroTierDataplaneRuntimeFactory localManagedIpsV4: localManagedIpsV4, localManagedIpsV6: localManagedIpsV6, inlineCom: inlineCom, - multipath: multipath); + multipath: multipath, + planetId: planet.Id, + planetTimestamp: planet.Timestamp, + localExternalSurfaceAddress: helloOk.ExternalSurfaceAddress, + initialExternalSurfaceObservations: helloOk.ExternalSurfaceObservations); await TrySubscribeForAddressResolutionAsync( udp, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs index a5945c2..9640946 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDataplaneRxLoops.cs @@ -14,8 +14,10 @@ internal sealed class ZeroTierDataplaneRxLoops private readonly NodeId _localNodeId; private readonly ZeroTierDataplaneRootClient _rootClient; private readonly IZeroTierDataplanePeerDatagramProcessor _peerDatagrams; - private readonly Func, IPEndPoint, CancellationToken, ValueTask>? _handleRootControlAsync; + private readonly ZeroTierInboundDatagramDiagnostics _diagnostics; + private readonly Func, int, IPEndPoint, CancellationToken, ValueTask>? _handleRootControlAsync; private readonly Action? _onPeerQueueDrop; + private readonly bool _acceptDirectPeerDatagrams; private int _traceRxRemaining = 200; @@ -27,7 +29,9 @@ public ZeroTierDataplaneRxLoops( NodeId localNodeId, ZeroTierDataplaneRootClient rootClient, IZeroTierDataplanePeerDatagramProcessor peerDatagrams, - Func, IPEndPoint, CancellationToken, ValueTask>? handleRootControlAsync = null, + ZeroTierInboundDatagramDiagnostics? diagnostics = null, + bool acceptDirectPeerDatagrams = false, + Func, int, IPEndPoint, CancellationToken, ValueTask>? handleRootControlAsync = null, Action? onPeerQueueDrop = null) { ArgumentNullException.ThrowIfNull(udp); @@ -43,12 +47,16 @@ public ZeroTierDataplaneRxLoops( _localNodeId = localNodeId; _rootClient = rootClient; _peerDatagrams = peerDatagrams; + _diagnostics = diagnostics ?? new ZeroTierInboundDatagramDiagnostics(localNodeId, rootEndpoint, rawBudget: 0, dropBudget: 0); + _acceptDirectPeerDatagrams = acceptDirectPeerDatagrams; _handleRootControlAsync = handleRootControlAsync; _onPeerQueueDrop = onPeerQueueDrop; } - public async Task DispatcherLoopAsync(ChannelWriter peerWriter, CancellationToken cancellationToken) + public async Task DispatcherLoopAsync(Channel peerQueue, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(peerQueue); + var peerWriter = peerQueue.Writer; while (!cancellationToken.IsCancellationRequested) { ZeroTierUdpDatagram datagram; @@ -69,17 +77,40 @@ public async Task DispatcherLoopAsync(ChannelWriter peerWri return; } + _diagnostics.TraceInboundRaw(datagram); + + if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) + { + var peek = datagram.Payload.AsSpan(); + if (peek.Length < ZeroTierPacketHeader.Length) + { + continue; + } + + var source = new NodeId( + ZeroTierBinaryPrimitives.ReadUInt40BigEndian( + peek.Slice(ZeroTierPacketHeader.IndexSource, 5))); + if (source != _rootNodeId) + { + continue; + } + } + var packetBytes = datagram.Payload; if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var decoded)) { + _diagnostics.TraceDecodeFailure(datagram); continue; } if (decoded.Header.Destination != _localNodeId) { + _diagnostics.TraceDestinationMismatch(datagram, decoded); continue; } + _diagnostics.TraceDecodedInbound(datagram, decoded); + if (ZeroTierTrace.Enabled && _traceRxRemaining > 0) { _traceRxRemaining--; @@ -116,7 +147,7 @@ public async Task DispatcherLoopAsync(ChannelWriter peerWri { try { - await _handleRootControlAsync(verb, payload, datagram.RemoteEndPoint, cancellationToken).ConfigureAwait(false); + await _handleRootControlAsync(verb, payload, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -132,6 +163,11 @@ public async Task DispatcherLoopAsync(ChannelWriter peerWri continue; } + if (!_acceptDirectPeerDatagrams && !datagram.RemoteEndPoint.Equals(_rootEndpoint)) + { + continue; + } + if (!peerWriter.TryWrite(datagram)) { if (cancellationToken.IsCancellationRequested) @@ -139,7 +175,20 @@ public async Task DispatcherLoopAsync(ChannelWriter peerWri return; } + if (peerWriter.TryWrite(datagram)) + { + continue; + } + _onPeerQueueDrop?.Invoke(); + if (peerQueue.Reader.TryRead(out _)) + { + if (peerWriter.TryWrite(datagram)) + { + continue; + } + } + continue; } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs new file mode 100644 index 0000000..a17e93c --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectBootstrapControlPolicy.cs @@ -0,0 +1,102 @@ +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierDirectBootstrapControlPolicy +{ + public static bool ShouldSuppressRelayedRootMaintenanceWhilePinnedRendezvousUnresolved( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints) + => !allowRootRelayFallback && + !hasConfirmedDirectPath && + hasPinnedRendezvousEndpoints; + + public static bool ShouldRelayRootControlOnRendezvous( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath) + => allowRootRelayFallback || hasConfirmedDirectPath; + + public static bool ShouldSuppressRelayedRootControlDuringPinnedRendezvousBootstrap( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousRelayQuietMs) + { + if (allowRootRelayFallback || + hasConfirmedDirectPath || + !hasPinnedRendezvousEndpoints || + !canUseEchoForDirectBootstrap || + !pinnedRendezvousObservedAtMs.HasValue) + { + return false; + } + + return unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousRelayQuietMs; + } + + public static bool ShouldDelayPinnedRendezvousHintProbeUntilPeerVersionKnown( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool canUseEchoForDirectBootstrap, + long nowMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousVersionGraceMs) + { + if (allowRootRelayFallback || + hasConfirmedDirectPath || + !hasPinnedRendezvousEndpoints || + canUseEchoForDirectBootstrap || + !pinnedRendezvousObservedAtMs.HasValue) + { + return false; + } + + return unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousVersionGraceMs; + } + + public static bool ShouldRefreshRelayedRootControl( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath) + => allowRootRelayFallback || !hasConfirmedDirectPath; + + public static bool ShouldDelayDirectOnlyHintedControlUntilRendezvous( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + bool endpointIsPinnedRendezvous) + { + if (allowRootRelayFallback || hasConfirmedDirectPath || endpointIsPinnedRendezvous) + { + return false; + } + + return !hasPinnedRendezvousEndpoints; + } + + public static bool ShouldWaitForDirectOnlyHintedPayload( + bool allowRootRelayFallback, + bool hasConfirmedDirectPath, + bool hasPinnedRendezvousEndpoints, + long nowMs, + long? bootstrapStartedAtMs, + long directOnlyRendezvousGraceMs, + long? pinnedRendezvousObservedAtMs, + long pinnedRendezvousSettleMs) + { + if (allowRootRelayFallback || hasConfirmedDirectPath) + { + return false; + } + + if (!hasPinnedRendezvousEndpoints) + { + return true; + } + + return pinnedRendezvousObservedAtMs.HasValue && + unchecked(nowMs - pinnedRendezvousObservedAtMs.Value) < pinnedRendezvousSettleMs; + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointKey.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointKey.cs new file mode 100644 index 0000000..8b6cf96 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointKey.cs @@ -0,0 +1,19 @@ +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierDirectEndpointKey +{ + public static string Format(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var address = endpoint.Address; + if (address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + return $"{address}:{endpoint.Port}"; + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs index b8a07a1..8ef5d93 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointManager.cs @@ -1,242 +1,496 @@ -using System.Net; -using System.Collections.Concurrent; -using System.Net.Sockets; -using System.Security.Cryptography; -using System.Linq; -using ZTSharp.ZeroTier.Protocol; -using ZTSharp.ZeroTier.Transport; - -namespace ZTSharp.ZeroTier.Internal; - +using System.Net; +using System.Collections.Concurrent; +using System.Globalization; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Linq; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + internal sealed class ZeroTierDirectEndpointManager { - private const int MaxEndpoints = 8; - private const long HolePunchMinIntervalMs = 5_000; + private const int MaxEndpoints = 64; + private const int MaxEndpointsPerScopeAndFamily = 8; + private const int RendezvousHolePunchHopLimit = 2; private const long PushDirectPathsCutoffTimeMs = 30_000; private const int PushDirectPathsCutoffLimit = 8; - - private const byte PushDirectPathsFlagForgetPath = 0x01; - private const byte PushDirectPathsFlagClusterRedirect = 0x02; - - private readonly IZeroTierUdpTransport _udp; - private readonly IPEndPoint _relayEndpoint; - private readonly NodeId _remoteNodeId; + + private const byte PushDirectPathsFlagForgetPath = 0x01; + private const byte PushDirectPathsFlagClusterRedirect = 0x02; + + private readonly IZeroTierUdpTransport _udp; + private readonly IPEndPoint _relayEndpoint; + private readonly NodeId _remoteNodeId; + private readonly Func? _handleDirectEndpointHintAsync; + private readonly Func? _shouldAcceptEndpoint; private readonly object _lock = new(); + private readonly ZeroTierDirectHolePunchLimiter _holePunchLimiter = new(); private IPEndPoint[] _directEndpoints = Array.Empty(); - private readonly ConcurrentDictionary _holePunchLastSentMs = new(StringComparer.Ordinal); + private readonly Dictionary> _preferredLocalSocketsByEndpoint = new(StringComparer.Ordinal); + private readonly HashSet _pinnedRendezvousEndpointKeys = new(StringComparer.Ordinal); private long _lastDirectPathPushReceiveMs; private int _directPathPushCutoffCount; - - public ZeroTierDirectEndpointManager(IZeroTierUdpTransport udp, IPEndPoint relayEndpoint, NodeId remoteNodeId) - { - ArgumentNullException.ThrowIfNull(udp); - ArgumentNullException.ThrowIfNull(relayEndpoint); - - _udp = udp; - _relayEndpoint = relayEndpoint; - _remoteNodeId = remoteNodeId; - } - + + public ZeroTierDirectEndpointManager( + IZeroTierUdpTransport udp, + IPEndPoint relayEndpoint, + NodeId remoteNodeId, + Func? handleDirectEndpointHintAsync = null, + Func? shouldAcceptEndpoint = null) + { + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(relayEndpoint); + + _udp = udp; + _relayEndpoint = relayEndpoint; + _remoteNodeId = remoteNodeId; + _handleDirectEndpointHintAsync = handleDirectEndpointHintAsync; + _shouldAcceptEndpoint = shouldAcceptEndpoint; + } + public IPEndPoint[] Endpoints => _directEndpoints; - public ValueTask HandleRendezvousFromRootAsync(ReadOnlyMemory payload, IPEndPoint receivedVia, CancellationToken cancellationToken) + public bool HasPinnedRendezvousEndpoints { - cancellationToken.ThrowIfCancellationRequested(); - - if (ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) && rendezvous.With == _remoteNodeId) + get { - var endpoints = ZeroTierDirectEndpointSelection.Normalize([rendezvous.Endpoint], _relayEndpoint, maxEndpoints: MaxEndpoints); - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); - } - lock (_lock) { - _directEndpoints = endpoints; + return _pinnedRendezvousEndpointKeys.Count != 0; } + } + } - foreach (var endpoint in endpoints) + public int[] GetPreferredLocalSocketIds(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + int[] preferredSocketIds; + lock (_lock) + { + if (!_preferredLocalSocketsByEndpoint.TryGetValue(FormatEndpointKey(endpoint), out var socketIds) || socketIds.Count == 0) { - TrySendHolePunch(endpoint); - } + return Array.Empty(); + } + + preferredSocketIds = socketIds.ToArray(); + } + + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return preferredSocketIds.Contains(0) ? new[] { 0 } : Array.Empty(); + } + + var available = localSockets.Select(static socket => socket.Id).ToHashSet(); + return preferredSocketIds + .Where(available.Contains) + .Distinct() + .ToArray(); + } - return ValueTask.CompletedTask; - } + public bool IsPinnedRendezvousEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); - ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS (ignored) via {receivedVia}."); - return ValueTask.CompletedTask; + lock (_lock) + { + return _pinnedRendezvousEndpointKeys.Contains(FormatEndpointKey(endpoint)); + } } - public ValueTask HandlePushDirectPathsFromRemoteAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + public void RefreshPinnedRendezvousHolePunch(IPEndPoint endpoint, int[] preferredLocalSocketIds) { - cancellationToken.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentNullException.ThrowIfNull(preferredLocalSocketIds); - if (!ZeroTierPushDirectPathsCodec.TryParse(payload.Span, out var paths) || paths.Length == 0) + if (!IsPinnedRendezvousEndpoint(endpoint)) { - ZeroTierTrace.WriteLine("[zerotier] Drop: failed to parse PUSH_DIRECT_PATHS payload."); - return ValueTask.CompletedTask; + return; } - var now = Environment.TickCount64; - if (!RateGatePushDirectPaths(now)) - { - if (ZeroTierTrace.Enabled) + TrySendHolePunch( + endpoint, + preferredLocalSocketIds, + hopLimit: RendezvousHolePunchHopLimit); + } + + public async ValueTask HandleRendezvousFromRootAsync( + ReadOnlyMemory payload, + int receivedLocalSocketId, + IPEndPoint receivedVia, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (ZeroTierRendezvousCodec.TryParse(payload.Span, out var rendezvous) && rendezvous.With == _remoteNodeId) + { + var endpoints = ZeroTierDirectEndpointSelection.Normalize( + [rendezvous.Endpoint], + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS: {rendezvous.With} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} via {receivedVia}."); + } + + lock (_lock) { - ZeroTierTrace.WriteLine($"[zerotier] Drop: PUSH_DIRECT_PATHS rate-gated (peer={_remoteNodeId})."); + _directEndpoints = endpoints; + ReplacePinnedRendezvousEndpoints_NoLock(endpoints); + RememberPreferredLocalSockets_NoLock(endpoints, receivedLocalSocketId); } + + foreach (var endpoint in endpoints) + { + TrySendHolePunch( + endpoint, + preferredLocalSocketIds: new[] { receivedLocalSocketId }, + hopLimit: RendezvousHolePunchHopLimit); + if (_handleDirectEndpointHintAsync is not null) + { + await _handleDirectEndpointHintAsync( + _remoteNodeId, + receivedLocalSocketId, + endpoint, + false, + false, + cancellationToken) + .ConfigureAwait(false); + } + } + + return; + } + + ZeroTierTrace.WriteLine($"[zerotier] RX RENDEZVOUS (ignored) via {receivedVia}."); + return; + } - return ValueTask.CompletedTask; - } - + public async ValueTask HandlePushDirectPathsFromRemoteAsync( + ReadOnlyMemory payload, + int receivedLocalSocketId, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!ZeroTierPushDirectPathsCodec.TryParse(payload.Span, out var paths) || paths.Length == 0) + { + ZeroTierTrace.WriteLine("[zerotier] Drop: failed to parse PUSH_DIRECT_PATHS payload."); + return; + } + + var now = Environment.TickCount64; + if (!RateGatePushDirectPaths(now)) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Drop: PUSH_DIRECT_PATHS rate-gated (peer={_remoteNodeId})."); + } + + return; + } + var forget = new HashSet(StringComparer.Ordinal); var redirect = new List(); var add = new List(); - - for (var i = 0; i < paths.Length; i++) - { - var flags = paths[i].Flags; - var endpoint = paths[i].Endpoint; - var key = FormatEndpointKey(endpoint); - - if ((flags & PushDirectPathsFlagForgetPath) != 0) - { - forget.Add(key); - continue; - } - + var preferredSocketEndpoints = new List(); + var forceFullHelloByEndpoint = new Dictionary(StringComparer.Ordinal); + var useAllEligibleLocalSocketsByEndpoint = new Dictionary(StringComparer.Ordinal); + + for (var i = 0; i < paths.Length; i++) + { + var flags = paths[i].Flags; + var endpoint = paths[i].Endpoint; + var key = FormatEndpointKey(endpoint); + + if ((flags & PushDirectPathsFlagForgetPath) != 0) + { + forget.Add(key); + forceFullHelloByEndpoint.Remove(key); + useAllEligibleLocalSocketsByEndpoint.Remove(key); + continue; + } + if ((flags & PushDirectPathsFlagClusterRedirect) != 0) { redirect.Add(endpoint); + preferredSocketEndpoints.Add(endpoint); + forceFullHelloByEndpoint[key] = true; + useAllEligibleLocalSocketsByEndpoint[key] = false; } else { add.Add(endpoint); + forceFullHelloByEndpoint.TryAdd(key, false); + useAllEligibleLocalSocketsByEndpoint.TryAdd(key, true); } } - + IPEndPoint[] endpoints; + IPEndPoint[] endpointsToProbe; lock (_lock) { - var merged = _directEndpoints - .Where(ep => !forget.Contains(FormatEndpointKey(ep))) + var incomingKeys = redirect + .Concat(add) + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); + var pinnedRendezvous = _directEndpoints.Where(endpoint => + _pinnedRendezvousEndpointKeys.Contains(FormatEndpointKey(endpoint)) && + !forget.Contains(FormatEndpointKey(endpoint))); + var merged = pinnedRendezvous .Concat(redirect) - .Concat(add); - - endpoints = ZeroTierDirectEndpointSelection.Normalize(merged, _relayEndpoint, maxEndpoints: MaxEndpoints); + .Concat(add) + .Concat(_directEndpoints.Where(ep => + !forget.Contains(FormatEndpointKey(ep)) && + !incomingKeys.Contains(FormatEndpointKey(ep)) && + !_pinnedRendezvousEndpointKeys.Contains(FormatEndpointKey(ep)))); + + endpoints = ZeroTierDirectEndpointSelection.Normalize( + merged, + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); + endpointsToProbe = endpoints + .Where(endpoint => forceFullHelloByEndpoint.ContainsKey(FormatEndpointKey(endpoint))) + .ToArray(); _directEndpoints = endpoints; + PrunePinnedRendezvousEndpoints_NoLock(endpoints); + PrunePreferredLocalSockets_NoLock(endpoints); + RememberPreferredLocalSockets_NoLock(preferredSocketEndpoints, receivedLocalSocketId); } - - if (endpoints.Length == 0) - { - return ValueTask.CompletedTask; - } - - if (ZeroTierTrace.Enabled) - { - ZeroTierTrace.WriteLine($"[zerotier] RX PUSH_DIRECT_PATHS: endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)} (candidates: {paths.Length})."); - } - - foreach (var endpoint in endpoints) - { - TrySendHolePunch(endpoint); - } - - return ValueTask.CompletedTask; - } - - private bool RateGatePushDirectPaths(long nowMs) - { - lock (_lock) + + if (endpoints.Length == 0 || endpointsToProbe.Length == 0) + { + return; + } + + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] RX PUSH_DIRECT_PATHS: {FormatPaths(paths)} endpoints: {ZeroTierDirectEndpointSelection.Format(endpoints)}."); + } + + foreach (var endpoint in endpointsToProbe) + { + if (_handleDirectEndpointHintAsync is not null) + { + var forceFullHello = forceFullHelloByEndpoint[FormatEndpointKey(endpoint)]; + var useAllEligibleLocalSockets = useAllEligibleLocalSocketsByEndpoint[FormatEndpointKey(endpoint)]; + await _handleDirectEndpointHintAsync( + _remoteNodeId, + receivedLocalSocketId, + endpoint, + forceFullHello, + useAllEligibleLocalSockets, + cancellationToken) + .ConfigureAwait(false); + } + } + + return; + } + + public void SeedEndpoints(IEnumerable endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + IPEndPoint[] normalized; + lock (_lock) + { + normalized = ZeroTierDirectEndpointSelection.Normalize( + _directEndpoints.Concat(endpoints), + _relayEndpoint, + maxEndpoints: MaxEndpoints, + _shouldAcceptEndpoint, + maxPerScopeAndFamily: MaxEndpointsPerScopeAndFamily); + _directEndpoints = normalized; + PrunePreferredLocalSockets_NoLock(normalized); + } + + if (ZeroTierTrace.Enabled && normalized.Length > 0) + { + ZeroTierTrace.WriteLine($"[zerotier] Seed direct endpoints for {_remoteNodeId}: {ZeroTierDirectEndpointSelection.Format(normalized)}."); + } + + foreach (var endpoint in normalized) + { + TrySendHolePunch(endpoint); + } + } + + private bool RateGatePushDirectPaths(long nowMs) + { + lock (_lock) + { + if (unchecked(nowMs - _lastDirectPathPushReceiveMs) <= PushDirectPathsCutoffTimeMs) + { + _directPathPushCutoffCount++; + } + else + { + _directPathPushCutoffCount = 0; + } + + _lastDirectPathPushReceiveMs = nowMs; + return _directPathPushCutoffCount < PushDirectPathsCutoffLimit; + } + } + + private static string FormatPaths(ZeroTierPushedDirectPath[] paths) + { + if (paths.Length == 0) + { + return ""; + } + + return string.Join(", ", paths.Select(static path => $"{FormatFlags(path.Flags)}{path.Endpoint}")); + } + + private static string FormatFlags(byte flags) + { + if (flags == 0) + { + return string.Empty; + } + + var parts = new List(2); + if ((flags & PushDirectPathsFlagForgetPath) != 0) + { + parts.Add("forget"); + } + + if ((flags & PushDirectPathsFlagClusterRedirect) != 0) + { + parts.Add("redirect"); + } + + return "[" + string.Join("|", parts) + "] "; + } + + private void TrySendHolePunch(IPEndPoint endpoint, int[]? preferredLocalSocketIds = null, int? hopLimit = null) + { + var localSockets = _udp.LocalSockets; + var now = Environment.TickCount64; + _holePunchLimiter.CleanupIfNeeded(now); + + var junk = new byte[4]; + RandomNumberGenerator.Fill(junk); + + if (preferredLocalSocketIds is { Length: > 0 }) + { + for (var i = 0; i < preferredLocalSocketIds.Length; i++) + { + var socketId = preferredLocalSocketIds[i]; + if (!_holePunchLimiter.ShouldSend(socketId, endpoint, now)) + { + continue; + } + + TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); + } + + return; + } + + if (localSockets.Count == 0) { - if (unchecked(nowMs - _lastDirectPathPushReceiveMs) <= PushDirectPathsCutoffTimeMs) + if (!_holePunchLimiter.ShouldSend(localSocketId: 0, endpoint, now)) { - _directPathPushCutoffCount++; + return; } - else + + TrySendHolePunchCore(localSocketId: 0, endpoint, junk, hopLimit); + return; + } + + for (var i = 0; i < localSockets.Count; i++) + { + var socketId = localSockets[i].Id; + if (!_holePunchLimiter.ShouldSend(socketId, endpoint, now)) { - _directPathPushCutoffCount = 0; + continue; } - - _lastDirectPathPushReceiveMs = nowMs; - return _directPathPushCutoffCount < PushDirectPathsCutoffLimit; - } + + TrySendHolePunchCore(socketId, endpoint, junk, hopLimit); + } + } + + private void TrySendHolePunchCore(int localSocketId, IPEndPoint endpoint, byte[] junk, int? hopLimit) + { + Task sendTask; + try + { + ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint} (socket={localSocketId}, hopLimit={(hopLimit?.ToString(CultureInfo.InvariantCulture) ?? "default")})."); + sendTask = hopLimit is int ttl + ? _udp.SendWithHopLimitAsync(localSocketId, endpoint, junk, ttl, CancellationToken.None) + : _udp.SendAsync(localSocketId, endpoint, junk, CancellationToken.None); + } + catch (Exception ex) when (ex is ObjectDisposedException or SocketException or OperationCanceledException) + { + return; + } + + _ = sendTask.ContinueWith( + static t => _ = t.Exception, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); } - private void TrySendHolePunch(IPEndPoint endpoint) + private void ReplacePinnedRendezvousEndpoints_NoLock(IEnumerable endpoints) { - if (!ShouldSendHolePunch(endpoint)) + _pinnedRendezvousEndpointKeys.Clear(); + foreach (var endpoint in endpoints) { - return; + _pinnedRendezvousEndpointKeys.Add(FormatEndpointKey(endpoint)); } + } - var junk = new byte[4]; - RandomNumberGenerator.Fill(junk); - - Task sendTask; - try + private void RememberPreferredLocalSockets_NoLock(IEnumerable endpoints, int localSocketId) + { + foreach (var endpoint in endpoints) + { + var key = FormatEndpointKey(endpoint); + if (!_preferredLocalSocketsByEndpoint.TryGetValue(key, out var socketIds)) + { + socketIds = new HashSet(); + _preferredLocalSocketsByEndpoint[key] = socketIds; + } + + socketIds.Add(localSocketId); + } + } + + private void PrunePreferredLocalSockets_NoLock(IEnumerable endpoints) + { + var keep = endpoints + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); + + foreach (var key in _preferredLocalSocketsByEndpoint.Keys.ToArray()) { - ZeroTierTrace.WriteLine($"[zerotier] TX hole-punch to {endpoint}."); - sendTask = _udp.SendAsync(endpoint, junk, CancellationToken.None); + if (!keep.Contains(key)) + { + _preferredLocalSocketsByEndpoint.Remove(key); + } } - catch (Exception ex) when (ex is ObjectDisposedException or SocketException or OperationCanceledException) - { - return; - } - - _ = sendTask.ContinueWith( - static t => _ = t.Exception, - CancellationToken.None, - TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, - TaskScheduler.Default); } - private bool ShouldSendHolePunch(IPEndPoint endpoint) + private void PrunePinnedRendezvousEndpoints_NoLock(IEnumerable endpoints) { - var now = Environment.TickCount64; - var keyAddress = endpoint.Address; - if (keyAddress.AddressFamily == AddressFamily.InterNetworkV6 && keyAddress.IsIPv4MappedToIPv6) - { - keyAddress = keyAddress.MapToIPv4(); - } - - var key = $"{keyAddress}:{endpoint.Port}"; - - while (true) - { - if (_holePunchLastSentMs.TryGetValue(key, out var lastSent) && - unchecked(now - lastSent) < HolePunchMinIntervalMs) - { - return false; - } - - if (_holePunchLastSentMs.TryAdd(key, now)) - { - return true; - } - - _holePunchLastSentMs.TryGetValue(key, out lastSent); - if (unchecked(now - lastSent) < HolePunchMinIntervalMs) - { - return false; - } - - if (_holePunchLastSentMs.TryUpdate(key, now, lastSent)) - { - return true; - } - } + var keep = endpoints + .Select(FormatEndpointKey) + .ToHashSet(StringComparer.Ordinal); + _pinnedRendezvousEndpointKeys.RemoveWhere(key => !keep.Contains(key)); } private static string FormatEndpointKey(IPEndPoint endpoint) - { - var address = endpoint.Address; - if (address.IsIPv4MappedToIPv6) - { - address = address.MapToIPv4(); - } - - return $"{address}:{endpoint.Port}"; - } + => ZeroTierDirectEndpointKey.Format(endpoint); } + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs new file mode 100644 index 0000000..2bc36d7 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointPolicy.cs @@ -0,0 +1,379 @@ +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectEndpointPolicy +{ + private static readonly string[] ExcludedAdapterNameTokens = + [ + "zerotier", + "tailscale", + "nord", + "wintun", + "wireguard", + "openvpn" + ]; + + private readonly LocalNetwork[] _localNetworks; + private readonly bool _hasLocalIpv4; + private readonly bool _hasLocalIpv6; + private readonly bool _hasLocalIpv4SharedSpace; + + public ZeroTierDirectEndpointPolicy() + : this(GetLocalNetworks()) + { + } + + internal ZeroTierDirectEndpointPolicy(IReadOnlyList localNetworks) + { + ArgumentNullException.ThrowIfNull(localNetworks); + _localNetworks = localNetworks.ToArray(); + for (var i = 0; i < _localNetworks.Length; i++) + { + if (_localNetworks[i].Address.AddressFamily == AddressFamily.InterNetwork) + { + _hasLocalIpv4 = true; + if (IsIpv4SharedSpace(_localNetworks[i].Address)) + { + _hasLocalIpv4SharedSpace = true; + } + } + else if (_localNetworks[i].Address.AddressFamily == AddressFamily.InterNetworkV6) + { + _hasLocalIpv6 = true; + } + } + } + + public bool ShouldAccept(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + endpoint = Canonicalize(endpoint); + + if (IPAddress.IsLoopback(endpoint.Address)) + { + return true; + } + + if (!ZeroTierDirectEndpointSelection.IsUsablePathEndpoint(endpoint)) + { + return false; + } + + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + return ShouldAcceptPublicFamily(endpoint.AddressFamily); + } + + if (endpoint.Address.AddressFamily == AddressFamily.InterNetwork && + IsIpv4SharedSpace(endpoint.Address) && + _hasLocalIpv4SharedSpace) + { + return true; + } + + for (var i = 0; i < _localNetworks.Length; i++) + { + if (_localNetworks[i].Contains(endpoint.Address)) + { + return true; + } + } + + return false; + } + + public bool ShouldAdvertiseLocalEndpoint(IPEndPoint localEndpoint, IReadOnlyList peerEndpoints) + { + ArgumentNullException.ThrowIfNull(localEndpoint); + ArgumentNullException.ThrowIfNull(peerEndpoints); + + localEndpoint = Canonicalize(localEndpoint); + if (!ShouldAccept(localEndpoint)) + { + return false; + } + + if (ZeroTierDirectEndpointSelection.IsPublicEndpoint(localEndpoint)) + { + return true; + } + + for (var i = 0; i < peerEndpoints.Count; i++) + { + var peerEndpoint = Canonicalize(peerEndpoints[i]); + if (!ShouldAccept(peerEndpoint)) + { + continue; + } + + if (SharesLocalNetwork(localEndpoint.Address, peerEndpoint.Address)) + { + return true; + } + } + + return false; + } + + private bool ShouldAcceptPublicFamily(AddressFamily family) + { + if (_localNetworks.Length == 0) + { + return true; + } + + return family switch + { + AddressFamily.InterNetwork => _hasLocalIpv4, + AddressFamily.InterNetworkV6 => _hasLocalIpv6, + _ => false + }; + } + + private bool SharesLocalNetwork(IPAddress localAddress, IPAddress peerAddress) + { + if (localAddress.AddressFamily != peerAddress.AddressFamily) + { + return false; + } + + if (localAddress.AddressFamily == AddressFamily.InterNetwork && + IsIpv4SharedSpace(localAddress) && + IsIpv4SharedSpace(peerAddress) && + _hasLocalIpv4SharedSpace) + { + return true; + } + + for (var i = 0; i < _localNetworks.Length; i++) + { + var network = _localNetworks[i]; + if (!network.Contains(localAddress)) + { + continue; + } + + if (network.Contains(peerAddress)) + { + return true; + } + } + + return false; + } + + private static LocalNetwork[] GetLocalNetworks() + { + var networks = new List(); + var interfaces = NetworkInterface.GetAllNetworkInterfaces(); + for (var i = 0; i < interfaces.Length; i++) + { + var nic = interfaces[i]; + if (!ShouldUseInterface(nic)) + { + continue; + } + + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + continue; + } + + if (!HasDefaultGateway(properties)) + { + continue; + } + + foreach (var unicast in properties.UnicastAddresses) + { + var address = Canonicalize(unicast.Address); + if (address is null || !ShouldUseAddress(address)) + { + continue; + } + + var prefixLength = GetPrefixLength(unicast); + if (prefixLength < 0) + { + continue; + } + + networks.Add(new LocalNetwork(address, prefixLength)); + } + } + + return networks + .Distinct() + .ToArray(); + } + + private static bool ShouldUseInterface(NetworkInterface nic) + { + if (nic.OperationalStatus != OperationalStatus.Up) + { + return false; + } + + if (nic.NetworkInterfaceType is NetworkInterfaceType.Loopback or NetworkInterfaceType.Tunnel) + { + return false; + } + + var name = (nic.Name + " " + nic.Description).ToUpperInvariant(); + for (var i = 0; i < ExcludedAdapterNameTokens.Length; i++) + { + if (name.Contains(ExcludedAdapterNameTokens[i].ToUpperInvariant(), StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool HasDefaultGateway(IPInterfaceProperties properties) + { + foreach (var gateway in properties.GatewayAddresses) + { + var address = Canonicalize(gateway.Address); + if (address is null || IPAddress.IsLoopback(address)) + { + continue; + } + + if (address.AddressFamily == AddressFamily.InterNetwork && address.Equals(IPAddress.Any)) + { + continue; + } + + if (address.AddressFamily == AddressFamily.InterNetworkV6 && + (address.Equals(IPAddress.IPv6Any) || address.IsIPv6LinkLocal || address.IsIPv6SiteLocal)) + { + continue; + } + + return true; + } + + return false; + } + + private static bool ShouldUseAddress(IPAddress address) + { + if (IPAddress.IsLoopback(address)) + { + return false; + } + + return address.AddressFamily switch + { + AddressFamily.InterNetwork => !address.Equals(IPAddress.Any) && !IsIpv4LinkLocal(address), + AddressFamily.InterNetworkV6 => !address.Equals(IPAddress.IPv6Any) && + !address.IsIPv6LinkLocal && + !address.IsIPv6SiteLocal, + _ => false + }; + } + + private static int GetPrefixLength(UnicastIPAddressInformation unicast) + { + if (unicast.Address.AddressFamily == AddressFamily.InterNetwork) + { + if (unicast.IPv4Mask is null) + { + return -1; + } + + return CountPrefixBits(unicast.IPv4Mask.GetAddressBytes()); + } + + return unicast.PrefixLength; + } + + private static int CountPrefixBits(byte[] bytes) + { + var bits = 0; + for (var i = 0; i < bytes.Length; i++) + { + var current = bytes[i]; + for (var bit = 7; bit >= 0; bit--) + { + if ((current & (1 << bit)) == 0) + { + return bits; + } + + bits++; + } + } + + return bits; + } + + private static bool IsIpv4LinkLocal(IPAddress address) + { + var bytes = address.GetAddressBytes(); + return bytes is [169, 254, _, _]; + } + + private static bool IsIpv4SharedSpace(IPAddress address) + { + var bytes = address.GetAddressBytes(); + return bytes.Length == 4 && + bytes[0] == 100 && + bytes[1] >= 64 && + bytes[1] <= 127; + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => new(Canonicalize(endpoint.Address)!, endpoint.Port); + + private static IPAddress? Canonicalize(IPAddress? address) + => address is not null && + address.AddressFamily == AddressFamily.InterNetworkV6 && + address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; + + internal readonly record struct LocalNetwork(IPAddress Address, int PrefixLength) + { + public bool Contains(IPAddress address) + { + address = Canonicalize(address)!; + var networkAddress = Canonicalize(Address)!; + if (address.AddressFamily != networkAddress.AddressFamily) + { + return false; + } + + var addressBytes = address.GetAddressBytes(); + var networkBytes = networkAddress.GetAddressBytes(); + var fullBytes = PrefixLength / 8; + var remainingBits = PrefixLength % 8; + + for (var i = 0; i < fullBytes; i++) + { + if (addressBytes[i] != networkBytes[i]) + { + return false; + } + } + + if (remainingBits == 0) + { + return true; + } + + var mask = unchecked((byte)(0xFF << (8 - remainingBits))); + return (addressBytes[fullBytes] & mask) == (networkBytes[fullBytes] & mask); + } + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs index 3e41d17..1d9fbff 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectEndpointSelection.cs @@ -1,22 +1,27 @@ using System.Net; using System.Net.Sockets; using System.Globalization; +using System.Runtime.InteropServices; namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierDirectEndpointSelection { - public static IPEndPoint[] Normalize(IEnumerable endpoints, IPEndPoint relayEndpoint, int maxEndpoints) + public static IPEndPoint[] Normalize( + IEnumerable endpoints, + IPEndPoint relayEndpoint, + int maxEndpoints, + Func? shouldInclude = null, + int maxPerScopeAndFamily = int.MaxValue) { ArgumentNullException.ThrowIfNull(endpoints); ArgumentNullException.ThrowIfNull(relayEndpoint); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxEndpoints); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxPerScopeAndFamily); relayEndpoint = Canonicalize(relayEndpoint); - var publicV4 = new List(); - var publicV6 = new List(); - var privateV4 = new List(); - var privateV6 = new List(); + var buckets = new Dictionary<(EndpointScope Scope, AddressFamily Family), List>(); foreach (var endpoint in endpoints) { @@ -31,46 +36,55 @@ public static IPEndPoint[] Normalize(IEnumerable endpoints, IPEndPoi continue; } - if (canonical.Equals(relayEndpoint)) + if (shouldInclude is not null && !shouldInclude(canonical)) { continue; } - if (canonical.Address.Equals(IPAddress.Any) || canonical.Address.Equals(IPAddress.IPv6Any)) + if (canonical.Equals(relayEndpoint)) { continue; } - var isPublic = IsPublicAddress(canonical.Address); - if (canonical.AddressFamily == AddressFamily.InterNetwork) + if (canonical.Address.Equals(IPAddress.Any) || canonical.Address.Equals(IPAddress.IPv6Any)) { - (isPublic ? publicV4 : privateV4).Add(canonical); + continue; } - else if (canonical.AddressFamily == AddressFamily.InterNetworkV6) + + var scope = GetScope(canonical.Address); + var acceptsLoopback = scope == EndpointScope.Loopback && shouldInclude is not null; + if (!acceptsLoopback && scope is not (EndpointScope.Global or EndpointScope.Shared or EndpointScope.Pseudoprivate or EndpointScope.Private)) { - (isPublic ? publicV6 : privateV6).Add(canonical); + continue; } - } - var ordered = publicV4 - .Concat(publicV6) - .Concat(privateV4) - .Concat(privateV6); + var key = (scope, canonical.AddressFamily); + ref var bucket = ref CollectionsMarshal.GetValueRefOrAddDefault(buckets, key, out _); + bucket ??= new List(); + bucket.Add(canonical); + } var unique = new List(); var seen = new HashSet(StringComparer.Ordinal); - foreach (var endpoint in ordered) + foreach (var scope in ScopePriority) { - var key = endpoint.Address + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); - if (!seen.Add(key)) + if (unique.Count >= maxEndpoints) { - continue; + break; } - unique.Add(endpoint); - if (unique.Count >= maxEndpoints) + for (var i = 0; i < FamilyPriority.Length; i++) { - break; + if (!buckets.TryGetValue((scope, FamilyPriority[i]), out var bucket)) + { + continue; + } + + AddUniqueEndpoints(bucket, unique, seen, maxEndpoints, maxPerScopeAndFamily); + if (unique.Count >= maxEndpoints) + { + break; + } } } @@ -97,7 +111,55 @@ private static IPEndPoint Canonicalize(IPEndPoint endpoint) return endpoint; } + private static void AddUniqueEndpoints( + List bucket, + List unique, + HashSet seen, + int maxEndpoints, + int maxPerScopeAndFamily) + { + var kept = 0; + for (var i = 0; i < bucket.Count && unique.Count < maxEndpoints && kept < maxPerScopeAndFamily; i++) + { + var endpoint = bucket[i]; + var key = endpoint.Address + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); + if (!seen.Add(key)) + { + continue; + } + + unique.Add(endpoint); + kept++; + } + } + + public static bool IsPublicEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + return IsPublicAddress(Canonicalize(endpoint).Address); + } + + public static bool IsPrivateEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + return !IsPublicEndpoint(endpoint); + } + + public static bool IsUsablePathEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + endpoint = Canonicalize(endpoint); + return GetScope(endpoint.Address) is + EndpointScope.Private or + EndpointScope.Shared or + EndpointScope.Pseudoprivate or + EndpointScope.Global; + } + private static bool IsPublicAddress(IPAddress address) + => GetScope(address) == EndpointScope.Global; + + private static EndpointScope GetScope(IPAddress address) { if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) { @@ -106,7 +168,7 @@ private static bool IsPublicAddress(IPAddress address) if (IPAddress.IsLoopback(address)) { - return false; + return EndpointScope.Loopback; } if (address.AddressFamily == AddressFamily.InterNetwork) @@ -114,47 +176,34 @@ private static bool IsPublicAddress(IPAddress address) var bytes = address.GetAddressBytes(); if (bytes.Length != 4) { - return false; - } - - if (bytes[0] == 10) - { - return false; - } - - if (bytes[0] == 172 && bytes[1] is >= 16 and <= 31) - { - return false; - } - - if (bytes[0] == 192 && bytes[1] == 168) - { - return false; + return EndpointScope.None; } - if (bytes[0] == 169 && bytes[1] == 254) + return bytes[0] switch { - return false; - } - - if (bytes[0] == 100 && bytes[1] is >= 64 and <= 127) - { - return false; - } - - if (bytes[0] == 0 || bytes[0] >= 224) - { - return false; - } - - return true; + 0 => EndpointScope.None, + 6 or 11 or 21 or 22 or 25 or 26 or 28 or 29 or 30 or 51 or 55 or 56 => EndpointScope.Pseudoprivate, + 10 => EndpointScope.Private, + 100 when bytes[1] is >= 64 and <= 127 => EndpointScope.Shared, + 127 => EndpointScope.Loopback, + 169 when bytes[1] == 254 => EndpointScope.LinkLocal, + 172 when bytes[1] is >= 16 and <= 31 => EndpointScope.Private, + 192 when bytes[1] == 168 => EndpointScope.Private, + 192 when bytes[1] == 0 && bytes[2] == 2 => EndpointScope.Private, + 198 when bytes[1] is 18 or 19 => EndpointScope.Private, + 198 when bytes[1] == 51 && bytes[2] == 100 => EndpointScope.Private, + 203 when bytes[1] == 0 && bytes[2] == 113 => EndpointScope.Private, + >= 224 and <= 239 => EndpointScope.Multicast, + >= 240 => EndpointScope.Pseudoprivate, + _ => EndpointScope.Global + }; } if (address.AddressFamily == AddressFamily.InterNetworkV6) { if (address.Equals(IPAddress.IPv6Any) || address.Equals(IPAddress.IPv6None)) { - return false; + return EndpointScope.None; } if (address.IsIPv6LinkLocal || @@ -162,25 +211,63 @@ private static bool IsPublicAddress(IPAddress address) address.IsIPv6SiteLocal || address.Equals(IPAddress.IPv6Loopback)) { - return false; + return address.IsIPv6Multicast + ? EndpointScope.Multicast + : address.Equals(IPAddress.IPv6Loopback) + ? EndpointScope.Loopback + : EndpointScope.LinkLocal; } var bytes = address.GetAddressBytes(); if (bytes.Length != 16) { - return false; + return EndpointScope.None; } - // fc00::/7 Unique Local Address (ULA) if ((bytes[0] & 0xFE) == 0xFC) { - return false; + return EndpointScope.Private; } - return true; + if (bytes[0] == 0x20 && + bytes[1] == 0x01 && + bytes[2] == 0x04 && + bytes[3] == 0x70) + { + return EndpointScope.None; + } + + return EndpointScope.Global; } - return false; + return EndpointScope.None; } + + private enum EndpointScope + { + None, + Loopback, + LinkLocal, + Private, + Shared, + Pseudoprivate, + Global, + Multicast + } + + private static readonly EndpointScope[] ScopePriority = + [ + EndpointScope.Global, + EndpointScope.Shared, + EndpointScope.Pseudoprivate, + EndpointScope.Private, + EndpointScope.Loopback + ]; + + private static readonly AddressFamily[] FamilyPriority = + [ + AddressFamily.InterNetwork, + AddressFamily.InterNetworkV6 + ]; } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs new file mode 100644 index 0000000..a8f76d9 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHintPathPlanner.cs @@ -0,0 +1,315 @@ +using System.Collections.Concurrent; +using System.Net; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectHintPathPlanner +{ + private readonly IZeroTierUdpTransport _udp; + private readonly Func _getDirectEndpointManager; + private readonly ZeroTierDirectPathSocketAdmissibility? _socketAdmissibility; + private readonly ConcurrentDictionary _probeCursors = new(); + + public ZeroTierDirectHintPathPlanner( + IZeroTierUdpTransport udp, + Func getDirectEndpointManager, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility = null) + { + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(getDirectEndpointManager); + + _udp = udp; + _getDirectEndpointManager = getDirectEndpointManager; + _socketAdmissibility = socketAdmissibility; + } + + public IPEndPoint[] TakeNextHintedEndpoints(NodeId peerNodeId, IPEndPoint[] hinted, int budget) + => GetOrCreateCursor(peerNodeId).TakeNextEndpoints(hinted, budget); + + public int[] GetRotatingSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeFallbackLocalSockets) + { + var socketIds = GetSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets); + if (socketIds.Length <= 1) + { + return socketIds; + } + + var cursor = GetOrCreateCursor(peerNodeId); + var socketIndex = cursor.TakeNextSocketIndex(endpoint, socketIds.Length); + return new[] { socketIds[socketIndex] }; + } + + public int[] GetPreferredSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + { + endpoint = Canonicalize(endpoint); + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return Array.Empty(); + } + + var preferred = new List(localSockets.Count); + var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); + AddSocketIds(preferred, preferredFromHints, localSockets, endpoint, _socketAdmissibility); + return preferred.ToArray(); + } + + public int[] GetPreferredAndFallbackSocketIds(NodeId peerNodeId, IPEndPoint endpoint) + => GetSocketIds(peerNodeId, endpoint, includeFallbackLocalSockets: true); + + public ZeroTierSelectedPeerPath[] GetHintedCandidates(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths) + { + var hinted = _getDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length == 0) + { + return Array.Empty(); + } + + HashSet? observed = null; + if (observedPaths.Length > 0) + { + observed = observedPaths + .Select(static path => new ZeroTierPeerPhysicalPathKey(path.LocalSocketId, path.RemoteEndPoint)) + .ToHashSet(); + } + + var unique = new HashSet(); + var candidates = new List(hinted.Length); + for (var i = 0; i < hinted.Length; i++) + { + var localSocketIds = GetPreferredAndFallbackSocketIds(peerNodeId, hinted[i]); + for (var s = 0; s < localSocketIds.Length; s++) + { + var key = new ZeroTierPeerPhysicalPathKey(localSocketIds[s], hinted[i]); + if (observed is not null && observed.Contains(key)) + { + continue; + } + + if (!unique.Add(key)) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(localSocketIds[s], hinted[i])); + } + } + + return candidates.ToArray(); + } + + public ZeroTierSelectedPeerPath[] GetNextHintedCandidatesForMaintenance( + NodeId peerNodeId, + ZeroTierPeerPhysicalPath[] observedPaths, + int endpointBudget) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(endpointBudget); + + var hinted = _getDirectEndpointManager(peerNodeId).Endpoints; + if (hinted.Length == 0) + { + return Array.Empty(); + } + + HashSet? observed = null; + if (observedPaths.Length > 0) + { + observed = observedPaths + .Select(static path => new ZeroTierPeerPhysicalPathKey(path.LocalSocketId, path.RemoteEndPoint)) + .ToHashSet(); + } + + var selectedEndpoints = TakeNextHintedEndpoints(peerNodeId, hinted, endpointBudget); + var unique = new HashSet(); + var candidates = new List(selectedEndpoints.Length); + for (var i = 0; i < selectedEndpoints.Length; i++) + { + var localSocketIds = GetRotatingSocketIds( + peerNodeId, + selectedEndpoints[i], + includeFallbackLocalSockets: true); + for (var s = 0; s < localSocketIds.Length; s++) + { + var key = new ZeroTierPeerPhysicalPathKey(localSocketIds[s], selectedEndpoints[i]); + if (observed is not null && observed.Contains(key)) + { + continue; + } + + if (!unique.Add(key)) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(localSocketIds[s], selectedEndpoints[i])); + } + } + + return candidates.ToArray(); + } + + public void RemovePeer(NodeId peerNodeId) + { + _probeCursors.TryRemove(peerNodeId, out _); + } + + private int[] GetSocketIds(NodeId peerNodeId, IPEndPoint endpoint, bool includeFallbackLocalSockets) + { + endpoint = Canonicalize(endpoint); + var localSockets = _udp.LocalSockets; + if (localSockets.Count == 0) + { + return new[] { 0 }; + } + + var preferred = new List(localSockets.Count); + var preferredFromHints = _getDirectEndpointManager(peerNodeId).GetPreferredLocalSocketIds(endpoint); + AddSocketIds(preferred, preferredFromHints, localSockets, endpoint, _socketAdmissibility); + + if (preferred.Count != 0) + { + return preferred.ToArray(); + } + + if (includeFallbackLocalSockets) + { + AddSocketIds(preferred, GetAdmissibleSocketIds(localSockets, endpoint, _socketAdmissibility), localSockets, endpoint, _socketAdmissibility); + } + + if (preferred.Count != 0) + { + return preferred.ToArray(); + } + + return GetAdmissibleSocketIds(localSockets, endpoint, _socketAdmissibility); + } + + private static int[] GetAdmissibleSocketIds( + IReadOnlyList localSockets, + IPEndPoint endpoint, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility) + => localSockets + .Where(socket => SocketCanReachEndpoint(localSockets, socket.Id, endpoint, socketAdmissibility)) + .Select(static socket => socket.Id) + .ToArray(); + + private static void AddSocketIds( + List target, + int[] candidates, + IReadOnlyList localSockets, + IPEndPoint endpoint, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility) + { + for (var i = 0; i < candidates.Length; i++) + { + var socketId = candidates[i]; + if (!SocketCanReachEndpoint(localSockets, socketId, endpoint, socketAdmissibility) || + target.Contains(socketId)) + { + continue; + } + + target.Add(socketId); + } + } + + private DirectBootstrapProbeCursor GetOrCreateCursor(NodeId peerNodeId) + => _probeCursors.GetOrAdd(peerNodeId, static _ => new DirectBootstrapProbeCursor()); + + private static bool SocketCanReachEndpoint( + IReadOnlyList localSockets, + int socketId, + IPEndPoint endpoint, + ZeroTierDirectPathSocketAdmissibility? socketAdmissibility) + { + for (var i = 0; i < localSockets.Count; i++) + { + if (localSockets[i].Id != socketId) + { + continue; + } + + if (socketAdmissibility is not null) + { + return socketAdmissibility.ShouldUsePath(localSockets[i], endpoint); + } + + return AddressFamiliesAreCompatible(localSockets[i].LocalEndpoint.Address, endpoint.Address); + } + + return false; + } + + private static bool AddressFamiliesAreCompatible(IPAddress localAddress, IPAddress remoteAddress) + { + remoteAddress = Canonicalize(remoteAddress); + localAddress = Canonicalize(localAddress); + + if (localAddress.AddressFamily == remoteAddress.AddressFamily) + { + return true; + } + + return localAddress.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && + localAddress.Equals(IPAddress.IPv6Any) && + remoteAddress.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork; + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && endpoint.Address.IsIPv4MappedToIPv6 + ? new IPEndPoint(endpoint.Address.MapToIPv4(), endpoint.Port) + : endpoint; + + private static IPAddress Canonicalize(IPAddress address) + => address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; +} + +internal sealed class DirectBootstrapProbeCursor +{ + private readonly object _lock = new(); + private int _nextEndpointIndex; + private readonly Dictionary _nextSocketIndexByEndpoint = new(StringComparer.Ordinal); + + public IPEndPoint[] TakeNextEndpoints(IPEndPoint[] endpoints, int budget) + { + ArgumentNullException.ThrowIfNull(endpoints); + + if (endpoints.Length == 0 || budget <= 0) + { + return Array.Empty(); + } + + lock (_lock) + { + var count = Math.Min(budget, endpoints.Length); + var selected = new IPEndPoint[count]; + for (var i = 0; i < count; i++) + { + selected[i] = endpoints[(_nextEndpointIndex + i) % endpoints.Length]; + } + + _nextEndpointIndex = (_nextEndpointIndex + count) % endpoints.Length; + return selected; + } + } + + public int TakeNextSocketIndex(IPEndPoint endpoint, int socketCount) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(socketCount); + + var key = endpoint.ToString(); + lock (_lock) + { + var index = _nextSocketIndexByEndpoint.TryGetValue(key, out var current) + ? current + : 0; + _nextSocketIndexByEndpoint[key] = (index + 1) % socketCount; + return index; + } + } +} + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectHolePunchLimiter.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHolePunchLimiter.cs new file mode 100644 index 0000000..0d02cc7 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectHolePunchLimiter.cs @@ -0,0 +1,95 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectHolePunchLimiter +{ + private const long MinIntervalMs = 5_000; + private const long CacheTtlMs = 60_000; + private const int CacheMaxEntries = 2048; + + private readonly ConcurrentDictionary _lastSentMs = new(StringComparer.Ordinal); + private long _lastCleanupMs; + + public bool ShouldSend(int localSocketId, IPEndPoint endpoint, long nowMs) + { + var key = $"{localSocketId}|{ZeroTierDirectEndpointKey.Format(Normalize(endpoint))}"; + while (true) + { + if (_lastSentMs.TryGetValue(key, out var lastSent) && + unchecked(nowMs - lastSent) < MinIntervalMs) + { + return false; + } + + if (_lastSentMs.TryAdd(key, nowMs)) + { + return true; + } + + _lastSentMs.TryGetValue(key, out lastSent); + if (unchecked(nowMs - lastSent) < MinIntervalMs) + { + return false; + } + + if (_lastSentMs.TryUpdate(key, nowMs, lastSent)) + { + return true; + } + } + } + + public void CleanupIfNeeded(long nowMs) + { + if (_lastSentMs.Count <= CacheMaxEntries) + { + return; + } + + var last = Volatile.Read(ref _lastCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 10_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastCleanupMs, nowMs, last) != last) + { + return; + } + + var cutoff = nowMs - CacheTtlMs; + foreach (var entry in _lastSentMs) + { + if (entry.Value <= cutoff) + { + _lastSentMs.TryRemove(entry.Key, out _); + } + } + + if (_lastSentMs.Count <= CacheMaxEntries) + { + return; + } + + var snapshot = _lastSentMs.ToArray(); + Array.Sort(snapshot, static (left, right) => left.Value.CompareTo(right.Value)); + for (var i = 0; i < snapshot.Length && _lastSentMs.Count > CacheMaxEntries; i++) + { + _lastSentMs.TryRemove(snapshot[i].Key, out _); + } + } + + private static IPEndPoint Normalize(IPEndPoint endpoint) + { + var address = endpoint.Address; + if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + return ReferenceEquals(address, endpoint.Address) ? endpoint : new IPEndPoint(address, endpoint.Port); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs new file mode 100644 index 0000000..6a37719 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectPathSocketAdmissibility.cs @@ -0,0 +1,68 @@ +using System.Net; +using System.Net.Sockets; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectPathSocketAdmissibility +{ + private readonly ZeroTierDirectEndpointPolicy _endpointPolicy; + private readonly ZeroTierDirectRouteResolver _routeResolver; + + public ZeroTierDirectPathSocketAdmissibility( + ZeroTierDirectEndpointPolicy endpointPolicy, + ZeroTierDirectRouteResolver routeResolver) + { + _endpointPolicy = endpointPolicy ?? throw new ArgumentNullException(nameof(endpointPolicy)); + _routeResolver = routeResolver ?? throw new ArgumentNullException(nameof(routeResolver)); + } + + public bool ShouldUsePath(ZeroTierUdpLocalSocket localSocket, IPEndPoint remoteEndpoint) + { + ArgumentNullException.ThrowIfNull(remoteEndpoint); + + remoteEndpoint = Canonicalize(remoteEndpoint); + var localAddress = Canonicalize(localSocket.LocalEndpoint.Address); + if (!AddressFamiliesAreCompatible(localAddress, remoteEndpoint.Address)) + { + return false; + } + + if (!_routeResolver.TryResolve(remoteEndpoint, out var routedLocalAddress)) + { + return _endpointPolicy.ShouldAccept(remoteEndpoint); + } + + routedLocalAddress = Canonicalize(routedLocalAddress); + if (!IsWildcard(localAddress) && !localAddress.Equals(routedLocalAddress)) + { + return false; + } + + return _endpointPolicy.ShouldAccept(remoteEndpoint) || + _endpointPolicy.ShouldAccept(new IPEndPoint(routedLocalAddress, remoteEndpoint.Port)); + } + + private static bool IsWildcard(IPAddress address) + => address.Equals(IPAddress.Any) || address.Equals(IPAddress.IPv6Any); + + private static bool AddressFamiliesAreCompatible(IPAddress localAddress, IPAddress remoteAddress) + { + if (localAddress.AddressFamily == remoteAddress.AddressFamily) + { + return true; + } + + return localAddress.AddressFamily == AddressFamily.InterNetworkV6 && + localAddress.Equals(IPAddress.IPv6Any) && + remoteAddress.AddressFamily == AddressFamily.InterNetwork; + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => new(Canonicalize(endpoint.Address), endpoint.Port); + + private static IPAddress Canonicalize(IPAddress address) + => address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierDirectRouteResolver.cs b/ZTSharp/ZeroTier/Internal/ZeroTierDirectRouteResolver.cs new file mode 100644 index 0000000..c8e0cd9 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierDirectRouteResolver.cs @@ -0,0 +1,71 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierDirectRouteResolver +{ + private static readonly TimeSpan CacheTtl = TimeSpan.FromSeconds(30); + + private readonly Func _resolveLocalAddress; + private readonly Func _nowMs; + private readonly ConcurrentDictionary _cache = new(StringComparer.Ordinal); + + public ZeroTierDirectRouteResolver( + Func? resolveLocalAddress = null, + Func? nowMs = null) + { + _resolveLocalAddress = resolveLocalAddress ?? ResolveLocalAddressCore; + _nowMs = nowMs ?? (() => Environment.TickCount64); + } + + public bool TryResolve(IPEndPoint remoteEndpoint, out IPAddress localAddress) + { + ArgumentNullException.ThrowIfNull(remoteEndpoint); + + remoteEndpoint = Canonicalize(remoteEndpoint); + var now = _nowMs(); + var key = remoteEndpoint.ToString(); + if (_cache.TryGetValue(key, out var cached) && unchecked(now - cached.ExpiresAtMs) < 0) + { + localAddress = cached.LocalAddress; + return true; + } + + var resolved = _resolveLocalAddress(remoteEndpoint); + if (resolved is null) + { + localAddress = IPAddress.None; + return false; + } + + localAddress = Canonicalize(resolved); + _cache[key] = new CacheEntry(localAddress, now + (long)CacheTtl.TotalMilliseconds); + return true; + } + + private static IPAddress? ResolveLocalAddressCore(IPEndPoint remoteEndpoint) + { + try + { + using var socket = new Socket(remoteEndpoint.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + socket.Connect(remoteEndpoint); + return ((IPEndPoint?)socket.LocalEndPoint)?.Address; + } + catch (Exception ex) when (ex is SocketException or NotSupportedException or InvalidOperationException) + { + return null; + } + } + + private static IPEndPoint Canonicalize(IPEndPoint endpoint) + => new(Canonicalize(endpoint.Address), endpoint.Port); + + private static IPAddress Canonicalize(IPAddress address) + => address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6 + ? address.MapToIPv4() + : address; + + private readonly record struct CacheEntry(IPAddress LocalAddress, long ExpiresAtMs); +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs b/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs index 25589a7..7c169be 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierExternalSurfaceAddressTracker.cs @@ -18,16 +18,26 @@ public ZeroTierExternalSurfaceAddressTracker(TimeSpan ttl, Func? nowUnixMs } _ttl = ttl; - _nowUnixMs = nowUnixMs ?? (() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } - public void Observe(NodeId reportingPeerNodeId, int localSocketId, IPEndPoint surfaceAddress) + public void Observe( + NodeId reportingPeerNodeId, + int localSocketId, + IPEndPoint reporterPhysicalAddress, + IPEndPoint surfaceAddress) { + ArgumentNullException.ThrowIfNull(reporterPhysicalAddress); ArgumentNullException.ThrowIfNull(surfaceAddress); var now = _nowUnixMs(); - var key = new ZeroTierExternalSurfaceKey(localSocketId, reportingPeerNodeId); - _entries[key] = new Entry(surfaceAddress, now); + var key = new ZeroTierExternalSurfaceKey( + localSocketId, + reportingPeerNodeId, + Normalize(reporterPhysicalAddress), + GetScope(surfaceAddress.Address)); + var stored = new IPEndPoint(surfaceAddress.Address, surfaceAddress.Port); + _entries[key] = new Entry(stored, now); CleanupIfNeeded(now); } @@ -41,7 +51,7 @@ public IPEndPoint[] GetSnapshot(int localSocketId) { if (key.LocalSocketId == localSocketId) { - list.Add(entry.SurfaceAddress); + list.Add(new IPEndPoint(entry.SurfaceAddress.Address, entry.SurfaceAddress.Port)); } } @@ -74,7 +84,63 @@ private void CleanupIfNeeded(long nowUnixMs) } private readonly record struct Entry(IPEndPoint SurfaceAddress, long LastSeenUnixMs); + + private static IPEndPoint Normalize(IPEndPoint endpoint) + => endpoint.Address.IsIPv4MappedToIPv6 + ? new IPEndPoint(endpoint.Address.MapToIPv4(), endpoint.Port) + : endpoint; + + private static ZeroTierExternalSurfaceScope GetScope(IPAddress address) + { + if (address.IsIPv4MappedToIPv6) + { + address = address.MapToIPv4(); + } + + if (IPAddress.IsLoopback(address)) + { + return ZeroTierExternalSurfaceScope.Loopback; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + var bytes = address.GetAddressBytes(); + if (bytes is [10, _, _, _] || + bytes is [172, >= 16 and <= 31, _, _] || + bytes is [192, 168, _, _]) + { + return ZeroTierExternalSurfaceScope.Private; + } + + return ZeroTierExternalSurfaceScope.Global; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + var bytes = address.GetAddressBytes(); + if ((bytes[0] & 0xFE) == 0xFC) + { + return ZeroTierExternalSurfaceScope.Private; + } + + return ZeroTierExternalSurfaceScope.Global; + } + + return ZeroTierExternalSurfaceScope.None; + } } -internal readonly record struct ZeroTierExternalSurfaceKey(int LocalSocketId, NodeId ReportingPeerNodeId); +internal readonly record struct ZeroTierExternalSurfaceKey( + int LocalSocketId, + NodeId ReportingPeerNodeId, + IPEndPoint ReporterPhysicalAddress, + ZeroTierExternalSurfaceScope Scope); + +internal enum ZeroTierExternalSurfaceScope +{ + None, + Loopback, + Private, + Global +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs index 6b2722a..84926da 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloClient.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Linq; using ZTSharp.ZeroTier.Protocol; using ZTSharp.ZeroTier.Transport; @@ -13,7 +14,12 @@ internal readonly record struct ZeroTierHelloOk( byte RemoteMajorVersion, byte RemoteMinorVersion, ushort RemoteRevision, - IPEndPoint? ExternalSurfaceAddress); + IPEndPoint? ExternalSurfaceAddress, + IReadOnlyList ExternalSurfaceObservations); + +internal readonly record struct ZeroTierExternalSurfaceObservation( + int LocalSocketId, + IPEndPoint SurfaceAddress); internal static class ZeroTierHelloClient { @@ -21,6 +27,7 @@ internal static class ZeroTierHelloClient internal const byte AdvertisedMajorVersion = 1; internal const byte AdvertisedMinorVersion = 12; internal const ushort AdvertisedRevision = 0; + private const int RootSurfaceProbeRoundReceiveWindowMs = 1_000; public static async Task HelloRootsAsync( IZeroTierUdpTransport udp, @@ -44,8 +51,11 @@ public static async Task HelloRootsAsync( var rootKeys = ZeroTierRootKeyDerivation.BuildRootKeys(localIdentity, planet); - var helloTimestamp = (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - var pending = new Dictionary(capacity: planet.Roots.Count); + var helloTimestamp = (ulong)Environment.TickCount64; + var pending = new Dictionary(capacity: planet.Roots.Count); + var localSocketIds = udp.LocalSockets.Count == 0 + ? new[] { 0 } + : udp.LocalSockets.Select(static socket => socket.Id).ToArray(); foreach (var root in planet.Roots) { @@ -56,28 +66,40 @@ public static async Task HelloRootsAsync( foreach (var endpoint in root.StableEndpoints) { - var packet = ZeroTierHelloPacketBuilder.BuildPacket( - localIdentity, - destination: root.Identity.NodeId, - physicalDestination: endpoint, - planet, - helloTimestamp, - key, - advertisedProtocolVersion: AdvertisedProtocolVersion, - advertisedMajorVersion: AdvertisedMajorVersion, - advertisedMinorVersion: AdvertisedMinorVersion, - advertisedRevision: AdvertisedRevision, - out var packetId); - - try + for (var i = 0; i < localSocketIds.Length; i++) { - await udp.SendAsync(endpoint, packet, cancellationToken).ConfigureAwait(false); - pending[packetId] = root.Identity.NodeId; - } - catch (System.Net.Sockets.SocketException) - { - // Some environments don't have IPv6 connectivity. Ignore send failures and wait for any - // reachable root to respond. + var localSocketId = localSocketIds[i]; + var packet = ZeroTierHelloPacketBuilder.BuildPacket( + localIdentity, + destination: root.Identity.NodeId, + physicalDestination: endpoint, + planet, + helloTimestamp, + key, + advertisedProtocolVersion: AdvertisedProtocolVersion, + advertisedMajorVersion: AdvertisedMajorVersion, + advertisedMinorVersion: AdvertisedMinorVersion, + advertisedRevision: AdvertisedRevision, + out var packetId); + + try + { + if (udp.LocalSockets.Count == 0) + { + await udp.SendAsync(endpoint, packet, cancellationToken).ConfigureAwait(false); + } + else + { + await udp.SendAsync(localSocketId, endpoint, packet, cancellationToken).ConfigureAwait(false); + } + + pending[packetId] = new PendingRootHello(root.Identity.NodeId, localSocketId); + } + catch (System.Net.Sockets.SocketException) + { + // Some environments don't have IPv6 connectivity. Ignore send failures and wait for any + // reachable root to respond. + } } } } @@ -93,6 +115,7 @@ public static async Task HelloRootsAsync( async ValueTask WaitForHelloOkAsync(CancellationToken token) { + var surfacesBySocket = new Dictionary(); while (true) { var datagram = await udp.ReceiveAsync(token).ConfigureAwait(false); @@ -113,18 +136,23 @@ async ValueTask WaitForHelloOkAsync(CancellationToken token) continue; } - if (!pending.TryGetValue(ok.InRePacketId, out var rootNodeId)) + if (!pending.TryGetValue(ok.InRePacketId, out var pendingHello)) { continue; } - if (rootNodeId != packet.Header.Source) + if (pendingHello.RootNodeId != packet.Header.Source) { continue; } - return new ZeroTierHelloOk( - RootNodeId: rootNodeId, + if (ok.ExternalSurfaceAddress is { } surface) + { + surfacesBySocket[pendingHello.LocalSocketId] = surface; + } + + var result = new ZeroTierHelloOk( + RootNodeId: pendingHello.RootNodeId, RootEndpoint: datagram.RemoteEndPoint, HelloPacketId: ok.InRePacketId, HelloTimestampEcho: ok.TimestampEcho, @@ -132,9 +160,238 @@ async ValueTask WaitForHelloOkAsync(CancellationToken token) RemoteMajorVersion: ok.RemoteMajorVersion, RemoteMinorVersion: ok.RemoteMinorVersion, RemoteRevision: ok.RemoteRevision, - ExternalSurfaceAddress: ok.ExternalSurfaceAddress); + ExternalSurfaceAddress: ok.ExternalSurfaceAddress, + ExternalSurfaceObservations: Array.Empty()); + + return await CollectAdditionalSurfaceReportsAsync(result, surfacesBySocket, token).ConfigureAwait(false); + } + } + + async ValueTask CollectAdditionalSurfaceReportsAsync( + ZeroTierHelloOk initial, + Dictionary surfacesBySocket, + CancellationToken token) + { + var outstandingSockets = localSocketIds.Length - surfacesBySocket.Count; + if (outstandingSockets <= 0) + { + return initial with + { + ExternalSurfaceObservations = surfacesBySocket + .Select(static pair => new ZeroTierExternalSurfaceObservation(pair.Key, pair.Value)) + .OrderBy(static pair => pair.LocalSocketId) + .ToArray() + }; + } + + var graceDeadline = Environment.TickCount64 + 400; + while (outstandingSockets > 0) + { + var remainingMs = unchecked(graceDeadline - Environment.TickCount64); + if (remainingMs <= 0) + { + break; + } + + ZeroTierUdpDatagram datagram; + try + { + datagram = await udp.ReceiveAsync(TimeSpan.FromMilliseconds(remainingMs), token).ConfigureAwait(false); + } + catch (TimeoutException) + { + break; + } + + var packetBytes = datagram.Payload; + if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var packet)) + { + continue; + } + + if (!rootKeys.TryGetValue(packet.Header.Source, out var key)) + { + continue; + } + + if (!ZeroTierHelloOkParser.TryParse(packetBytes, key, out var ok)) + { + continue; + } + + if (!pending.TryGetValue(ok.InRePacketId, out var pendingHello)) + { + continue; + } + + if (pendingHello.RootNodeId != packet.Header.Source || ok.ExternalSurfaceAddress is not { } surface) + { + continue; + } + + if (surfacesBySocket.TryAdd(pendingHello.LocalSocketId, surface)) + { + outstandingSockets--; + } + } + + return initial with + { + ExternalSurfaceObservations = surfacesBySocket + .Select(static pair => new ZeroTierExternalSurfaceObservation(pair.Key, pair.Value)) + .OrderBy(static pair => pair.LocalSocketId) + .ToArray() + }; + } + } + + public static async Task> ProbeRootExternalSurfacesAsync( + IZeroTierUdpTransport udp, + ZeroTierIdentity localIdentity, + ZeroTierWorld planet, + NodeId rootNodeId, + IPEndPoint rootEndpoint, + byte[] rootKey, + IReadOnlyList knownObservations, + TimeSpan timeout, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(udp); + ArgumentNullException.ThrowIfNull(localIdentity); + ArgumentNullException.ThrowIfNull(planet); + ArgumentNullException.ThrowIfNull(rootEndpoint); + ArgumentNullException.ThrowIfNull(rootKey); + ArgumentNullException.ThrowIfNull(knownObservations); + if (timeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(timeout), timeout, "Timeout must be positive."); + } + + var observations = knownObservations + .Where(static observation => observation.SurfaceAddress.Port != 0) + .GroupBy(static observation => observation.LocalSocketId) + .ToDictionary(static group => group.Key, static group => group.Last().SurfaceAddress); + + var localSocketIds = udp.LocalSockets.Count == 0 + ? new[] { 0 } + : udp.LocalSockets.Select(static socket => socket.Id).ToArray(); + var deadline = Environment.TickCount64 + (long)timeout.TotalMilliseconds; + var pending = new Dictionary(); + + while (observations.Count < localSocketIds.Length) + { + var remainingMs = unchecked(deadline - Environment.TickCount64); + if (remainingMs <= 0) + { + break; + } + + var sentAny = false; + for (var i = 0; i < localSocketIds.Length; i++) + { + var localSocketId = localSocketIds[i]; + if (observations.ContainsKey(localSocketId)) + { + continue; + } + + var packet = ZeroTierHelloPacketBuilder.BuildPacket( + localIdentity, + destination: rootNodeId, + physicalDestination: rootEndpoint, + planet, + timestamp: (ulong)Environment.TickCount64, + rootKey, + advertisedProtocolVersion: AdvertisedProtocolVersion, + advertisedMajorVersion: AdvertisedMajorVersion, + advertisedMinorVersion: AdvertisedMinorVersion, + advertisedRevision: AdvertisedRevision, + out var packetId); + + try + { + if (udp.LocalSockets.Count == 0) + { + await udp.SendAsync(rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + else + { + await udp.SendAsync(localSocketId, rootEndpoint, packet, cancellationToken).ConfigureAwait(false); + } + } + catch (System.Net.Sockets.SocketException) + { + continue; + } + + pending[packetId] = localSocketId; + sentAny = true; + } + + if (!sentAny) + { + break; + } + + var roundDeadline = Math.Min( + deadline, + Environment.TickCount64 + Math.Min((int)remainingMs, RootSurfaceProbeRoundReceiveWindowMs)); + + while (pending.Count > 0) + { + remainingMs = Math.Min( + unchecked(deadline - Environment.TickCount64), + unchecked(roundDeadline - Environment.TickCount64)); + if (remainingMs <= 0) + { + break; + } + + ZeroTierUdpDatagram datagram; + try + { + datagram = await udp.ReceiveAsync(TimeSpan.FromMilliseconds(remainingMs), cancellationToken).ConfigureAwait(false); + } + catch (TimeoutException) + { + break; + } + + var packetBytes = datagram.Payload; + if (!ZeroTierPacketCodec.TryDecode(packetBytes, out var decoded)) + { + continue; + } + + if (decoded.Header.Source != rootNodeId) + { + continue; + } + + if (!ZeroTierHelloOkParser.TryParse(packetBytes, rootKey, out var ok)) + { + continue; + } + + if (!pending.Remove(ok.InRePacketId, out var localSocketId) || ok.ExternalSurfaceAddress is not { } surface) + { + continue; + } + + observations[localSocketId] = surface; + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine($"[zerotier] Root surface probe: socket={localSocketId} surface={surface} via {datagram.RemoteEndPoint}."); + } + + break; } } + + return observations + .Select(static pair => new ZeroTierExternalSurfaceObservation(pair.Key, pair.Value)) + .OrderBy(static observation => observation.LocalSocketId) + .ToArray(); } public static async Task HelloAsync( @@ -162,7 +419,7 @@ public static async Task HelloAsync( throw new InvalidOperationException("Local identity must contain a private key."); } - var helloTimestamp = (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var helloTimestamp = (ulong)Environment.TickCount64; var packet = ZeroTierHelloPacketBuilder.BuildPacket( localIdentity, @@ -233,3 +490,7 @@ public static async Task HelloRootsAsync( } } + +internal readonly record struct PendingRootHello( + NodeId RootNodeId, + int LocalSocketId); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs index 51c2c13..342ae50 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloOkPacketBuilder.cs @@ -24,7 +24,7 @@ public static byte[] BuildPacket( ? 0 : ZeroTierInetAddressCodec.GetSerializedLength(externalSurfaceAddress); - var payload = new byte[1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength]; + var payload = new byte[1 + 8 + 8 + 1 + 1 + 1 + 2 + surfaceLength + 2]; payload[0] = (byte)ZeroTierVerb.Hello; BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(1, 8), inRePacketId); BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(9, 8), helloTimestampEcho); @@ -38,6 +38,9 @@ public static byte[] BuildPacket( _ = ZeroTierInetAddressCodec.Serialize(externalSurfaceAddress, payload.AsSpan(22)); } + // Upstream always appends a u16 world-update payload length, even when zero. + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(payload.Length - 2, 2), 0); + var header = new ZeroTierPacketHeader( PacketId: packetId, Destination: destination, diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs b/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs index 0cf8d74..7ad3749 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierHelloPacketBuilder.cs @@ -10,7 +10,7 @@ internal static class ZeroTierHelloPacketBuilder public static byte[] BuildPacket( ZeroTierIdentity localIdentity, NodeId destination, - IPEndPoint physicalDestination, + IPEndPoint? physicalDestination, ZeroTierWorld planet, ulong timestamp, ReadOnlySpan sharedKey, @@ -19,13 +19,40 @@ public static byte[] BuildPacket( byte advertisedMinorVersion, ushort advertisedRevision, out ulong packetId) + => BuildPacket( + localIdentity, + destination, + physicalDestination, + timestamp, + planet.Id, + planet.Timestamp, + sharedKey, + advertisedProtocolVersion, + advertisedMajorVersion, + advertisedMinorVersion, + advertisedRevision, + out packetId); + + public static byte[] BuildPacket( + ZeroTierIdentity localIdentity, + NodeId destination, + IPEndPoint? physicalDestination, + ulong timestamp, + ulong planetId, + ulong planetTimestamp, + ReadOnlySpan sharedKey, + byte advertisedProtocolVersion, + byte advertisedMajorVersion, + byte advertisedMinorVersion, + ushort advertisedRevision, + out ulong packetId) { var iv = new byte[8]; RandomNumberGenerator.Fill(iv); packetId = BinaryPrimitives.ReadUInt64BigEndian(iv); var identityLength = ZeroTierIdentityCodec.GetSerializedLength(localIdentity, includePrivate: false); - var inetLength = ZeroTierInetAddressCodec.GetSerializedLength(physicalDestination); + var inetLength = ZeroTierInetAddressCodec.GetSerializedLengthNullable(physicalDestination); var payloadFixedLength = 1 + // protocol version @@ -53,11 +80,11 @@ public static byte[] BuildPacket( p += 8; p += ZeroTierIdentityCodec.Serialize(localIdentity, payload.AsSpan(p), includePrivate: false); - p += ZeroTierInetAddressCodec.Serialize(physicalDestination, payload.AsSpan(p)); + p += ZeroTierInetAddressCodec.SerializeNullable(physicalDestination, payload.AsSpan(p)); - BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planet.Id); + BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planetId); p += 8; - BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planet.Timestamp); + BinaryPrimitives.WriteUInt64BigEndian(payload.AsSpan(p, 8), planetTimestamp); p += 8; // Encrypted portion: moon count (0) (no moons advertised). diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs b/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs index 3c6f7f3..9aceee5 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierIdentityStore.cs @@ -94,7 +94,6 @@ public static void Save(string path, ZeroTierIdentity identity) identity.PublicKey.CopyTo(bytes.AsSpan(5 + 8, ZeroTierIdentity.PublicKeyLength)); identity.PrivateKey.CopyTo(bytes.AsSpan(5 + 8 + ZeroTierIdentity.PublicKeyLength, ZeroTierIdentity.PrivateKeyLength)); - AtomicFile.WriteAllBytes(path, bytes); - SecretFilePermissions.TryHardenSecretFile(path); + AtomicFile.WriteSecretBytes(path, bytes); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierInboundDatagramDiagnostics.cs b/ZTSharp/ZeroTier/Internal/ZeroTierInboundDatagramDiagnostics.cs new file mode 100644 index 0000000..99088ed --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierInboundDatagramDiagnostics.cs @@ -0,0 +1,115 @@ +using System.Net; +using ZTSharp.ZeroTier.Protocol; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierInboundDatagramDiagnostics +{ + private readonly NodeId _localNodeId; + private readonly IPEndPoint _rootEndpoint; + private int _rawRemaining; + private int _dropRemaining; + + public ZeroTierInboundDatagramDiagnostics(NodeId localNodeId, IPEndPoint rootEndpoint, int rawBudget = 200, int dropBudget = 100) + { + ArgumentNullException.ThrowIfNull(rootEndpoint); + _localNodeId = localNodeId; + _rootEndpoint = rootEndpoint; + _rawRemaining = rawBudget; + _dropRemaining = dropBudget; + } + + public void TraceInboundRaw(ZeroTierUdpDatagram datagram) + { + if (!ShouldTraceRaw(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] RX direct raw: socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint} bytes={datagram.Payload.Length}."); + } + + public void TraceDecodeFailure(ZeroTierUdpDatagram datagram) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct raw: decode failed on socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint} bytes={datagram.Payload.Length}."); + } + + public void TraceDestinationMismatch(ZeroTierUdpDatagram datagram, in ZeroTierPacketView decoded) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct raw: destination mismatch src={decoded.Header.Source} dst={decoded.Header.Destination} local={_localNodeId} hop={decoded.Header.HopCount} via={datagram.RemoteEndPoint}."); + } + + public void TraceDecodedInbound(ZeroTierUdpDatagram datagram, in ZeroTierPacketView decoded) + { + if (!ShouldTraceRaw(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] RX direct decoded: src={decoded.Header.Source} dst={decoded.Header.Destination} hop={decoded.Header.HopCount} cipher={decoded.Header.CipherSuite} verbRaw=0x{decoded.Header.VerbRaw:x2} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint}."); + } + + public void TracePeerKeyMissing(NodeId peerNodeId, ZeroTierUdpDatagram datagram, byte hopCount) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct peer: missing shared key peer={peerNodeId} hop={hopCount} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint}."); + } + + public void TraceDearmorFailure(NodeId peerNodeId, ZeroTierUdpDatagram datagram, byte hopCount) + { + if (!ShouldTraceDrop(datagram.RemoteEndPoint)) + { + return; + } + + ZeroTierTrace.WriteLine( + $"[zerotier] Drop direct peer: failed dearmor peer={peerNodeId} hop={hopCount} socket={datagram.LocalSocketId} via={datagram.RemoteEndPoint}."); + } + + private bool ShouldTraceRaw(IPEndPoint remoteEndPoint) + => ZeroTierTrace.Enabled && + !remoteEndPoint.Equals(_rootEndpoint) && + Consume(ref _rawRemaining); + + private bool ShouldTraceDrop(IPEndPoint remoteEndPoint) + => ZeroTierTrace.Enabled && + !remoteEndPoint.Equals(_rootEndpoint) && + Consume(ref _dropRemaining); + + private static bool Consume(ref int budget) + { + while (true) + { + var current = Volatile.Read(ref budget); + if (current <= 0) + { + return false; + } + + if (Interlocked.CompareExchange(ref budget, current - 1, current) == current) + { + return true; + } + } + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs b/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs index da680ee..c617d1c 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierIpv4LinkReceiver.cs @@ -157,13 +157,13 @@ public async ValueTask> ReceiveAsync(CancellationToken canc case ZeroTierVerb.Rendezvous when isFromRoot: { await _directEndpoints - .HandleRendezvousFromRootAsync(payload, datagram.RemoteEndPoint, cancellationToken) + .HandleRendezvousFromRootAsync(payload, datagram.LocalSocketId, datagram.RemoteEndPoint, cancellationToken) .ConfigureAwait(false); continue; } case ZeroTierVerb.PushDirectPaths when isFromRemote: { - await _directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, cancellationToken).ConfigureAwait(false); + await _directEndpoints.HandlePushDirectPathsFromRemoteAsync(payload, datagram.LocalSocketId, cancellationToken).ConfigureAwait(false); continue; } case ZeroTierVerb.MulticastFrame: diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs new file mode 100644 index 0000000..3523937 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementPlanner.cs @@ -0,0 +1,75 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierLocalDirectPathAdvertisementPlanner +{ + private readonly ZeroTierDirectEndpointPolicy _policy; + + public ZeroTierLocalDirectPathAdvertisementPlanner(ZeroTierDirectEndpointPolicy policy) + { + _policy = policy ?? throw new ArgumentNullException(nameof(policy)); + } + + public IPEndPoint[] SelectForPeer( + IReadOnlyList localAdvertisements, + IReadOnlyList hintedPeerEndpoints, + IReadOnlyList observedPeerPaths) + { + ArgumentNullException.ThrowIfNull(localAdvertisements); + ArgumentNullException.ThrowIfNull(hintedPeerEndpoints); + ArgumentNullException.ThrowIfNull(observedPeerPaths); + + var peerEndpoints = BuildPeerEndpoints(hintedPeerEndpoints, observedPeerPaths); + if (peerEndpoints.Any(static endpoint => !ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint))) + { + return localAdvertisements.ToArray(); + } + + var selected = new List(localAdvertisements.Count); + for (var i = 0; i < localAdvertisements.Count; i++) + { + var localEndpoint = localAdvertisements[i]; + if (_policy.ShouldAdvertiseLocalEndpoint(localEndpoint, peerEndpoints)) + { + selected.Add(localEndpoint); + } + } + + return selected.Count == 0 + ? localAdvertisements.ToArray() + : selected.ToArray(); + } + + private static IPEndPoint[] BuildPeerEndpoints( + IReadOnlyList hintedPeerEndpoints, + IReadOnlyList observedPeerPaths) + { + var endpoints = new List(hintedPeerEndpoints.Count + observedPeerPaths.Count); + var seen = new HashSet(StringComparer.Ordinal); + + for (var i = 0; i < hintedPeerEndpoints.Count; i++) + { + AddUnique(endpoints, seen, hintedPeerEndpoints[i]); + } + + for (var i = 0; i < observedPeerPaths.Count; i++) + { + AddUnique(endpoints, seen, observedPeerPaths[i].RemoteEndPoint); + } + + return endpoints.ToArray(); + } + + private static void AddUnique(List endpoints, HashSet seen, IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var canonical = UdpEndpointNormalization.Normalize(endpoint); + if (seen.Add(canonical.ToString())) + { + endpoints.Add(canonical); + } + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs new file mode 100644 index 0000000..e15260e --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierLocalDirectPathAdvertisementSource.cs @@ -0,0 +1,153 @@ +using System.Net; +using System.Net.NetworkInformation; +using ZTSharp.ZeroTier.Transport; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierLocalDirectPathAdvertisementSource +{ + private static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(30); + + private readonly Func _getLocalAddresses; + private readonly Func _nowMs; + private readonly object _lock = new(); + + private IPAddress[] _cachedLocalAddresses = Array.Empty(); + private long _nextRefreshMs; + + public ZeroTierLocalDirectPathAdvertisementSource( + Func? getLocalAddresses = null, + Func? nowMs = null) + { + _getLocalAddresses = getLocalAddresses ?? GetLocalAddresses; + _nowMs = nowMs ?? (() => Environment.TickCount64); + } + + public IPEndPoint[] GetSnapshot(IReadOnlyList localSockets) + { + ArgumentNullException.ThrowIfNull(localSockets); + + var localAddresses = GetCachedLocalAddresses(); + if (localAddresses.Length == 0 || localSockets.Count == 0) + { + return Array.Empty(); + } + + var endpoints = new List(localAddresses.Length * localSockets.Count); + for (var i = 0; i < localSockets.Count; i++) + { + var localEndpoint = localSockets[i].LocalEndpoint; + var port = localEndpoint.Port; + if (port == 0) + { + continue; + } + + var boundAddress = Canonicalize(localEndpoint.Address); + var allowAnyPublicAddress = boundAddress is null || + boundAddress.Equals(IPAddress.Any) || + boundAddress.Equals(IPAddress.IPv6Any); + + for (var a = 0; a < localAddresses.Length; a++) + { + var candidateAddress = localAddresses[a]; + if (!ShouldAdvertise(candidateAddress) || + !ShouldAdvertiseForSocket(boundAddress, candidateAddress, allowAnyPublicAddress)) + { + continue; + } + + endpoints.Add(new IPEndPoint(candidateAddress, port)); + } + } + + return endpoints + .Distinct() + .ToArray(); + } + + private IPAddress[] GetCachedLocalAddresses() + { + var now = _nowMs(); + lock (_lock) + { + if (_cachedLocalAddresses.Length == 0 || unchecked(now - _nextRefreshMs) >= 0) + { + _cachedLocalAddresses = _getLocalAddresses(); + _nextRefreshMs = now + (long)RefreshInterval.TotalMilliseconds; + } + + return _cachedLocalAddresses; + } + } + + private static IPAddress[] GetLocalAddresses() + { + var addresses = new List(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + { + if (nic.OperationalStatus != OperationalStatus.Up) + { + continue; + } + + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + continue; + } + + foreach (var unicast in properties.UnicastAddresses) + { + var address = Canonicalize(unicast.Address); + if (address is null || IPAddress.IsLoopback(address)) + { + continue; + } + + if (!ShouldAdvertise(address)) + { + continue; + } + + addresses.Add(address); + } + } + + return addresses + .Distinct() + .ToArray(); + } + + private static bool ShouldAdvertise(IPAddress address) + => ZeroTierDirectEndpointSelection.IsPublicEndpoint(new IPEndPoint(address, 1)); + + private static bool ShouldAdvertiseForSocket(IPAddress? boundAddress, IPAddress candidateAddress, bool allowAnyPublicAddress) + { + if (allowAnyPublicAddress) + { + return true; + } + + return boundAddress is not null && boundAddress.Equals(candidateAddress); + } + + private static IPAddress? Canonicalize(IPAddress? address) + { + if (address is null) + { + return null; + } + + if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + return address.MapToIPv4(); + } + + return address; + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierNetworkCredentialsPacketBuilder.cs b/ZTSharp/ZeroTier/Internal/ZeroTierNetworkCredentialsPacketBuilder.cs new file mode 100644 index 0000000..e0f6fab --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierNetworkCredentialsPacketBuilder.cs @@ -0,0 +1,45 @@ +using System.Buffers.Binary; +using ZTSharp.ZeroTier.Protocol; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierNetworkCredentialsPacketBuilder +{ + public static byte[] BuildPacket( + ulong packetId, + NodeId destination, + NodeId source, + ReadOnlySpan inlineCom, + ReadOnlySpan sharedKey) + { + if (inlineCom.IsEmpty) + { + throw new ArgumentException("Inline COM must not be empty.", nameof(inlineCom)); + } + + var payload = new byte[inlineCom.Length + 1 + (sizeof(ushort) * 4)]; + inlineCom.CopyTo(payload); + + var ptr = inlineCom.Length; + payload[ptr++] = 0; // End of COM array. + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + ptr += 2; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + ptr += 2; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + ptr += 2; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); + + var header = new ZeroTierPacketHeader( + PacketId: packetId, + Destination: destination, + Source: source, + Flags: 0, + Mac: 0, + VerbRaw: (byte)ZeroTierVerb.NetworkCredentials); + + var packet = ZeroTierPacketCodec.Encode(header, payload); + ZeroTierPacketCrypto.Armor(packet, sharedKey, encryptPayload: true); + return packet; + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs index c61e838..bcbe900 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerBondPolicyEngine.cs @@ -9,6 +9,9 @@ internal sealed class ZeroTierPeerBondPolicyEngine { private const long AwareFlowTtlMs = 120_000; private const int AwareLatencySlackMs = 25; + private const long ActiveBackupMinHoldMs = 10_000; + private const int ActiveBackupLatencySlackMs = 25; + private const long PeerStateTtlMs = 600_000; private readonly Func _getLatencyMs; private readonly Func _getRemoteUtility; @@ -54,6 +57,7 @@ public bool TrySelectSinglePath( case ZeroTierBondPolicy.BalanceRoundRobin: StableSort(observedPaths); var state = _peerStates.GetOrAdd(peerNodeId, static _ => new PeerState()); + Volatile.Write(ref state.LastUsedMs, _nowMs()); var rr = Interlocked.Increment(ref state.RoundRobinCounter); return SelectByIndex(observedPaths, index: (int)((uint)rr % (uint)observedPaths.Length), out selected); @@ -62,6 +66,9 @@ public bool TrySelectSinglePath( return SelectBalanceAware(peerNodeId, observedPaths, flowId, out selected); case ZeroTierBondPolicy.ActiveBackup: + StableSort(observedPaths); + return SelectActiveBackup(peerNodeId, observedPaths, out selected); + case ZeroTierBondPolicy.Off: default: return SelectBest(peerNodeId, observedPaths, out selected); @@ -71,9 +78,18 @@ public bool TrySelectSinglePath( public void MaintenanceTick() { var now = _nowMs(); - foreach (var state in _peerStates.Values) + foreach (var pair in _peerStates) { + var state = pair.Value; CleanupFlowsIfNeeded(state, now); + + var lastUsed = Volatile.Read(ref state.LastUsedMs); + if (lastUsed != 0 && + unchecked(now - lastUsed) > PeerStateTtlMs && + state.Flows.IsEmpty) + { + _peerStates.TryRemove(pair.Key, out _); + } } } @@ -115,7 +131,6 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa var bestHasLatency = false; var bestLatency = int.MaxValue; var bestUtility = short.MinValue; - var bestLastSeen = long.MinValue; for (var i = 0; i < observedPaths.Length; i++) { @@ -129,7 +144,7 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa (!bestHasLatency) || (latencyMs < bestLatency) || (latencyMs == bestLatency && remoteUtility > bestUtility) || - (latencyMs == bestLatency && remoteUtility == bestUtility && path.LastSeenUnixMs > bestLastSeen); + (latencyMs == bestLatency && remoteUtility == bestUtility && (bestIndex < 0 || StableComparer.Instance.Compare(path, observedPaths[bestIndex]) < 0)); if (better) { @@ -137,7 +152,6 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa bestHasLatency = true; bestLatency = latencyMs; bestUtility = remoteUtility; - bestLastSeen = path.LastSeenUnixMs; } } else if (!bestHasLatency) @@ -145,13 +159,12 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa var better = (bestIndex < 0) || (remoteUtility > bestUtility) || - (remoteUtility == bestUtility && path.LastSeenUnixMs > bestLastSeen); + (remoteUtility == bestUtility && (bestIndex < 0 || StableComparer.Instance.Compare(path, observedPaths[bestIndex]) < 0)); if (better) { bestIndex = i; bestUtility = remoteUtility; - bestLastSeen = path.LastSeenUnixMs; } } } @@ -166,6 +179,92 @@ private bool SelectBest(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPa return true; } + private bool SelectActiveBackup(NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths, out ZeroTierSelectedPeerPath selected) + { + var now = _nowMs(); + var state = _peerStates.GetOrAdd(peerNodeId, static _ => new PeerState()); + Volatile.Write(ref state.LastUsedMs, now); + + lock (state.Gate) + { + if (state.ActiveBackupPath is { } existing && TryFindPath(observedPaths, existing, out var existingPath)) + { + selected = new ZeroTierSelectedPeerPath(existingPath.LocalSocketId, existingPath.RemoteEndPoint); + + if (state.ActiveBackupSelectedAtMs != 0 && unchecked(now - state.ActiveBackupSelectedAtMs) < ActiveBackupMinHoldMs) + { + return true; + } + + if (!SelectBest(peerNodeId, observedPaths, out var best) || best == selected) + { + return true; + } + + if (!ShouldSwitchActiveBackup(peerNodeId, existingPath, best)) + { + return true; + } + + state.ActiveBackupPath = new ZeroTierPeerPhysicalPathKey(best.LocalSocketId, best.RemoteEndPoint); + state.ActiveBackupSelectedAtMs = now; + selected = best; + return true; + } + + if (!SelectBest(peerNodeId, observedPaths, out selected)) + { + return false; + } + + state.ActiveBackupPath = new ZeroTierPeerPhysicalPathKey(selected.LocalSocketId, selected.RemoteEndPoint); + state.ActiveBackupSelectedAtMs = now; + return true; + } + } + + private bool ShouldSwitchActiveBackup(NodeId peerNodeId, ZeroTierPeerPhysicalPath current, ZeroTierSelectedPeerPath best) + { + var bestLatency = _getLatencyMs(peerNodeId, best.LocalSocketId, best.RemoteEndPoint); + var currentLatency = _getLatencyMs(peerNodeId, current.LocalSocketId, current.RemoteEndPoint); + + if (bestLatency is int bestLatencyMs) + { + if (currentLatency is not int currentLatencyMs) + { + return true; + } + + if (bestLatencyMs + ActiveBackupLatencySlackMs < currentLatencyMs) + { + return true; + } + } + else if (currentLatency is int) + { + return false; + } + + var bestUtility = _getRemoteUtility(peerNodeId, best.LocalSocketId, best.RemoteEndPoint); + var currentUtility = _getRemoteUtility(peerNodeId, current.LocalSocketId, current.RemoteEndPoint); + return bestUtility > currentUtility; + } + + private static bool TryFindPath(ZeroTierPeerPhysicalPath[] observedPaths, ZeroTierPeerPhysicalPathKey key, out ZeroTierPeerPhysicalPath path) + { + for (var i = 0; i < observedPaths.Length; i++) + { + if (observedPaths[i].LocalSocketId == key.LocalSocketId && observedPaths[i].RemoteEndPoint.Equals(key.RemoteEndPoint)) + { + path = observedPaths[i]; + return true; + } + } + + path = default; + return false; + } + private bool SelectBalanceAware( NodeId peerNodeId, ZeroTierPeerPhysicalPath[] observedPaths, @@ -174,6 +273,7 @@ private bool SelectBalanceAware( { var now = _nowMs(); var state = _peerStates.GetOrAdd(peerNodeId, static _ => new PeerState()); + Volatile.Write(ref state.LastUsedMs, now); CleanupFlowsIfNeeded(state, now); @@ -264,9 +364,13 @@ private static void StableSort(ZeroTierPeerPhysicalPath[] observedPaths) private sealed class PeerState { + public object Gate { get; } = new(); public int RoundRobinCounter; public ConcurrentDictionary Flows { get; } = new(); public long LastFlowCleanupMs; + public ZeroTierPeerPhysicalPathKey? ActiveBackupPath; + public long ActiveBackupSelectedAtMs; + public long LastUsedMs; } private readonly record struct FlowAssignment(ZeroTierPeerPhysicalPathKey Path, long LastUsedMs); @@ -289,7 +393,7 @@ public int Compare(ZeroTierPeerPhysicalPath x, ZeroTierPeerPhysicalPath y) return c; } - return x.LastSeenUnixMs.CompareTo(y.LastSeenUnixMs); + return 0; } private static int CompareEndpoints(IPEndPoint x, IPEndPoint y) @@ -317,6 +421,15 @@ private static int CompareEndpoints(IPEndPoint x, IPEndPoint y) return xb.Length.CompareTo(yb.Length); } + if (x.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + var scopeCompare = x.Address.ScopeId.CompareTo(y.Address.ScopeId); + if (scopeCompare != 0) + { + return scopeCompare; + } + } + return x.Port.CompareTo(y.Port); } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs index ea952f5..2f29bf1 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerEchoManager.cs @@ -10,6 +10,8 @@ internal sealed class ZeroTierPeerEchoManager { private const long EchoMinIntervalMs = 5_000; private const long PendingEchoTtlMs = 30_000; + private const long EchoPathCacheTtlMs = 120_000; + private const long RttCacheTtlMs = 120_000; private readonly IZeroTierUdpTransport _udp; private readonly NodeId _localNodeId; @@ -18,8 +20,9 @@ internal sealed class ZeroTierPeerEchoManager private readonly ConcurrentDictionary _lastEchoSentUnixMs = new(); private readonly ConcurrentDictionary _pendingByPacketId = new(); - private readonly ConcurrentDictionary _lastRttMsByPath = new(); + private readonly ConcurrentDictionary _lastRttMsByPath = new(); private long _lastPendingCleanupUnixMs; + private long _lastCacheCleanupMs; public ZeroTierPeerEchoManager( IZeroTierUdpTransport udp, @@ -33,14 +36,21 @@ public ZeroTierPeerEchoManager( _udp = udp; _localNodeId = localNodeId; _getPeerProtocolVersion = getPeerProtocolVersion; - _nowUnixMs = nowUnixMs ?? (() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } public bool TryGetLastRttMs(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, out int rttMs) { ArgumentNullException.ThrowIfNull(remoteEndPoint); var key = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); - return _lastRttMsByPath.TryGetValue(key, out rttMs); + if (_lastRttMsByPath.TryGetValue(key, out var entry)) + { + rttMs = entry.RttMs; + return true; + } + + rttMs = 0; + return false; } public void ObserveHelloOkRtt(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, ulong helloTimestampEcho) @@ -55,7 +65,7 @@ public void ObserveHelloOkRtt(NodeId peerNodeId, int localSocketId, IPEndPoint r } var key = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); - _lastRttMsByPath[key] = (int)rtt; + _lastRttMsByPath[key] = new RttEntry((int)rtt, LastUpdatedMs: now); } public async ValueTask TrySendEchoProbeAsync( @@ -71,6 +81,7 @@ public async ValueTask TrySendEchoProbeAsync( var now = _nowUnixMs(); CleanupPendingIfNeeded(now); + CleanupCachesIfNeeded(now); var pathKey = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); if (_lastEchoSentUnixMs.TryGetValue(pathKey, out var lastSent) && unchecked(now - lastSent) < EchoMinIntervalMs) @@ -80,9 +91,6 @@ public async ValueTask TrySendEchoProbeAsync( _lastEchoSentUnixMs[pathKey] = now; - var payload = new byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(payload, (ulong)now); - var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); var header = new ZeroTierPacketHeader( PacketId: packetId, @@ -92,7 +100,7 @@ public async ValueTask TrySendEchoProbeAsync( Mac: 0, VerbRaw: (byte)ZeroTierVerb.Echo); - var packet = ZeroTierPacketCodec.Encode(header, payload); + var packet = ZeroTierPacketCodec.Encode(header, ReadOnlySpan.Empty); var peerProtocolVersion = _getPeerProtocolVersion(peerNodeId); ZeroTierPacketCrypto.Armor(packet, ZeroTierPacketCrypto.SelectOutboundKey(sharedKey, peerProtocolVersion), encryptPayload: true); var onWirePacketId = BinaryPrimitives.ReadUInt64BigEndian(packet.AsSpan(0, 8)); @@ -123,17 +131,10 @@ public async ValueTask HandleEchoRequestAsync( cancellationToken.ThrowIfCancellationRequested(); var payloadSpan = payload.Span; - if (payloadSpan.Length < 8) - { - return; - } - - var timestampEcho = BinaryPrimitives.ReadUInt64BigEndian(payloadSpan.Slice(0, 8)); - - var okPayload = new byte[1 + 8 + 8]; + var okPayload = new byte[1 + 8 + payloadSpan.Length]; okPayload[0] = (byte)ZeroTierVerb.Echo; BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1, 8), inRePacketId); - BinaryPrimitives.WriteUInt64BigEndian(okPayload.AsSpan(1 + 8, 8), timestampEcho); + payloadSpan.CopyTo(okPayload.AsSpan(1 + 8)); var packetId = ZeroTierPacketIdGenerator.GeneratePacketId(); var header = new ZeroTierPacketHeader( @@ -150,7 +151,7 @@ public async ValueTask HandleEchoRequestAsync( await _udp.SendAsync(localSocketId, remoteEndPoint, packet, cancellationToken).ConfigureAwait(false); } - public void HandleEchoOk( + public bool HandleEchoOk( NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint, @@ -159,32 +160,45 @@ public void HandleEchoOk( { ArgumentNullException.ThrowIfNull(remoteEndPoint); - if (!_pendingByPacketId.TryRemove(inRePacketId, out var pending)) + if (!TryTakePendingEcho(peerNodeId, inRePacketId, out var pending)) { - return; + return false; } - if (pending.PathKey.PeerNodeId != peerNodeId || - pending.PathKey.Path.LocalSocketId != localSocketId || - !pending.PathKey.Path.RemoteEndPoint.Equals(remoteEndPoint)) + var now = _nowUnixMs(); + var rtt = unchecked(now - pending.TimestampUnixMs); + if (rtt < 0 || rtt > int.MaxValue) { - return; + return false; } - if (okPayloadTail.Length < 8) + var observedPath = new ZeroTierPeerEchoPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); + _lastRttMsByPath[observedPath] = new RttEntry((int)rtt, LastUpdatedMs: now); + return true; + } + + private bool TryTakePendingEcho(NodeId peerNodeId, ulong inRePacketId, out PendingEcho pending) + { + if (_pendingByPacketId.TryRemove(inRePacketId, out pending) && pending.PathKey.PeerNodeId == peerNodeId) { - return; + return true; } - var timestampEcho = BinaryPrimitives.ReadUInt64BigEndian(okPayloadTail.Slice(0, 8)); - var now = _nowUnixMs(); - var rtt = unchecked((long)now - (long)timestampEcho); - if (rtt < 0 || rtt > int.MaxValue) + foreach (var candidate in _pendingByPacketId) { - return; + if (!ZeroTierReplyCorrelation.Matches(candidate.Key, inRePacketId)) + { + continue; + } + + if (_pendingByPacketId.TryRemove(candidate.Key, out pending) && pending.PathKey.PeerNodeId == peerNodeId) + { + return true; + } } - _lastRttMsByPath[pending.PathKey] = (int)rtt; + pending = default; + return false; } private void CleanupPendingIfNeeded(long nowUnixMs) @@ -210,7 +224,42 @@ private void CleanupPendingIfNeeded(long nowUnixMs) } } + private void CleanupCachesIfNeeded(long nowMs) + { + var last = Volatile.Read(ref _lastCacheCleanupMs); + if (last != 0 && unchecked(nowMs - last) < 10_000) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastCacheCleanupMs, nowMs, last) != last) + { + return; + } + + var echoCutoff = nowMs - EchoPathCacheTtlMs; + foreach (var pair in _lastEchoSentUnixMs) + { + if (pair.Value <= echoCutoff) + { + _lastEchoSentUnixMs.TryRemove(pair.Key, out _); + } + } + + var rttCutoff = nowMs - RttCacheTtlMs; + foreach (var pair in _lastRttMsByPath) + { + if (pair.Value.LastUpdatedMs <= rttCutoff) + { + _lastRttMsByPath.TryRemove(pair.Key, out _); + } + } + } + private readonly record struct PendingEcho(ZeroTierPeerEchoPathKey PathKey, long TimestampUnixMs); + + private readonly record struct RttEntry(int RttMs, long LastUpdatedMs); } internal readonly record struct ZeroTierPeerEchoPathKey(NodeId PeerNodeId, ZeroTierPeerPhysicalPathKey Path); + diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs index 63d9760..ecac0dc 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPathNegotiationManager.cs @@ -7,9 +7,12 @@ namespace ZTSharp.ZeroTier.Internal; internal sealed class ZeroTierPeerPathNegotiationManager { private const long MinSendIntervalMs = 10_000; + private const long NegotiationStateTtlMs = 300_000; + private const int MaxNegotiationStates = 4096; private readonly Func _nowMs; private readonly ConcurrentDictionary _state = new(); + private long _lastCleanupMs; public ZeroTierPeerPathNegotiationManager(Func? nowMs = null) { @@ -21,6 +24,7 @@ public void HandleInboundRequest(NodeId peerNodeId, int localSocketId, IPEndPoin ArgumentNullException.ThrowIfNull(remoteEndPoint); var now = _nowMs(); + CleanupIfNeeded(now); var key = new ZeroTierPeerNegotiationPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); _state.AddOrUpdate( key, @@ -32,9 +36,18 @@ public bool TryGetRemoteUtility(NodeId peerNodeId, int localSocketId, IPEndPoint { ArgumentNullException.ThrowIfNull(remoteEndPoint); + var now = _nowMs(); + CleanupIfNeeded(now); var key = new ZeroTierPeerNegotiationPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); if (_state.TryGetValue(key, out var state)) { + if (state.LastReceivedMs != 0 && unchecked(now - state.LastReceivedMs) > NegotiationStateTtlMs) + { + _state.TryRemove(new KeyValuePair(key, state)); + remoteUtility = 0; + return false; + } + remoteUtility = state.RemoteUtility; return true; } @@ -48,6 +61,7 @@ public bool TryMarkSent(NodeId peerNodeId, int localSocketId, IPEndPoint remoteE ArgumentNullException.ThrowIfNull(remoteEndPoint); var now = _nowMs(); + CleanupIfNeeded(now); var key = new ZeroTierPeerNegotiationPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); while (true) @@ -74,6 +88,30 @@ public bool TryMarkSent(NodeId peerNodeId, int localSocketId, IPEndPoint remoteE } } + private void CleanupIfNeeded(long now) + { + var last = Volatile.Read(ref _lastCleanupMs); + if (last != 0 && unchecked(now - last) < 10_000 && _state.Count <= MaxNegotiationStates) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastCleanupMs, now, last) != last) + { + return; + } + + foreach (var pair in _state) + { + var state = pair.Value; + var touched = Math.Max(state.LastReceivedMs, state.LastSentMs); + if (touched != 0 && unchecked(now - touched) > NegotiationStateTtlMs) + { + _state.TryRemove(pair); + } + } + } + private readonly record struct NegotiationState(short RemoteUtility, long LastReceivedMs, long LastSentMs); } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs index 414c54b..3d9b39d 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerPhysicalPathTracker.cs @@ -18,18 +18,44 @@ public ZeroTierPeerPhysicalPathTracker(TimeSpan ttl, Func? nowUnixMs = nul } _ttl = ttl; - _nowUnixMs = nowUnixMs ?? (() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + _nowUnixMs = nowUnixMs ?? (() => Environment.TickCount64); } - public void ObserveHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + public bool ObserveHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) { ArgumentNullException.ThrowIfNull(remoteEndPoint); var now = _nowUnixMs(); var peer = _peers.GetOrAdd(peerNodeId, _ => new PeerState()); - peer.Paths[new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)] = now; + var key = new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint); + var isNewPath = !peer.Paths.ContainsKey(key); + peer.Paths[key] = now; CleanupIfNeeded(now); + return isNewPath; + } + + public bool ObserveKnownHop0(NodeId peerNodeId, int localSocketId, IPEndPoint remoteEndPoint) + { + ArgumentNullException.ThrowIfNull(remoteEndPoint); + + var now = _nowUnixMs(); + var peer = _peers.GetOrAdd(peerNodeId, _ => new PeerState()); + var key = new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint); + if (!peer.Paths.ContainsKey(key)) + { + CleanupIfNeeded(now); + if (peer.Paths.IsEmpty) + { + _peers.TryRemove(peerNodeId, out _); + } + + return false; + } + + peer.Paths[key] = now; + CleanupIfNeeded(now); + return true; } public ZeroTierPeerPhysicalPath[] GetSnapshot(NodeId peerNodeId) diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs index 3a00cd9..9deb2ee 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerQosManager.cs @@ -20,9 +20,13 @@ internal sealed class ZeroTierPeerQosManager // See: ZeroTierOne node/Bond.cpp (qosStatsOut timeout = _qosSendInterval * 3). private const long DefaultRecordTimeoutMs = 30_000; + private const long PathStateTtlMs = 300_000; + private const long LatencyAverageTtlMs = 120_000; + private const int MaxPathStates = 4096; private readonly Func _nowMs; private readonly ConcurrentDictionary _paths = new(); + private long _lastPathCleanupMs; public ZeroTierPeerQosManager(Func? nowMs = null) { @@ -41,6 +45,8 @@ public void RecordIncomingPacket(NodeId peerNodeId, int localSocketId, IPEndPoin var now = _nowMs(); var key = new ZeroTierPeerQosPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); var state = _paths.GetOrAdd(key, static _ => new PathState()); + Volatile.Write(ref state.LastTouchedMs, now); + CleanupPathsIfNeeded(now); if (Volatile.Read(ref state.InboundCount) >= MaxPendingRecords) { @@ -63,6 +69,8 @@ public void RecordOutgoingPacket(NodeId peerNodeId, int localSocketId, IPEndPoin var now = _nowMs(); var key = new ZeroTierPeerQosPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); var state = _paths.GetOrAdd(key, static _ => new PathState()); + Volatile.Write(ref state.LastTouchedMs, now); + CleanupPathsIfNeeded(now); CleanupOutboundIfNeeded(state, now); @@ -169,6 +177,9 @@ public void HandleInboundMeasurement( return; } + Volatile.Write(ref state.LastTouchedMs, now); + CleanupPathsIfNeeded(now); + CleanupOutboundIfNeeded(state, now); var count = 0; @@ -213,8 +224,23 @@ public bool TryGetLastLatencyAverageMs( ArgumentNullException.ThrowIfNull(remoteEndPoint); var key = new ZeroTierPeerQosPathKey(peerNodeId, new ZeroTierPeerPhysicalPathKey(localSocketId, remoteEndPoint)); - if (_paths.TryGetValue(key, out var state) && Volatile.Read(ref state.LastLatencyUpdatedMs) != 0) + if (_paths.TryGetValue(key, out var state)) { + var updated = Volatile.Read(ref state.LastLatencyUpdatedMs); + if (updated == 0) + { + averageLatencyMs = 0; + return false; + } + + var now = _nowMs(); + var age = unchecked(now - updated); + if (age < 0 || age > LatencyAverageTtlMs) + { + averageLatencyMs = 0; + return false; + } + averageLatencyMs = Volatile.Read(ref state.LastLatencyAvgMs); return true; } @@ -224,6 +250,7 @@ public bool TryGetLastLatencyAverageMs( } private static bool ShouldTrack(ulong packetId) + // Intentionally mirrors upstream (Bond.cpp): record/ack most packets and skip every Nth (power-of-two) packet id. => (packetId & (ulong)(QosAckDivisor - 1)) != 0; private static void CleanupOutboundIfNeeded(PathState state, long now) @@ -248,11 +275,38 @@ private static void CleanupOutboundIfNeeded(PathState state, long now) } } + private void CleanupPathsIfNeeded(long now) + { + var last = Volatile.Read(ref _lastPathCleanupMs); + if (last != 0 && unchecked(now - last) < 10_000 && _paths.Count <= MaxPathStates) + { + return; + } + + if (Interlocked.CompareExchange(ref _lastPathCleanupMs, now, last) != last) + { + return; + } + + foreach (var pair in _paths) + { + var state = pair.Value; + var touched = Volatile.Read(ref state.LastTouchedMs); + if (touched != 0 && unchecked(now - touched) > PathStateTtlMs && + Volatile.Read(ref state.InboundCount) <= 0 && + state.OutboundSentMs.IsEmpty) + { + _paths.TryRemove(pair.Key, out _); + } + } + } + private sealed class PathState { public ConcurrentQueue Inbound { get; } = new(); public int InboundCount; public long LastSentMs; + public long LastTouchedMs; public ConcurrentDictionary OutboundSentMs { get; } = new(); public long LastOutboundCleanupMs; diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPeerRootSocketAffinity.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPeerRootSocketAffinity.cs new file mode 100644 index 0000000..12590aa --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPeerRootSocketAffinity.cs @@ -0,0 +1,36 @@ +using System.Collections.Concurrent; +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierPeerRootSocketAffinity +{ + private readonly IPEndPoint _rootEndpoint; + private readonly ConcurrentDictionary _socketIdByPeer = new(); + + public ZeroTierPeerRootSocketAffinity(IPEndPoint rootEndpoint) + { + ArgumentNullException.ThrowIfNull(rootEndpoint); + _rootEndpoint = rootEndpoint; + } + + public void Observe(NodeId peerNodeId, int localSocketId, IPEndPoint receivedVia) + { + ArgumentNullException.ThrowIfNull(receivedVia); + + if (!receivedVia.Equals(_rootEndpoint)) + { + return; + } + + _socketIdByPeer[peerNodeId] = localSocketId; + } + + public bool TryGet(NodeId peerNodeId, out int localSocketId) + => _socketIdByPeer.TryGetValue(peerNodeId, out localSocketId); + + public void Remove(NodeId peerNodeId) + { + _socketIdByPeer.TryRemove(peerNodeId, out _); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousAlternateHintPolicy.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousAlternateHintPolicy.cs new file mode 100644 index 0000000..839c342 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousAlternateHintPolicy.cs @@ -0,0 +1,12 @@ +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierPinnedRendezvousAlternateHintPolicy +{ + public static bool ShouldProbe( + bool endpointIsPublic, + bool hasPinnedPublicSibling, + bool hasUsableSocketPath) + => endpointIsPublic + ? hasPinnedPublicSibling + : hasUsableSocketPath; +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousSiblingPortPredictor.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousSiblingPortPredictor.cs new file mode 100644 index 0000000..d57a4f4 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPinnedRendezvousSiblingPortPredictor.cs @@ -0,0 +1,67 @@ +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierPinnedRendezvousSiblingPortPredictor +{ + private const int AdjacentPortRadius = 2; + + public static IPEndPoint[] Expand( + IReadOnlyList endpoints, + Func isPinnedRendezvousEndpoint) + { + ArgumentNullException.ThrowIfNull(endpoints); + ArgumentNullException.ThrowIfNull(isPinnedRendezvousEndpoint); + + var expanded = new List(endpoints.Count + (AdjacentPortRadius * 2)); + var seen = new HashSet(StringComparer.Ordinal); + + for (var i = 0; i < endpoints.Count; i++) + { + AddUnique(expanded, seen, endpoints[i]); + } + + var pinnedPublic = endpoints.FirstOrDefault(endpoint => + isPinnedRendezvousEndpoint(endpoint) && + ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)); + if (pinnedPublic is null) + { + return expanded.ToArray(); + } + + for (var delta = 1; delta <= AdjacentPortRadius; delta++) + { + AddAdjacent(expanded, seen, pinnedPublic, -delta); + AddAdjacent(expanded, seen, pinnedPublic, delta); + } + + return expanded.ToArray(); + } + + private static void AddAdjacent( + List expanded, + HashSet seen, + IPEndPoint endpoint, + int delta) + { + var port = endpoint.Port + delta; + if (port is < 1 or > ushort.MaxValue) + { + return; + } + + AddUnique(expanded, seen, new IPEndPoint(endpoint.Address, port)); + } + + private static void AddUnique( + List expanded, + HashSet seen, + IPEndPoint endpoint) + { + var key = endpoint.Address + ":" + endpoint.Port; + if (seen.Add(key)) + { + expanded.Add(endpoint); + } + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierPredictedPublicSurfaceGenerator.cs b/ZTSharp/ZeroTier/Internal/ZeroTierPredictedPublicSurfaceGenerator.cs new file mode 100644 index 0000000..87f2962 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierPredictedPublicSurfaceGenerator.cs @@ -0,0 +1,67 @@ +using System.Net; +using ZTSharp.Transport.Internal; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierPredictedPublicSurfaceGenerator +{ + private const int AdjacentPortRadius = 2; + + public static IPEndPoint[] Expand(IEnumerable endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + var expanded = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var endpoint in endpoints) + { + if (endpoint is null) + { + continue; + } + + AddUnique(expanded, seen, endpoint); + if (!ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint)) + { + continue; + } + + for (var delta = 1; delta <= AdjacentPortRadius; delta++) + { + AddAdjacent(expanded, seen, endpoint, -delta); + AddAdjacent(expanded, seen, endpoint, delta); + } + } + + return expanded.ToArray(); + } + + private static void AddAdjacent( + List expanded, + HashSet seen, + IPEndPoint endpoint, + int delta) + { + var port = endpoint.Port + delta; + if (port is < 1 or > ushort.MaxValue) + { + return; + } + + AddUnique(expanded, seen, new IPEndPoint(endpoint.Address, port)); + } + + private static void AddUnique( + List expanded, + HashSet seen, + IPEndPoint endpoint) + { + var canonical = UdpEndpointNormalization.Normalize(endpoint); + var key = canonical.Address + ":" + canonical.Port; + if (seen.Add(key)) + { + expanded.Add(canonical); + } + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierReplyCorrelation.cs b/ZTSharp/ZeroTier/Internal/ZeroTierReplyCorrelation.cs new file mode 100644 index 0000000..2941e0e --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierReplyCorrelation.cs @@ -0,0 +1,10 @@ +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierReplyCorrelation +{ + public static uint GetReplyKey(ulong packetId) + => (uint)(packetId >> 32); + + public static bool Matches(ulong expectedPacketId, ulong actualPacketId) + => GetReplyKey(expectedPacketId) == GetReplyKey(actualPacketId); +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs index 8c20a45..75da8ba 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketFactory.cs @@ -44,12 +44,24 @@ public static Task CreateAsync(ZeroTierSocketOptions options, Ca throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts length must match UdpSocketCount."); } + HashSet? seenNonZeroPorts = null; + if (options.Multipath.Enabled) + { + seenNonZeroPorts = new HashSet(); + } + for (var i = 0; i < ports.Count; i++) { if (ports[i] < 0 || ports[i] > 65535) { throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts entries must be in the range [0, 65535]."); } + + var port = ports[i]; + if (seenNonZeroPorts is not null && port != 0 && !seenNonZeroPorts.Add(port)) + { + throw new ArgumentOutOfRangeException(nameof(options), "Multipath LocalUdpPorts must not contain duplicate non-zero ports."); + } } } @@ -60,7 +72,8 @@ public static Task CreateAsync(ZeroTierSocketOptions options, Ca BondPolicy = options.Multipath.BondPolicy, UdpSocketCount = options.Multipath.UdpSocketCount, LocalUdpPorts = options.Multipath.LocalUdpPorts is { } localPorts ? localPorts.ToArray() : null, - WarmupDuplicateToRoot = options.Multipath.WarmupDuplicateToRoot + WarmupDuplicateToRoot = options.Multipath.WarmupDuplicateToRoot, + AllowRootRelayFallback = options.Multipath.AllowRootRelayFallback }; var normalizedOptions = new ZeroTierSocketOptions diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs index 3174c44..ab0e669 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketRuntimeBootstrapper.cs @@ -9,33 +9,48 @@ namespace ZTSharp.ZeroTier.Internal; internal static class ZeroTierSocketRuntimeBootstrapper { - internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOptions multipath, bool enableIpv6) + internal readonly record struct ZeroTierUdpSocketBinding(IPAddress? LocalAddress, int LocalPort, int LocalSocketId); + + internal static async ValueTask CreateUdpTransportAsync( + ZeroTierMultipathOptions multipath, + bool enableIpv6, + AddressFamily? preferredBindAddressFamily = null, + IPAddress? preferredLocalBindAddress = null) { ArgumentNullException.ThrowIfNull(multipath); - if (!multipath.Enabled || multipath.UdpSocketCount == 1) + if (!multipath.Enabled) { return new ZeroTierUdpTransport(localPort: 0, enableIpv6: enableIpv6, localSocketId: 0); } - var ports = multipath.LocalUdpPorts; - if (ports is null) + var bindings = CreateUdpSocketBindings( + multipath, + enableIpv6, + preferredBindAddressFamily, + preferredLocalBindAddress); + if (bindings.Length == 1) { - ports = Enumerable.Repeat(0, multipath.UdpSocketCount).ToArray(); + var binding = bindings[0]; + return new ZeroTierUdpTransport( + localPort: binding.LocalPort, + enableIpv6: enableIpv6, + localSocketId: binding.LocalSocketId, + localBindAddress: binding.LocalAddress); } - if (ports.Count != multipath.UdpSocketCount) - { - throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts length must match UdpSocketCount."); - } - - var sockets = new List(multipath.UdpSocketCount); + var sockets = new List(bindings.Length); var success = false; try { - for (var i = 0; i < multipath.UdpSocketCount; i++) + for (var i = 0; i < bindings.Length; i++) { - sockets.Add(new ZeroTierUdpTransport(localPort: ports[i], enableIpv6: enableIpv6, localSocketId: i)); + var binding = bindings[i]; + sockets.Add(new ZeroTierUdpTransport( + localPort: binding.LocalPort, + enableIpv6: enableIpv6, + localSocketId: binding.LocalSocketId, + localBindAddress: binding.LocalAddress)); } var transport = new ZeroTierUdpMultiTransport(sockets); @@ -50,7 +65,7 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption { try { - socket.DisposeAsync().AsTask().GetAwaiter().GetResult(); + await socket.DisposeAsync().ConfigureAwait(false); } catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException or SocketException or InvalidOperationException) { @@ -60,6 +75,105 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption } } + internal static ZeroTierUdpSocketBinding[] CreateUdpSocketBindings( + ZeroTierMultipathOptions multipath, + bool enableIpv6, + AddressFamily? preferredBindAddressFamily = null, + IPAddress? preferredLocalBindAddress = null, + Func? getLocalBindAddresses = null) + { + ArgumentNullException.ThrowIfNull(multipath); + + if (!multipath.Enabled) + { + return [new ZeroTierUdpSocketBinding(LocalAddress: null, LocalPort: 0, LocalSocketId: 0)]; + } + + if (multipath.UdpSocketCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(multipath), multipath.UdpSocketCount, "UdpSocketCount must be at least 1 when multipath is enabled."); + } + + var ports = multipath.LocalUdpPorts; + if (ports is null) + { + ports = Enumerable.Repeat(0, multipath.UdpSocketCount).ToArray(); + } + + if (ports.Count != multipath.UdpSocketCount) + { + throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts length must match UdpSocketCount."); + } + + var seenNonZeroPorts = new HashSet(); + for (var i = 0; i < ports.Count; i++) + { + var port = ports[i]; + if (port < 0 || port > 65535) + { + throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts entries must be in the range [0, 65535]."); + } + + if (port != 0 && !seenNonZeroPorts.Add(port)) + { + throw new ArgumentOutOfRangeException(nameof(multipath), "LocalUdpPorts must not contain duplicate non-zero ports."); + } + } + + var bindAddresses = multipath.UdpSocketCount > 1 + ? (getLocalBindAddresses ?? (() => ZeroTierUdpLocalBindAddressSource.GetSnapshot(enableIpv6))).Invoke() + : Array.Empty(); + bindAddresses = OrderBindAddresses(bindAddresses, preferredBindAddressFamily, preferredLocalBindAddress); + + var bindings = new ZeroTierUdpSocketBinding[multipath.UdpSocketCount]; + for (var i = 0; i < bindings.Length; i++) + { + var localAddress = bindAddresses.Length == 0 + ? null + : bindAddresses[i % bindAddresses.Length]; + bindings[i] = new ZeroTierUdpSocketBinding(localAddress, ports[i], i); + } + + return bindings; + } + + private static IPAddress[] OrderBindAddresses( + IPAddress[] bindAddresses, + AddressFamily? preferredBindAddressFamily, + IPAddress? preferredLocalBindAddress) + { + if (bindAddresses.Length == 0) + { + return bindAddresses; + } + + if (preferredLocalBindAddress is not null) + { + var routed = bindAddresses + .Where(address => address.Equals(preferredLocalBindAddress)) + .ToArray(); + if (routed.Length != 0) + { + return routed; + } + } + + if (preferredBindAddressFamily is null) + { + return bindAddresses; + } + + var preferred = bindAddresses + .Where(address => address.AddressFamily == preferredBindAddressFamily) + .ToArray(); + if (preferred.Length == 0) + { + return bindAddresses; + } + + return preferred; + } + [SuppressMessage( "Reliability", "CA2000:Dispose objects before losing scope", @@ -80,7 +194,21 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption ArgumentNullException.ThrowIfNull(managedIps); ArgumentNullException.ThrowIfNull(inlineCom); - var udp = CreateUdpTransport(multipath, enableIpv6: true); + var preferredRootEndpoint = SelectPreferredBindEndpoint(planet); + var preferredBindAddressFamily = preferredRootEndpoint?.AddressFamily; + var routeResolver = new ZeroTierDirectRouteResolver(); + var preferredLocalBindAddress = + preferredRootEndpoint is not null && + routeResolver.TryResolve(preferredRootEndpoint, out var routedLocalAddress) + ? routedLocalAddress + : null; + + var udp = await CreateUdpTransportAsync( + multipath, + enableIpv6: true, + preferredBindAddressFamily: preferredBindAddressFamily, + preferredLocalBindAddress: preferredLocalBindAddress) + .ConfigureAwait(false); try { var localManagedIpsV6 = managedIps @@ -112,4 +240,33 @@ internal static IZeroTierUdpTransport CreateUdpTransport(ZeroTierMultipathOption throw; } } + + private static IPEndPoint? SelectPreferredBindEndpoint(ZeroTierWorld planet) + { + for (var i = 0; i < planet.Roots.Count; i++) + { + var root = planet.Roots[i]; + for (var e = 0; e < root.StableEndpoints.Count; e++) + { + if (root.StableEndpoints[e].AddressFamily == AddressFamily.InterNetwork) + { + return root.StableEndpoints[e]; + } + } + } + + for (var i = 0; i < planet.Roots.Count; i++) + { + var root = planet.Roots[i]; + for (var e = 0; e < root.StableEndpoints.Count; e++) + { + if (root.StableEndpoints[e].AddressFamily == AddressFamily.InterNetworkV6) + { + return root.StableEndpoints[e]; + } + } + } + + return null; + } } diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs index 8929851..078bc78 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketStatePersistence.cs @@ -23,22 +23,14 @@ public static IPAddress[] LoadManagedIps(string statePath, ulong networkId) try { - using var stream = new FileStream( - path, - FileMode.Open, - FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete, - bufferSize: 16 * 1024, - options: FileOptions.SequentialScan); - - if (stream.Length <= 0 || stream.Length > MaxManagedIpsFileBytes) + if (!BoundedFileIO.TryReadAllText(path, maxBytes: MaxManagedIpsFileBytes, Encoding.UTF8, out var text)) { return Array.Empty(); } - using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 8 * 1024, leaveOpen: true); var ips = new List(); + using var reader = new StringReader(text); while (true) { var line = reader.ReadLine(); diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs index 5bb0ab5..8e28b57 100644 --- a/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs +++ b/ZTSharp/ZeroTier/Internal/ZeroTierSocketTcpConnector.cs @@ -160,7 +160,8 @@ public static async ValueTask ConnectAsync( localAddress, remote.Address, remotePort: (ushort)remote.Port, - localPort: localPort); + localPort: localPort, + connectOptions: runtime.GetSuggestedTcpConnectOptions()); try { diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs new file mode 100644 index 0000000..1c7a1dc --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs @@ -0,0 +1,95 @@ +using System.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal sealed class ZeroTierStickyHintMaintenanceSelector +{ + private readonly ZeroTierDirectHintPathPlanner _directHintPlanner; + + public ZeroTierStickyHintMaintenanceSelector(ZeroTierDirectHintPathPlanner directHintPlanner) + { + ArgumentNullException.ThrowIfNull(directHintPlanner); + _directHintPlanner = directHintPlanner; + } + + public ZeroTierSelectedPeerPath[] SelectCandidates( + NodeId peerNodeId, + IPEndPoint[] hinted, + int endpointBudget, + bool restrictToPinnedRendezvous, + Func isPinnedRendezvousEndpoint, + Func shouldAllowAlternatePinnedRendezvousHint, + Func getPinnedSocketIds, + Func getAlternateSocketIds) + { + ArgumentNullException.ThrowIfNull(hinted); + ArgumentNullException.ThrowIfNull(isPinnedRendezvousEndpoint); + ArgumentNullException.ThrowIfNull(shouldAllowAlternatePinnedRendezvousHint); + ArgumentNullException.ThrowIfNull(getPinnedSocketIds); + ArgumentNullException.ThrowIfNull(getAlternateSocketIds); + + if (hinted.Length == 0 || endpointBudget <= 0) + { + return Array.Empty(); + } + + if (!restrictToPinnedRendezvous) + { + return CreatePinnedCandidates(hinted[0], getPinnedSocketIds(hinted[0])); + } + + var pinned = hinted + .Where(isPinnedRendezvousEndpoint) + .Take(1) + .ToArray(); + if (pinned.Length == 0) + { + return CreatePinnedCandidates(hinted[0], getPinnedSocketIds(hinted[0])); + } + + var candidates = new List(endpointBudget); + candidates.AddRange(CreatePinnedCandidates(pinned[0], getPinnedSocketIds(pinned[0]))); + if (candidates.Count >= endpointBudget) + { + return candidates.ToArray(); + } + + var alternates = hinted + .Where(endpoint => + !isPinnedRendezvousEndpoint(endpoint) && + shouldAllowAlternatePinnedRendezvousHint(endpoint)) + .OrderBy(static endpoint => ZeroTierDirectEndpointSelection.IsPublicEndpoint(endpoint) ? 1 : 0) + .ToArray(); + if (alternates.Length == 0) + { + return candidates.ToArray(); + } + + var rotatedAlternates = _directHintPlanner.TakeNextHintedEndpoints( + peerNodeId, + alternates, + endpointBudget - candidates.Count); + for (var i = 0; i < rotatedAlternates.Length; i++) + { + var socketIds = getAlternateSocketIds(rotatedAlternates[i]); + if (socketIds.Length == 0) + { + continue; + } + + candidates.Add(new ZeroTierSelectedPeerPath(socketIds[0], rotatedAlternates[i])); + } + + return candidates.ToArray(); + } + + private static ZeroTierSelectedPeerPath[] CreatePinnedCandidates(IPEndPoint endpoint, int[] socketIds) + { + if (socketIds.Length == 0) + { + return Array.Empty(); + } + + return [new ZeroTierSelectedPeerPath(socketIds[0], endpoint)]; + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierTcpHandshakeClassifier.cs b/ZTSharp/ZeroTier/Internal/ZeroTierTcpHandshakeClassifier.cs new file mode 100644 index 0000000..40f2831 --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierTcpHandshakeClassifier.cs @@ -0,0 +1,47 @@ +using ZTSharp.ZeroTier.Net; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierTcpHandshakeClassifier +{ + public static bool IsInitialSyn(ReadOnlySpan ipPacket) + => TryGetTcpFlags(ipPacket, out var flags) && + (flags & TcpCodec.Flags.Syn) != 0 && + (flags & TcpCodec.Flags.Ack) == 0; + + private static bool TryGetTcpFlags(ReadOnlySpan ipPacket, out TcpCodec.Flags flags) + { + flags = 0; + + if (Ipv4Codec.TryParse(ipPacket, out _, out _, out var protocol, out var payload)) + { + return TryParseFlags(protocol, payload, out flags); + } + + if (Ipv6Codec.TryParseTransportPayload(ipPacket, out _, out _, out var nextHeader, out _, out payload)) + { + return TryParseFlags(nextHeader, payload, out flags); + } + + return false; + } + + private static bool TryParseFlags(byte protocol, ReadOnlySpan payload, out TcpCodec.Flags flags) + { + flags = 0; + if (protocol != TcpCodec.ProtocolNumber) + { + return false; + } + + return TcpCodec.TryParse( + payload, + out _, + out _, + out _, + out _, + out flags, + out _, + out _); + } +} diff --git a/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs new file mode 100644 index 0000000..cdd06fd --- /dev/null +++ b/ZTSharp/ZeroTier/Internal/ZeroTierUdpLocalBindAddressSource.cs @@ -0,0 +1,184 @@ +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; + +namespace ZTSharp.ZeroTier.Internal; + +internal static class ZeroTierUdpLocalBindAddressSource +{ + private static readonly string[] ExcludedAdapterNameTokens = + [ + "zerotier", + "tailscale", + "nord", + "wintun", + "wireguard", + "openvpn" + ]; + + public static IPAddress[] GetSnapshot(bool enableIpv6) + { + return SelectAddresses( + NetworkInterface.GetAllNetworkInterfaces() + .Select(CreateInterfaceInfo) + .Where(static info => info is not null) + .Cast() + .ToArray(), + enableIpv6); + } + + internal static IPAddress[] SelectAddresses(IReadOnlyList interfaces, bool enableIpv6) + { + ArgumentNullException.ThrowIfNull(interfaces); + + var addresses = new List(); + for (var i = 0; i < interfaces.Count; i++) + { + var info = interfaces[i]; + if (!ShouldUseInterface(info)) + { + continue; + } + + for (var a = 0; a < info.UnicastAddresses.Length; a++) + { + var address = Canonicalize(info.UnicastAddresses[a]); + if (address is null || !ShouldUseAddress(address, enableIpv6)) + { + continue; + } + + addresses.Add(address); + } + } + + return addresses + .Distinct() + .OrderBy(static address => address.AddressFamily == AddressFamily.InterNetwork ? 0 : 1) + .ToArray(); + } + + private static InterfaceInfo? CreateInterfaceInfo(NetworkInterface nic) + { + IPInterfaceProperties properties; + try + { + properties = nic.GetIPProperties(); + } + catch (NetworkInformationException) + { + return null; + } + + return new InterfaceInfo( + nic.Name, + nic.Description, + nic.NetworkInterfaceType, + nic.OperationalStatus, + properties.UnicastAddresses.Select(static address => address.Address).ToArray(), + properties.GatewayAddresses.Select(static gateway => gateway.Address).ToArray()); + } + + private static bool ShouldUseInterface(InterfaceInfo info) + { + if (info.OperationalStatus != OperationalStatus.Up) + { + return false; + } + + if (info.NetworkInterfaceType is NetworkInterfaceType.Loopback or NetworkInterfaceType.Tunnel) + { + return false; + } + + if (!HasDefaultGateway(info.GatewayAddresses)) + { + return false; + } + + var name = (info.Name + " " + info.Description).ToUpperInvariant(); + for (var i = 0; i < ExcludedAdapterNameTokens.Length; i++) + { + if (name.Contains(ExcludedAdapterNameTokens[i].ToUpperInvariant(), StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool HasDefaultGateway(IPAddress[] gatewayAddresses) + { + for (var i = 0; i < gatewayAddresses.Length; i++) + { + var gateway = Canonicalize(gatewayAddresses[i]); + if (gateway is null || IPAddress.IsLoopback(gateway)) + { + continue; + } + + if (gateway.AddressFamily == AddressFamily.InterNetworkV6 && + (gateway.Equals(IPAddress.IPv6Any) || gateway.IsIPv6LinkLocal || gateway.IsIPv6SiteLocal)) + { + continue; + } + + if (gateway.AddressFamily == AddressFamily.InterNetwork && gateway.Equals(IPAddress.Any)) + { + continue; + } + + return true; + } + + return false; + } + + private static bool ShouldUseAddress(IPAddress? address, bool enableIpv6) + { + if (address is null || IPAddress.IsLoopback(address)) + { + return false; + } + + return address.AddressFamily switch + { + AddressFamily.InterNetwork => !address.Equals(IPAddress.Any) && !IsIpv4LinkLocal(address), + AddressFamily.InterNetworkV6 => enableIpv6 && + !address.Equals(IPAddress.IPv6Any) && + !address.IsIPv6LinkLocal && + !address.IsIPv6SiteLocal, + _ => false + }; + } + + private static bool IsIpv4LinkLocal(IPAddress address) + { + var bytes = address.GetAddressBytes(); + return bytes is [169, 254, _, _]; + } + + private static IPAddress? Canonicalize(IPAddress? address) + { + if (address is null) + { + return null; + } + + if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) + { + return address.MapToIPv4(); + } + + return address; + } + + internal sealed record InterfaceInfo( + string Name, + string Description, + NetworkInterfaceType NetworkInterfaceType, + OperationalStatus OperationalStatus, + IPAddress[] UnicastAddresses, + IPAddress[] GatewayAddresses); +} diff --git a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs index f38d4e7..5df485d 100644 --- a/ZTSharp/ZeroTier/Net/Ipv6Codec.cs +++ b/ZTSharp/ZeroTier/Net/Ipv6Codec.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Net; +using ZTSharp.ZeroTier.Internal; namespace ZTSharp.ZeroTier.Net; @@ -7,9 +8,10 @@ internal static class Ipv6Codec { public const byte Version = 6; public const int HeaderLength = 40; + private const int MaxExtensionHeaderChain = 8; public static bool IsExtensionHeader(byte nextHeader) - => nextHeader is 0 or 43 or 44 or 50 or 51 or 60; + => nextHeader is 0 or 43 or 44 or 50 or 51 or 60 or 135 or 139 or 140; public static byte[] Encode( IPAddress source, @@ -92,4 +94,153 @@ public static bool TryParse( payload = packet.Slice(HeaderLength, payloadLength); return true; } + + public static bool TryParseTransportPayload( + ReadOnlySpan packet, + out IPAddress source, + out IPAddress destination, + out byte protocol, + out byte hopLimit, + out ReadOnlySpan transportPayload) + { + transportPayload = default; + protocol = 0; + + if (!TryParse(packet, out source, out destination, out var nextHeader, out hopLimit, out var payload)) + { + return false; + } + + return TryWalkExtensionHeaders(payload, nextHeader, out protocol, out transportPayload, out _); + } + + public static bool TryParseTransportPayload( + ReadOnlySpan packet, + out IPAddress source, + out IPAddress destination, + out byte protocol, + out byte hopLimit, + out ReadOnlySpan transportPayload, + out int transportPayloadOffset) + { + transportPayload = default; + transportPayloadOffset = 0; + protocol = 0; + + if (!TryParse(packet, out source, out destination, out var nextHeader, out hopLimit, out var payload)) + { + return false; + } + + if (!TryWalkExtensionHeaders(payload, nextHeader, out protocol, out transportPayload, out var offsetFromPayload)) + { + return false; + } + + transportPayloadOffset = HeaderLength + offsetFromPayload; + return true; + } + + private static bool TryWalkExtensionHeaders( + ReadOnlySpan payload, + byte nextHeader, + out byte protocol, + out ReadOnlySpan transportPayload, + out int transportPayloadOffsetFromPayload) + { + protocol = nextHeader; + transportPayload = payload; + transportPayloadOffsetFromPayload = 0; + + var offset = 0; + for (var i = 0; i < MaxExtensionHeaderChain && IsExtensionHeader(protocol); i++) + { + var remaining = payload.Length - offset; + if (remaining < 2) + { + return false; + } + + if (protocol is 0 or 43 or 60 or 135 or 139 or 140) + { + if (remaining < 8) + { + return false; + } + + var headerNext = payload[offset]; + var hdrExtLen = payload[offset + 1]; + var headerLength = (hdrExtLen + 1) * 8; + if (headerLength > remaining) + { + return false; + } + + offset += headerLength; + protocol = headerNext; + continue; + } + + if (protocol == 44) + { + if (remaining < 8) + { + return false; + } + + var headerNext = payload[offset]; + var fragmentOffsetAndFlags = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(offset + 2, 2)); + var fragmentOffset = (fragmentOffsetAndFlags >> 3) & 0x1FFF; + var reservedBits = fragmentOffsetAndFlags & 0x6; + var moreFragments = (fragmentOffsetAndFlags & 0x1) != 0; + if (fragmentOffset != 0 || moreFragments || reservedBits != 0) + { + if (ZeroTierTrace.Enabled) + { + ZeroTierTrace.WriteLine("[zerotier] Drop: IPv6 fragments are not supported."); + } + + return false; + } + + offset += 8; + protocol = headerNext; + continue; + } + + if (protocol == 51) + { + if (remaining < 8) + { + return false; + } + + var headerNext = payload[offset]; + var payloadLen32 = payload[offset + 1]; + var headerLength = (payloadLen32 + 2) * 4; + if (headerLength < 12) + { + return false; + } + + if (headerLength > remaining) + { + return false; + } + + offset += headerLength; + protocol = headerNext; + continue; + } + + if (protocol == 50) + { + return false; + } + } + + transportPayloadOffsetFromPayload = offset; + transportPayload = payload.Slice(offset); + return !IsExtensionHeader(protocol); + } } diff --git a/ZTSharp/ZeroTier/Net/UdpCodec.cs b/ZTSharp/ZeroTier/Net/UdpCodec.cs index 7c05aec..68b0332 100644 --- a/ZTSharp/ZeroTier/Net/UdpCodec.cs +++ b/ZTSharp/ZeroTier/Net/UdpCodec.cs @@ -112,9 +112,10 @@ public static bool TryParseWithChecksum( return false; // IPv6 UDP checksum is mandatory } + var udpLength = HeaderLength + payload.Length; try { - return ComputeChecksum(sourceIp, destinationIp, segment) == 0; + return ComputeChecksum(sourceIp, destinationIp, segment.Slice(0, udpLength)) == 0; } catch (ArgumentOutOfRangeException) { diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs index 84b99cf..3333728 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpClient.cs @@ -1,4 +1,5 @@ using System.Buffers.Binary; +using System.Diagnostics; using System.IO; using System.Net; using System.Security.Cryptography; @@ -15,6 +16,7 @@ internal sealed class UserSpaceTcpClient : IAsyncDisposable private readonly ushort _remotePort; private readonly ushort _localPort; private readonly ushort _mss; + private readonly UserSpaceTcpConnectOptions _connectOptions; private readonly UserSpaceTcpConnectionSignals _signals = new(); private readonly UserSpaceTcpReceiver _receiver = new(); @@ -26,6 +28,7 @@ internal sealed class UserSpaceTcpClient : IAsyncDisposable private Task? _receiveLoopTask; private bool _disposed; + private int _disposeState; public UserSpaceTcpClient( IUserSpaceIpLink link, @@ -33,7 +36,8 @@ public UserSpaceTcpClient( IPAddress remoteAddress, ushort remotePort, ushort? localPort = null, - ushort mss = DefaultMss) + ushort mss = DefaultMss, + UserSpaceTcpConnectOptions? connectOptions = null) { ArgumentNullException.ThrowIfNull(link); ArgumentNullException.ThrowIfNull(localAddress); @@ -66,6 +70,7 @@ public UserSpaceTcpClient( _remotePort = remotePort; _localPort = localPort ?? GenerateEphemeralPort(); _mss = mss; + _connectOptions = connectOptions ?? UserSpaceTcpConnectOptions.Default; _sender = new UserSpaceTcpSender( link, @@ -112,10 +117,10 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default) var synOptions = TcpCodec.EncodeMssOption(_mss); var synSeq = _sender.AllocateNextSequence(bytes: 1); - const int synRetries = 5; - var delay = TimeSpan.FromMilliseconds(250); + var delay = _connectOptions.InitialRetryDelay; + var deadline = Stopwatch.GetTimestamp() + (long)(_connectOptions.ConnectTimeout.TotalSeconds * Stopwatch.Frequency); - for (var attempt = 0; attempt <= synRetries; attempt++) + while (true) { cancellationToken.ThrowIfCancellationRequested(); @@ -137,14 +142,14 @@ await _sender } catch (TimeoutException) { - if (attempt >= synRetries) + if (Stopwatch.GetTimestamp() >= deadline) { break; } - delay = delay < TimeSpan.FromSeconds(2) + delay = delay < _connectOptions.MaxRetryDelay ? delay + delay - : TimeSpan.FromSeconds(2); + : _connectOptions.MaxRetryDelay; } } @@ -181,7 +186,7 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposeState, 1) != 0) { return; } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpConnectOptions.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpConnectOptions.cs new file mode 100644 index 0000000..11eb297 --- /dev/null +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpConnectOptions.cs @@ -0,0 +1,19 @@ +namespace ZTSharp.ZeroTier.Net; + +internal sealed class UserSpaceTcpConnectOptions +{ + public static UserSpaceTcpConnectOptions Default { get; } = new(); + + public static UserSpaceTcpConnectOptions DirectOnlyMultipath { get; } = new() + { + ConnectTimeout = TimeSpan.FromSeconds(30), + InitialRetryDelay = TimeSpan.FromMilliseconds(250), + MaxRetryDelay = TimeSpan.FromSeconds(2) + }; + + public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(8); + + public TimeSpan InitialRetryDelay { get; init; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan MaxRetryDelay { get; init; } = TimeSpan.FromSeconds(2); +} diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs index e2f52f9..3152a03 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpReceiveLoop.cs @@ -170,7 +170,7 @@ private bool TryParseAndFilterTcpPacket( } else { - if (!Ipv6Codec.TryParse(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) { return false; } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs index df1ff76..49ca57d 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpRemoteSendWindow.cs @@ -2,12 +2,12 @@ namespace ZTSharp.ZeroTier.Net; internal sealed class UserSpaceTcpRemoteSendWindow { - private ushort _window = ushort.MaxValue; + private int _window = ushort.MaxValue; private TaskCompletionSource? _windowTcs; private Exception? _terminalException; private readonly object _lock = new(); - public ushort Window => _window; + public ushort Window => (ushort)Volatile.Read(ref _window); public void Update(ushort windowSize) { @@ -28,9 +28,11 @@ public void Update(ushort windowSize) public void SignalWaiters(Exception exception) { TaskCompletionSource? toRelease = null; + Exception terminal; lock (_lock) { _terminalException ??= exception; + terminal = _terminalException; if (_windowTcs is not null) { toRelease = _windowTcs; @@ -38,7 +40,7 @@ public void SignalWaiters(Exception exception) } } - toRelease?.TrySetException(exception); + toRelease?.TrySetException(terminal); } public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) @@ -47,12 +49,13 @@ public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (_terminalException is not null) + var terminal = Volatile.Read(ref _terminalException); + if (terminal is not null) { - throw _terminalException; + throw terminal; } - if (_window != 0) + if (Volatile.Read(ref _window) != 0) { return; } @@ -73,7 +76,14 @@ public async Task WaitForNonZeroAsync(CancellationToken cancellationToken) tcs = _windowTcs ??= new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); } - await tcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await tcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (Volatile.Read(ref _terminalException) is not null && ex is not OperationCanceledException) + { + throw Volatile.Read(ref _terminalException)!; + } } } } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs index d7e823a..6dc9e76 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpSender.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; @@ -12,6 +13,10 @@ internal sealed class UserSpaceTcpSender : IAsyncDisposable private readonly UserSpaceTcpRemoteSendWindow _remoteSendWindow = new(); private readonly UserSpaceTcpRtoEstimator _rtoEstimator = new(); + [SuppressMessage( + "Reliability", + "CA2213:Disposable fields should be disposed", + Justification = "DisposeAsync must be idempotent; disposing this lock can throw in concurrent send paths.")] private readonly SemaphoreSlim _sendLock = new(1, 1); private int _effectiveMss; @@ -26,7 +31,8 @@ internal sealed class UserSpaceTcpSender : IAsyncDisposable private ushort _lastAdvertisedWindow = ushort.MaxValue; private readonly UserSpaceTcpWindowUpdateTrigger _windowUpdateTrigger; - private bool _disposed; + private int _disposeState; + private volatile bool _disposed; public UserSpaceTcpSender( IUserSpaceIpLink link, @@ -113,18 +119,20 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, Func getAck return; } - if (_finSeq is not null) - { - throw new IOException("Local has closed the connection."); - } - await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_finSeq is not null) + { + throw new IOException("Local has closed the connection."); + } + var remaining = buffer; while (!remaining.IsEmpty) { cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_disposed, this); await WaitForRemoteSendWindowAsync(cancellationToken).ConfigureAwait(false); @@ -190,6 +198,7 @@ public async Task SendFinWithRetriesAsync(uint ack, CancellationToken cancellati await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { + ObjectDisposedException.ThrowIf(_disposed, this); if (_finSeq is null) { _finSeq = AllocateNextSequence(bytes: 1); @@ -237,6 +246,12 @@ private async Task SendTcpWithRetriesAsync( try { + if (_disposed) + { + ackTcs.TrySetException(new ObjectDisposedException(nameof(UserSpaceTcpSender))); + return; + } + for (var attempt = 0; attempt < retries; attempt++) { if (UserSpaceTcpSequenceNumbers.GreaterThanOrEqual(_sendUna, expectedAck)) @@ -318,8 +333,13 @@ private async Task WaitForRemoteSendWindowAsync(CancellationToken cancellationTo public ValueTask DisposeAsync() { + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + { + return ValueTask.CompletedTask; + } + _disposed = true; - _sendLock.Dispose(); + FailPendingOperations(new ObjectDisposedException(nameof(UserSpaceTcpSender))); return ValueTask.CompletedTask; } } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs index d048d51..7c0a410 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerConnection.cs @@ -20,6 +20,7 @@ internal sealed class UserSpaceTcpServerConnection : IAsyncDisposable private Task? _receiveLoopTask; private bool _disposed; + private int _disposeState; public UserSpaceTcpServerConnection( IUserSpaceIpLink link, @@ -130,7 +131,7 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposeState, 1) != 0) { return; } diff --git a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs index 8c933f4..f0e155e 100644 --- a/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs +++ b/ZTSharp/ZeroTier/Net/UserSpaceTcpServerReceiveLoop.cs @@ -273,7 +273,7 @@ private bool TryParseAndFilterTcpPacket( } else { - if (!Ipv6Codec.TryParse(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) { return false; } diff --git a/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs b/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs index e910786..0df1f85 100644 --- a/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs +++ b/ZTSharp/ZeroTier/Net/ZeroTierFlowId.cs @@ -11,7 +11,7 @@ public static uint Derive(ReadOnlySpan ipPacket) return DeriveFromTransportTuple(src.GetAddressBytes(), dst.GetAddressBytes(), protocol, payload); } - if (Ipv6Codec.TryParse(ipPacket, out src, out dst, out var nextHeader, out _, out payload)) + if (Ipv6Codec.TryParseTransportPayload(ipPacket, out src, out dst, out var nextHeader, out _, out payload)) { return DeriveFromTransportTuple(src.GetAddressBytes(), dst.GetAddressBytes(), nextHeader, payload); } @@ -30,11 +30,17 @@ private static uint DeriveFromTransportTuple( if (protocol == TcpCodec.ProtocolNumber) { - _ = TcpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _, out _, out _, out _, out _); + if (!TcpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _, out _, out _, out _, out _)) + { + return 0; + } } else if (protocol == UdpCodec.ProtocolNumber) { - _ = UdpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _); + if (!UdpCodec.TryParse(transportPayload, out srcPort, out dstPort, out _)) + { + return 0; + } } else { diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs index 9a4e13f..912f564 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierInetAddressCodec.cs @@ -5,6 +5,9 @@ namespace ZTSharp.ZeroTier.Protocol; internal static class ZeroTierInetAddressCodec { + public static int GetSerializedLengthNullable(IPEndPoint? endpoint) + => endpoint is null ? 1 : GetSerializedLength(endpoint); + public static int GetSerializedLength(IPEndPoint endpoint) { ArgumentNullException.ThrowIfNull(endpoint); @@ -17,6 +20,22 @@ public static int GetSerializedLength(IPEndPoint endpoint) }; } + public static int SerializeNullable(IPEndPoint? endpoint, Span destination) + { + if (endpoint is null) + { + if (destination.IsEmpty) + { + throw new ArgumentException("Destination must be at least 1 byte.", nameof(destination)); + } + + destination[0] = 0; + return 1; + } + + return Serialize(endpoint, destination); + } + public static int Serialize(IPEndPoint endpoint, Span destination) { ArgumentNullException.ThrowIfNull(endpoint); @@ -127,4 +146,3 @@ public static bool TryDeserialize(ReadOnlySpan data, out IPEndPoint? endpo } } } - diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs index 206f101..8a70262 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierPacketCompression.cs @@ -42,7 +42,7 @@ public static bool TryUncompress(ReadOnlySpan packet, out byte[] uncompres } finally { - System.Buffers.ArrayPool.Shared.Return(payload, clearArray: false); + System.Buffers.ArrayPool.Shared.Return(payload, clearArray: true); } } } diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs index 608f173..856f424 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierPushDirectPathsCodec.cs @@ -9,6 +9,75 @@ internal readonly record struct ZeroTierPushedDirectPath( internal static class ZeroTierPushDirectPathsCodec { + public static byte[] BuildPayload(IEnumerable endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + var normalized = endpoints + .Where(static ep => ep is not null && ep.Port != 0) + .Select(static ep => + { + var address = ep.Address.IsIPv4MappedToIPv6 ? ep.Address.MapToIPv4() : ep.Address; + return new IPEndPoint(address, ep.Port); + }) + .Distinct() + .Take(ZeroTierProtocolLimits.MaxPushedDirectPaths) + .ToArray(); + + var payloadLength = 2; + for (var i = 0; i < normalized.Length; i++) + { + payloadLength += normalized[i].AddressFamily switch + { + System.Net.Sockets.AddressFamily.InterNetwork => 1 + 2 + 1 + 1 + 6, + System.Net.Sockets.AddressFamily.InterNetworkV6 => 1 + 2 + 1 + 1 + 18, + _ => 0 + }; + } + + var payload = new byte[payloadLength]; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(0, 2), (ushort)normalized.Length); + + var ptr = 2; + for (var i = 0; i < normalized.Length; i++) + { + var endpoint = normalized[i]; + payload[ptr++] = 0; // flags + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), 0); // extension length + ptr += 2; + + if (endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + payload[ptr++] = 4; + payload[ptr++] = 6; + endpoint.Address.GetAddressBytes().CopyTo(payload, ptr); + ptr += 4; + } + else if (endpoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + payload[ptr++] = 6; + payload[ptr++] = 18; + endpoint.Address.GetAddressBytes().CopyTo(payload, ptr); + ptr += 16; + } + else + { + continue; + } + + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(ptr, 2), (ushort)endpoint.Port); + ptr += 2; + } + + if (ptr != payload.Length) + { + Array.Resize(ref payload, ptr); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(0, 2), (ushort)Math.Min(normalized.Length, ZeroTierProtocolLimits.MaxPushedDirectPaths)); + } + + return payload; + } + public static bool TryParse(ReadOnlySpan payload, out ZeroTierPushedDirectPath[] paths) { if (payload.Length < 2) diff --git a/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs b/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs index ada7f5b..d7087e2 100644 --- a/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs +++ b/ZTSharp/ZeroTier/Protocol/ZeroTierRendezvousCodec.cs @@ -13,6 +13,28 @@ internal static class ZeroTierRendezvousCodec private const int AddressLength = 5; private const int MinPayloadLength = 1 + AddressLength + 2 + 1; + public static byte[] BuildPayload(NodeId with, IPEndPoint endpoint, byte flags = 0) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var address = endpoint.Address.IsIPv4MappedToIPv6 + ? endpoint.Address.MapToIPv4() + : endpoint.Address; + var addressBytes = address.GetAddressBytes(); + if (endpoint.Port == 0 || (addressBytes.Length != 4 && addressBytes.Length != 16)) + { + throw new ArgumentOutOfRangeException(nameof(endpoint), endpoint, "Rendezvous endpoint must be IPv4 or IPv6 with a non-zero port."); + } + + var payload = new byte[MinPayloadLength + addressBytes.Length]; + payload[0] = flags; + ZeroTierBinaryPrimitives.WriteUInt40BigEndian(payload.AsSpan(1, AddressLength), with.Value); + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(1 + AddressLength, 2), (ushort)endpoint.Port); + payload[1 + AddressLength + 2] = (byte)addressBytes.Length; + addressBytes.CopyTo(payload.AsSpan(MinPayloadLength)); + return payload; + } + public static bool TryParse(ReadOnlySpan payload, out ZeroTierRendezvous rendezvous) { if (payload.Length < MinPayloadLength) diff --git a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs index 8abd97f..2f8a125 100644 --- a/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs +++ b/ZTSharp/ZeroTier/Sockets/ManagedTcpSocketBackend.cs @@ -60,6 +60,11 @@ public override async ValueTask BindAsync(EndPoint localEndPoint, CancellationTo { ObjectDisposedException.ThrowIf(Disposed, this); + if (Volatile.Read(ref _connectInProgress) != 0) + { + throw new InvalidOperationException("Socket connect is in progress."); + } + if (_stream is not null || _listener is not null) { throw new InvalidOperationException("Socket is already initialized."); @@ -85,6 +90,11 @@ public override async ValueTask ListenAsync(int backlog, CancellationToken cance { ObjectDisposedException.ThrowIf(Disposed, this); + if (Volatile.Read(ref _connectInProgress) != 0) + { + throw new InvalidOperationException("Socket connect is in progress."); + } + if (_listener is not null) { throw new InvalidOperationException("Socket is already listening."); @@ -188,7 +198,15 @@ public override async ValueTask ConnectAsync(EndPoint remoteEndPoint, Cancellati ? await Zt.ConnectTcpWithLocalEndpointAsync(remoteIp, token).ConfigureAwait(false) : await Zt.ConnectTcpWithLocalEndpointAsync(local, remoteIp, token).ConfigureAwait(false); - await InitLock.WaitAsync(token).ConfigureAwait(false); + try + { + await InitLock.WaitAsync(token).ConfigureAwait(false); + } + catch + { + await stream.DisposeAsync().ConfigureAwait(false); + throw; + } try { ObjectDisposedException.ThrowIf(Disposed, this); @@ -206,9 +224,14 @@ public override async ValueTask ConnectAsync(EndPoint remoteEndPoint, Cancellati InitLock.Release(); } } - catch (OperationCanceledException) when (ShutdownToken.IsCancellationRequested) + catch (OperationCanceledException ex) { - throw new ObjectDisposedException(GetType().FullName); + if (ShutdownToken.IsCancellationRequested || Disposed) + { + throw new ObjectDisposedException(GetType().FullName, ex); + } + + throw; } finally { diff --git a/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs index 07f1493..baa66e9 100644 --- a/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/IZeroTierUdpTransport.cs @@ -13,5 +13,12 @@ internal interface IZeroTierUdpTransport : IAsyncDisposable Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default); Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default); + + Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default); } diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs index 0c7dc4c..8bf9582 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpMultiTransport.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Diagnostics; using System.Threading.Channels; using ZTSharp.ZeroTier.Internal; @@ -10,7 +11,7 @@ internal sealed class ZeroTierUdpMultiTransport : IZeroTierUdpTransport private readonly Channel _incoming; private readonly CancellationTokenSource _cts = new(); private readonly Task[] _forwarders; - private bool _disposed; + private int _disposed; public ZeroTierUdpMultiTransport(IReadOnlyList sockets) { @@ -37,57 +38,116 @@ public ZeroTierUdpMultiTransport(IReadOnlyList sockets) } public IReadOnlyList LocalSockets - => _sockets.Select(socket => new ZeroTierUdpLocalSocket(socket.LocalSocketId, socket.LocalEndpoint)).ToArray(); + { + get + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + return _sockets.Select(socket => new ZeroTierUdpLocalSocket(socket.LocalSocketId, socket.LocalEndpoint)).ToArray(); + } + } public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return _incoming.Reader.ReadAsync(cancellationToken); } public async ValueTask ReceiveAsync(TimeSpan timeout, CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return await ZeroTierTimeouts .RunWithTimeoutAsync(timeout, operation: "UDP receive", _incoming.Reader.ReadAsync, cancellationToken) .ConfigureAwait(false); } public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) - => _sockets[0].SendAsync(remoteEndpoint, payload, cancellationToken); + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + return SendAllAsync(remoteEndpoint, payload, cancellationToken); + } public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); var socket = GetSocket(localSocketId); return socket.SendAsync(remoteEndpoint, payload, cancellationToken); } + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + var socket = GetSocket(localSocketId); + return socket.SendWithHopLimitAsync(localSocketId, remoteEndpoint, payload, hopLimit, cancellationToken); + } + public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } - _disposed = true; _incoming.Writer.TryComplete(); - await _cts.CancelAsync().ConfigureAwait(false); + var forwarderCompletion = Task.WhenAll(_forwarders); try { - await Task.WhenAll(_forwarders).ConfigureAwait(false); - } - catch (Exception ex) when (ex is OperationCanceledException or ChannelClosedException) - { + try + { + await _cts.CancelAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] CancelAsync failed: {ex}"); +#else + _ = ex; +#endif + } + + foreach (var socket in _sockets) + { + try + { + await socket.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] Socket dispose failed: {ex}"); +#else + _ = ex; +#endif + } + } } finally { - _cts.Dispose(); - } + try + { + await forwarderCompletion.ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] Forwarder completion failed: {ex}"); +#else + _ = ex; +#endif + } - foreach (var socket in _sockets) - { - await socket.DisposeAsync().ConfigureAwait(false); + _cts.Dispose(); } } @@ -105,6 +165,35 @@ private ZeroTierUdpTransport GetSocket(int localSocketId) throw new ArgumentOutOfRangeException(nameof(localSocketId), localSocketId, "Unknown local socket id."); } + private async Task SendAllAsync( + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + CancellationToken cancellationToken) + { + var sent = false; + for (var i = 0; i < _sockets.Count; i++) + { + try + { + await _sockets[i].SendAsync(remoteEndpoint, payload, cancellationToken).ConfigureAwait(false); + sent = true; + } + catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException) + { + throw; + } + catch (Exception ex) when (ex is System.Net.Sockets.SocketException or InvalidOperationException) + { + Debug.WriteLine($"[{nameof(ZeroTierUdpMultiTransport)}] SendAsync failed on socket {i}: {ex}"); + } + } + + if (!sent) + { + throw new InvalidOperationException("All UDP socket sends failed."); + } + } + private async Task ForwardLoopAsync(ZeroTierUdpTransport socket, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) @@ -139,6 +228,10 @@ private async Task ForwardLoopAsync(ZeroTierUdpTransport socket, CancellationTok { return; } + catch (ObjectDisposedException) + { + return; + } } } } diff --git a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs index cb3abb4..f02cc7f 100644 --- a/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs +++ b/ZTSharp/ZeroTier/Transport/ZeroTierUdpTransport.cs @@ -1,5 +1,6 @@ using System.Net; using System.Net.Sockets; +using System.Diagnostics; using System.Threading.Channels; using ZTSharp.Transport.Internal; using ZTSharp.ZeroTier.Internal; @@ -14,14 +15,20 @@ internal sealed class ZeroTierUdpTransport : IZeroTierUdpTransport private readonly CancellationTokenSource _cts = new(); private readonly Task _receiverLoop; private readonly int _localSocketId; + private readonly SemaphoreSlim _sendGate = new(1, 1); private long _incomingBackpressureCount; - private bool _disposed; + private int _disposed; - public ZeroTierUdpTransport(int localPort = 0, bool enableIpv6 = true, Action? log = null, int localSocketId = 0) + public ZeroTierUdpTransport( + int localPort = 0, + bool enableIpv6 = true, + Action? log = null, + int localSocketId = 0, + IPAddress? localBindAddress = null) { _log = log; _localSocketId = localSocketId; - _udp = OsUdpSocketFactory.Create(localPort, enableIpv6, Log); + _udp = OsUdpSocketFactory.Create(localPort, enableIpv6, Log, localBindAddress); _incoming = Channel.CreateBounded(new BoundedChannelOptions(capacity: 2048) { @@ -36,13 +43,19 @@ public ZeroTierUdpTransport(int localPort = 0, bool enableIpv6 = true, Action UdpEndpointNormalization.Normalize((IPEndPoint)_udp.Client.LocalEndPoint!); public IReadOnlyList LocalSockets - => new[] { new ZeroTierUdpLocalSocket(_localSocketId, LocalEndpoint) }; + { + get + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + return new[] { new ZeroTierUdpLocalSocket(_localSocketId, LocalEndpoint) }; + } + } public long IncomingBackpressureCount => Interlocked.Read(ref _incomingBackpressureCount); public ValueTask ReceiveAsync(CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return _incoming.Reader.ReadAsync(cancellationToken); } @@ -50,18 +63,33 @@ public async ValueTask ReceiveAsync( TimeSpan timeout, CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); return await ZeroTierTimeouts .RunWithTimeoutAsync(timeout, operation: "UDP receive", _incoming.Reader.ReadAsync, cancellationToken) .ConfigureAwait(false); } public Task SendAsync(IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) + => SendCoreAsync(remoteEndpoint, payload, hopLimit: null, cancellationToken); + + public Task SendWithHopLimitAsync( + int localSocketId, + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int hopLimit, + CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(remoteEndpoint); - ObjectDisposedException.ThrowIf(_disposed, this); - UdpEndpointNormalization.ValidateRemoteEndpoint(remoteEndpoint, nameof(remoteEndpoint)); - return _udp.SendAsync(payload, remoteEndpoint, cancellationToken).AsTask(); + if (localSocketId != _localSocketId) + { + throw new ArgumentOutOfRangeException(nameof(localSocketId), localSocketId, $"Invalid local socket id. This transport exposes only id {_localSocketId}."); + } + + if (hopLimit <= 0) + { + throw new ArgumentOutOfRangeException(nameof(hopLimit), hopLimit, "Hop limit must be positive."); + } + + return SendCoreAsync(remoteEndpoint, payload, hopLimit, cancellationToken); } public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemory payload, CancellationToken cancellationToken = default) @@ -71,21 +99,89 @@ public Task SendAsync(int localSocketId, IPEndPoint remoteEndpoint, ReadOnlyMemo throw new ArgumentOutOfRangeException(nameof(localSocketId), localSocketId, $"Invalid local socket id. This transport exposes only id {_localSocketId}."); } - return SendAsync(remoteEndpoint, payload, cancellationToken); + return SendCoreAsync(remoteEndpoint, payload, hopLimit: null, cancellationToken); + } + + private async Task SendCoreAsync( + IPEndPoint remoteEndpoint, + ReadOnlyMemory payload, + int? hopLimit, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(remoteEndpoint); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + UdpEndpointNormalization.ValidateRemoteEndpoint(remoteEndpoint, nameof(remoteEndpoint)); + + await _sendGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + SocketOptionState? optionState = null; + if (hopLimit is int limit) + { + optionState = TryCreateSocketOptionState(remoteEndpoint, limit); + optionState?.Apply(_udp.Client); + } + + try + { + await _udp.SendAsync(payload, remoteEndpoint, cancellationToken).ConfigureAwait(false); + } + finally + { + optionState?.Restore(_udp.Client); + } + } + finally + { + _sendGate.Release(); + } } public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } - _disposed = true; - await _cts.CancelAsync().ConfigureAwait(false); - _udp.Dispose(); _incoming.Writer.TryComplete(); - _cts.Dispose(); + try + { + try + { + await _cts.CancelAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] CancelAsync failed: {ex}"); +#else + _ = ex; +#endif + } + + try + { + _udp.Dispose(); + } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] UdpClient dispose failed: {ex}"); +#else + _ = ex; +#endif + } + } + finally + { + _cts.Dispose(); + _sendGate.Dispose(); + } try { @@ -94,6 +190,16 @@ public async ValueTask DisposeAsync() catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException) { } +#pragma warning disable CA1031 // Dispose must be best-effort. + catch (Exception ex) +#pragma warning restore CA1031 + { +#if DEBUG + Debug.WriteLine($"[{nameof(ZeroTierUdpTransport)}] Receiver loop completion failed: {ex}"); +#else + _ = ex; +#endif + } } private async Task ProcessReceiveLoopAsync() @@ -125,27 +231,21 @@ private async Task ProcessReceiveLoopAsync() continue; } - if (!_incoming.Writer.TryWrite(new ZeroTierUdpDatagram( - _localSocketId, - UdpEndpointNormalization.Normalize(result.RemoteEndPoint), - result.Buffer))) + var datagram = new ZeroTierUdpDatagram( + _localSocketId, + UdpEndpointNormalization.Normalize(result.RemoteEndPoint), + result.Buffer); + if (!_incoming.Writer.TryWrite(datagram)) { - try + Interlocked.Increment(ref _incomingBackpressureCount); + if (_incoming.Writer.TryWrite(datagram)) { - var write = _incoming.Writer.WriteAsync(new ZeroTierUdpDatagram( - _localSocketId, - UdpEndpointNormalization.Normalize(result.RemoteEndPoint), - result.Buffer), token); - if (!write.IsCompletedSuccessfully) - { - Interlocked.Increment(ref _incomingBackpressureCount); - } - - await write.ConfigureAwait(false); + continue; } - catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException or ChannelClosedException) + + if (_incoming.Reader.TryRead(out _)) { - return; + _incoming.Writer.TryWrite(datagram); } } } @@ -155,4 +255,79 @@ private void Log(string message) { _log?.Invoke(message); } + + private SocketOptionState? TryCreateSocketOptionState(IPEndPoint remoteEndpoint, int hopLimit) + { + var socket = _udp.Client; + var addressFamily = remoteEndpoint.AddressFamily == AddressFamily.InterNetworkV6 && remoteEndpoint.Address.IsIPv4MappedToIPv6 + ? AddressFamily.InterNetwork + : remoteEndpoint.AddressFamily; + + try + { + return addressFamily switch + { + AddressFamily.InterNetwork => TryCreateSocketOptionState( + socket, + SocketOptionLevel.IP, + SocketOptionName.IpTimeToLive, + hopLimit), + AddressFamily.InterNetworkV6 => TryCreateSocketOptionState( + socket, + SocketOptionLevel.IPv6, + SocketOptionName.HopLimit, + hopLimit), + _ => null + }; + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or NotSupportedException) + { + return null; + } + } + + private static SocketOptionState? TryCreateSocketOptionState( + Socket socket, + SocketOptionLevel level, + SocketOptionName name, + int hopLimit) + { + var option = socket.GetSocketOption(level, name); + if (option is not int originalValue) + { + return null; + } + + return new SocketOptionState(level, name, originalValue, hopLimit); + } + + private sealed class SocketOptionState + { + private readonly SocketOptionLevel _level; + private readonly SocketOptionName _name; + private readonly int _originalValue; + private readonly int _temporaryValue; + + public SocketOptionState(SocketOptionLevel level, SocketOptionName name, int originalValue, int temporaryValue) + { + _level = level; + _name = name; + _originalValue = originalValue; + _temporaryValue = temporaryValue; + } + + public void Apply(Socket socket) + => socket.SetSocketOption(_level, _name, _temporaryValue); + + public void Restore(Socket socket) + { + try + { + socket.SetSocketOption(_level, _name, _originalValue); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException or NotSupportedException) + { + } + } + } } diff --git a/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs b/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs index 89c2efb..cab71b2 100644 --- a/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs +++ b/ZTSharp/ZeroTier/ZeroTierMultipathOptions.cs @@ -28,6 +28,12 @@ public sealed class ZeroTierMultipathOptions /// When true, unconfirmed direct paths are "warmed up" by duplicating outbound packets to the root relay as a fallback. /// public bool WarmupDuplicateToRoot { get; init; } = true; + + /// + /// When false, payload traffic must use a negotiated direct peer path and will not be relayed via the root. + /// Root/controller traffic needed for join, address resolution, and path negotiation still uses the root. + /// + public bool AllowRootRelayFallback { get; init; } = true; } public enum ZeroTierBondPolicy diff --git a/ZTSharp/ZeroTier/ZeroTierSocket.cs b/ZTSharp/ZeroTier/ZeroTierSocket.cs index 5f9dcce..ac90bad 100644 --- a/ZTSharp/ZeroTier/ZeroTierSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierSocket.cs @@ -44,6 +44,11 @@ internal ZeroTierSocket(ZeroTierSocketOptions options, string statePath, ZeroTie public IReadOnlyList ManagedIps { get; private set; } = Array.Empty(); + internal TimeSpan SuggestedHttpConnectTimeout + => _options.Multipath.Enabled && !_options.Multipath.AllowRootRelayFallback + ? TimeSpan.FromSeconds(120) + : TimeSpan.FromSeconds(10); + public static Task CreateAsync( ZeroTierSocketOptions options, CancellationToken cancellationToken = default) @@ -57,6 +62,7 @@ public async Task JoinAsync(CancellationToken cancellationToken = default) using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdown.Token); var token = linkedCts.Token; + Task joinTask; try { await _joinLock.WaitAsync(token).ConfigureAwait(false); @@ -79,6 +85,7 @@ public async Task JoinAsync(CancellationToken cancellationToken = default) } _joinTask ??= JoinCoreAsync(_shutdown.Token); + joinTask = _joinTask; } finally { @@ -87,7 +94,7 @@ public async Task JoinAsync(CancellationToken cancellationToken = default) try { - await _joinTask.WaitAsync(token).ConfigureAwait(false); + await joinTask.WaitAsync(token).ConfigureAwait(false); } catch (OperationCanceledException) when (_shutdown.IsCancellationRequested) { @@ -355,6 +362,7 @@ private async Task GetOrCreateRuntimeAsync( using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdown.Token); var token = linkedCts.Token; + Task runtimeTask; try { await _runtimeLock.WaitAsync(token).ConfigureAwait(false); @@ -377,6 +385,7 @@ private async Task GetOrCreateRuntimeAsync( } _runtimeTask ??= CreateRuntimeAsync(inlineCom, _shutdown.Token); + runtimeTask = _runtimeTask; } finally { @@ -385,7 +394,7 @@ private async Task GetOrCreateRuntimeAsync( try { - return await _runtimeTask.WaitAsync(token).ConfigureAwait(false); + return await runtimeTask.WaitAsync(token).ConfigureAwait(false); } catch (OperationCanceledException) when (_shutdown.IsCancellationRequested) { @@ -408,39 +417,66 @@ private async Task CreateRuntimeAsync(byte[] inlineCom cancellationToken) .ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); - ThrowIfDisposed(); - - ZeroTierDataplaneRuntime? toDispose = null; - ZeroTierDataplaneRuntime runtime; - await _runtimeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + var createdNeedsDispose = true; try { + cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); - if (_runtime is not null) + + ZeroTierDataplaneRuntime? toDispose = null; + ZeroTierDataplaneRuntime runtime; + var published = false; + await _runtimeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - toDispose = created; - runtime = _runtime; + ThrowIfDisposed(); + if (_runtime is not null) + { + toDispose = created; + runtime = _runtime; + } + else + { + _upstreamRoot ??= helloOk; + _upstreamRootKey ??= rootKey; + _runtime = created; + runtime = created; + createdNeedsDispose = false; + published = true; + } } - else + finally { - _upstreamRoot ??= helloOk; - _upstreamRootKey ??= rootKey; - _runtime = created; - runtime = created; + _runtimeLock.Release(); } + + if (toDispose is not null) + { + await toDispose.DisposeAsync().ConfigureAwait(false); + createdNeedsDispose = false; + } + + if (published && Volatile.Read(ref _disposeState) != 0) + { + if (ReferenceEquals(Interlocked.CompareExchange(ref _runtime, null, created), created)) + { + await created.DisposeAsync().ConfigureAwait(false); + } + + throw new ObjectDisposedException(nameof(ZeroTierSocket)); + } + + return runtime; } - finally + catch { - _runtimeLock.Release(); - } + if (createdNeedsDispose) + { + await created.DisposeAsync().ConfigureAwait(false); + } - if (toDispose is not null) - { - await toDispose.DisposeAsync().ConfigureAwait(false); + throw; } - - return runtime; } private void ThrowIfDisposed() diff --git a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs index c90fd9e..c6bd735 100644 --- a/ZTSharp/ZeroTier/ZeroTierTcpListener.cs +++ b/ZTSharp/ZeroTier/ZeroTierTcpListener.cs @@ -103,7 +103,15 @@ public async ValueTask DisposeAsync() while (_acceptQueue.Reader.TryRead(out var accepted)) { Interlocked.Decrement(ref _pendingAcceptCount); - await accepted.Stream.DisposeAsync().ConfigureAwait(false); + try + { + await accepted.Stream.DisposeAsync().ConfigureAwait(false); + } +#pragma warning disable CA1031 // Dispose is best-effort during shutdown/drain. + catch +#pragma warning restore CA1031 + { + } } using var drainCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); @@ -151,12 +159,12 @@ private Task OnSynAsync(NodeId peerNodeId, ReadOnlyMemory ipPacket, Cancel } else { - if (!Ipv6Codec.TryParse(ipPacket.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(ipPacket.Span, out src, out dst, out var protocol, out _, out ipPayload)) { return Task.CompletedTask; } - if ((!_isWildcardBind && !ZeroTierIpAddressCanonicalization.EqualsForManagedIpComparison(dst, _localAddressMatch)) || nextHeader != TcpCodec.ProtocolNumber) + if ((!_isWildcardBind && !ZeroTierIpAddressCanonicalization.EqualsForManagedIpComparison(dst, _localAddressMatch)) || protocol != TcpCodec.ProtocolNumber) { return Task.CompletedTask; } @@ -211,8 +219,10 @@ private async Task HandleAcceptedConnectionAsync( await connection.AcceptAsync(token).ConfigureAwait(false); stream = connection.GetStream(); + Interlocked.Increment(ref _pendingAcceptCount); if (!_acceptQueue.Writer.TryWrite(new AcceptedTcpConnection(stream, localEndpoint, remoteEndpoint))) { + Interlocked.Decrement(ref _pendingAcceptCount); Interlocked.Increment(ref _droppedAcceptCount); if (ZeroTierTrace.Enabled) { @@ -225,7 +235,6 @@ private async Task HandleAcceptedConnectionAsync( return; } - Interlocked.Increment(ref _pendingAcceptCount); handedOff = true; stream = null; } diff --git a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs index b1b7975..9d5bc7f 100644 --- a/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs +++ b/ZTSharp/ZeroTier/ZeroTierUdpSocket.cs @@ -10,8 +10,15 @@ namespace ZTSharp.ZeroTier; public sealed class ZeroTierUdpSocket : IAsyncDisposable { + private const int MaxQueuedDatagrams = 1024; + private readonly SemaphoreSlim _disposeLock = new(1, 1); - private readonly Channel _incoming = Channel.CreateUnbounded(); + private readonly Channel _incoming = Channel.CreateBounded(new BoundedChannelOptions(MaxQueuedDatagrams) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleWriter = false, + SingleReader = true + }); private readonly ZeroTierDataplaneRuntime _runtime; private readonly IPAddress _localAddress; private readonly ushort _localPort; @@ -133,24 +140,31 @@ public async ValueTask ReceiveFromAsync( continue; } - if (!dst.Equals(_localAddress) || protocol != UdpCodec.ProtocolNumber) + if (protocol != UdpCodec.ProtocolNumber) { continue; } } else { - if (!Ipv6Codec.TryParse(routed.Packet.Span, out src, out dst, out var nextHeader, out _, out ipPayload)) + if (!Ipv6Codec.TryParseTransportPayload(routed.Packet.Span, out src, out dst, out var protocol, out _, out ipPayload)) { continue; } - if (!dst.Equals(_localAddress) || nextHeader != UdpCodec.ProtocolNumber) + if (protocol != UdpCodec.ProtocolNumber) { continue; } } + if (!_localAddress.Equals(IPAddress.Any) && + !_localAddress.Equals(IPAddress.IPv6Any) && + !dst.Equals(_localAddress)) + { + continue; + } + if (!UdpCodec.TryParse(ipPayload, out srcPort, out dstPort, out udpPayload)) { continue; diff --git a/docs/ZEROTIER_SOCKETS.md b/docs/ZEROTIER_SOCKETS.md index fd039f2..01e5770 100644 --- a/docs/ZEROTIER_SOCKETS.md +++ b/docs/ZEROTIER_SOCKETS.md @@ -131,7 +131,9 @@ There is no virtual network interface visible to the operating system. - `ZeroTierSocket.ListenTcpAsync(IPAddress.IPv6Any, port)` / `ManagedSocket.BindAsync(IPAddress.IPv6Any, port)` + `ListenAsync(...)` create a wildcard listener that accepts connections addressed to any of the node's managed IPv6s. - TCP connect may use `IPAddress.Any` / `IPAddress.IPv6Any` as the local endpoint to mean "choose a managed IP" (the selected local endpoint is returned by APIs that surface it). - TCP listen with port `0` is **not supported** (`NotSupportedException`). -- UDP bind supports port `0` (ephemeral); the bound local port is assigned by the OS and can be read back from the socket. +- UDP bind supports port `0` (ephemeral); the bound local port is selected internally via `ZeroTierEphemeralPorts.Generate()` and can be read back from the socket. +- UDP bind is currently **per address family + port** (not per IP + port). A UDP socket may receive datagrams destined to any of the node's managed IPs on that port. +- `ReceiveFromAsync(...)` does not currently surface the local destination IP. ### Socket Options @@ -154,3 +156,5 @@ Accepted sockets populate `ManagedSocket.RemoteEndPoint` (peer IP/port). The user-space TCP stack is correctness-oriented. It currently lacks OS optimizations like congestion control and high-throughput sender pipelines. Large transfers may be significantly slower than OS TCP. + +UDP receive queues are bounded and may drop datagrams under load. diff --git a/docs/ZEROTIER_STACK.md b/docs/ZEROTIER_STACK.md index 14da5aa..2891e3d 100644 --- a/docs/ZEROTIER_STACK.md +++ b/docs/ZEROTIER_STACK.md @@ -64,6 +64,7 @@ Managed stack CLI commands: `call`, `listen`, `udp-listen`, `udp-send`. To enable experimental multipath/bonding for the managed stack, use the CLI flags: `--multipath`, `--mp-bond`, `--mp-udp-sockets`, `--mp-udp-ports`, `--mp-warmup-root`. +Constraints: `--mp-udp-sockets` must be in `1..8`. If `--mp-udp-ports` is set, it must include exactly that many entries, each in `0..65535` (`0` = ephemeral). --- diff --git a/docs/logbook/direct-p2p-jaeger.md b/docs/logbook/direct-p2p-jaeger.md new file mode 100644 index 0000000..8c3b608 --- /dev/null +++ b/docs/logbook/direct-p2p-jaeger.md @@ -0,0 +1,1031 @@ +# Direct P2P Jaeger Logbook + +Append-only record for direct-P2P work against `https://jaeger.pdcs.kamsker.at/`. +New notes are appended with timestamps. Older entries are not rewritten. + +## 2026-03-16T14:35:52.4539008+01:00 + +- Created this logbook to keep an append-only protocol of direct-P2P investigation and wire-level results. +- Current branch head at creation time: `1f12e26`. +- Current definition of done remains unchanged: payload to and from `https://jaeger.pdcs.kamsker.at/` must flow hop-0 without relay-forwarded data. + +## 2026-03-16T14:36:00+01:00 + +- Checkpoint `93c9c07` added managed handling for authenticated peer `RENDEZVOUS` control and tests for that path. +- Why it was tried: + - the runtime was already handling peer `PUSH_DIRECT_PATHS` + - adding peer `RENDEZVOUS` looked like one more way to seed direct endpoint bootstrap on the receiving socket +- Issue update at that time: + - live runs still showed no `RX direct raw`, no `RX OK(ECHO)`, no `hop=0` + - the peer only replied with relayed `OK(HELLO)` via `84.17.53.155:9993` + +## 2026-03-16T14:36:10+01:00 + +- Checkpoint `9052307` broadened relayed `RENDEZVOUS` advertisement across all public local surfaces instead of a single public root-affine surface. +- Why it was tried: + - the peer only ever saw one relayed introduction surface from us + - widening that set was the last obvious local lever for peer introduction breadth +- Issue update after the live rerun: + - the runtime emitted relayed `RENDEZVOUS` for all 8 public surfaces + - there was still no `RX peer RENDEZVOUS`, no `RX OK(ECHO)`, no `hop=0` + - the HTTPS call still timed out waiting for TCP SYN-ACK + +## 2026-03-16T14:36:20+01:00 + +- Upstream review of `external/libzt/ext/ZeroTierOne/node/IncomingPacket.cpp` and `Switch.cpp` changed the understanding of the blocker. +- Issue change: + - upstream only honors `VERB_RENDEZVOUS` from an upstream peer + - upstream `introduce()` is triggered by the relay/switch when it forwards packets between peers + - this means the relayed peer-`RENDEZVOUS` path added locally was not actually upstream-compatible and was likely ignored by the remote side + +## 2026-03-16T14:36:30+01:00 + +- Checkpoint `1f12e26` restored upstream direct-rendezvous semantics. +- What changed: + - removed relayed peer `RENDEZVOUS` sending + - removed managed peer `RENDEZVOUS` control handling + - restored direct-only relayed root control fanout across sockets until a confirmed hop-0 path exists +- Why it was tried: + - to realign with vendored upstream behavior + - to keep root seeing traffic from all local public ports instead of prematurely pinning to one observed root socket +- Current issue state: + - direct P2P is still not established + - the remaining live-wire blocker is unchanged: the remote peer still does not return confirmed hop-0 traffic to this runtime + +## 2026-03-16T14:40:55.8501212+01:00 + +- After restoring upstream semantics, ordinary relayed `PUSH_DIRECT_PATHS` still looked over-pinned to the one receiving root socket. +- Checkpoint in progress: + - ordinary pushed hints were changed to bootstrap on all admissible local sockets again + - strict socket affinity is now reserved for root `RENDEZVOUS` / redirect style hints, not ordinary pushed hints +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `ZeroTierDirectEndpointManagerSocketAffinityTests` + - `ZeroTierDirectHintPathPlannerTests` +- Live result in `.codex-debug/direct-only-post-ordinary-hint-fanout.log`: + - ordinary public hints like `176.66.90.119:19665`, `:8955`, `:45185`, `:35838`, `:27605`, `:62120`, `:8140` now fan out across sockets `0..7` + - private/shared hints like `100.85.196.109:*`, `10.22.10.94:*`, `172.17.0.1:*`, `172.18.0.1:*` also fan out across sockets `0..7` + - direct-only payload fanout now follows that wider matrix instead of staying pinned to one `(endpoint, socket)` path +- Issue update: + - this materially widened direct probing, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the run still ended with `Timed out waiting for TCP SYN-ACK` + - current blocker remains at the network edge: even after all-socket ordinary-hint fanout, the peer still only responds through the root relay + +## 2026-03-16T14:51:26.6980465+01:00 + +- Checkpoint in progress: + - ordinary pushed hints no longer mean immediate all-socket spray + - normal `PUSH_DIRECT_PATHS` now bootstrap on one planner-selected admissible socket at a time unless the hint carries real socket affinity + - root `RENDEZVOUS` stays pinned to its receiving socket, and direct-only payload still clamps to the pinned rendezvous path +- Why it was tried: + - vendored upstream ordinary hint handling uses an unspecified local socket, which is closer to "let the node choose one route" than "fan out to every socket immediately" + - the previous all-socket spray had become broad enough that it was obscuring whether a narrower route-selected hint path behaved any better +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `ZeroTierDirectHintPathPlannerTests` + - `ZeroTierDirectHintPathPlannerRouteTests` + - there was one transient stale-binary issue during rerun: + - `ZTSharp.Cli` still held `samples/ZTSharp.Cli/bin/Release/net10.0/ZTSharp.dll` open + - the first live run after the code change used the old binary and was discarded + - after the stale process exited, the CLI rebuilt cleanly and the fresh-binary run was used for evaluation +- Live result in `.codex-debug/direct-only-post-rotating-ordinary-hints.log`: + - ordinary alternate hints like `176.66.90.119:19665`, `:27568`, `:44886`, `:48446`, `:8140`, `100.85.196.109:*`, `10.22.10.94:*`, `172.17.0.1:*`, and `172.18.0.1:*` now probe on one socket at a time instead of sockets `0..7` + - the selected socket rotated between update waves, for example socket `0` on one push batch and socket `1` on the next + - root `RENDEZVOUS` remained pinned to `176.66.90.119:62120@3`, and direct-only payload stayed clamped to that single path +- Issue update: + - this successfully narrowed ordinary hint behavior, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer still only returned relayed `OK(HELLO)` and relayed `MulticastLike` traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - current blocker remains unchanged at the wire level: even with route-selected ordinary hint probing and pinned rendezvous payload, the remote peer still does not return confirmed direct traffic to this runtime + +## 2026-03-16T14:55:13.0296475+01:00 + +- Checkpoint in progress: + - once direct-only mode has a pinned root `RENDEZVOUS` endpoint, alternate pushed hints are now skipped entirely instead of still being probed + - direct-only post-rendezvous behavior is now strictly rendezvous-only for both bootstrap and payload +- Why it was tried: + - the previous run still spent time probing alternate public/private/shared hinted endpoints even after root had already introduced a specific rendezvous endpoint + - that extra hint traffic was the clearest remaining local source of bootstrap noise after payload clamping +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `ZeroTierDirectHintPathPlannerTests` + - `ZeroTierDirectHintPathPlannerRouteTests` +- Live result in `.codex-debug/direct-only-post-rendezvous-only.log`: + - before root sent `RENDEZVOUS`, the peer first sent `PUSH_DIRECT_PATHS`, so the runtime still tried the ordinary hinted set across rotating single sockets and direct-only payload fanout still followed that broad hinted matrix + - later root sent `RENDEZVOUS` for `176.66.90.119:62120` + - after that, the new behavior activated exactly as intended: + - repeated `Direct hint bootstrap skipped ... mode=alternate-hint payloadPinned=True` + - direct-only payload collapsed to `176.66.90.119:62120@7` + - only same-socket hole-punch / `HELLO` / `ECHO` remained on that pinned rendezvous path +- Issue update: + - this removed the post-rendezvous alternate-hint noise, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer still only returned relayed `OK(HELLO)` via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - the blocker is now narrower again: + - pre-rendezvous, root may delay its explicit `RENDEZVOUS` introduction long enough that the runtime still spends part of the connect budget on ordinary hints + - post-rendezvous, even with truly rendezvous-only direct probing and payload, the remote peer still does not return confirmed direct traffic + +## 2026-03-16T15:03:47.8133143+01:00 + +- Checkpoint in progress: + - direct-only hinted payload now waits behind a short startup grace window before using ordinary hinted endpoints + - bootstrap still starts immediately, so direct `ECHO` / `HELLO` probing can run while payload holds off waiting for a root `RENDEZVOUS` +- Why it was tried: + - the last issue update showed the remaining local noise was almost entirely pre-rendezvous payload fanout + - root sometimes introduces a rendezvous endpoint quickly enough that sending SYN payload to ordinary hints first was just spending budget before the better path arrived +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_DirectOnly_SynPayload_WaitsBrieflyForRendezvous_BeforeUsingHintedPaths` +- Live result in `.codex-debug/direct-only-post-rendezvous-grace-trace.log`: + - in this traced run, no `TX direct-only payload fanout` happened before root sent `RX RENDEZVOUS: 0x9e072772f9 endpoints: 176.66.90.119:62120` + - after rendezvous arrived, payload clamped immediately to `176.66.90.119:62120@1` + - alternate pushed hints were skipped post-rendezvous as intended +- Issue update: + - this removes the specific pre-rendezvous payload-fanout problem for the traced run, so the remaining blocker is now even cleaner + - despite cleaner startup, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet at all + - all successful peer replies still came back relayed via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + +## 2026-03-16T15:16:06.4435500+01:00 + +- Checkpoint in progress: + - once root introduces a pinned rendezvous endpoint, the runtime now immediately relays current local direct-path advertisements back to the peer + - direct-only payload also stays suppressed briefly after that pinned rendezvous is learned +- Why it was tried: + - the previous checkpoint cleaned up pre-rendezvous payload fanout, but payload still started almost immediately after rendezvous + - if the peer needed one more relayed advertisement round to learn our public surfaces, that narrow post-rendezvous window was the last obvious local bootstrap race +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_DirectOnly_WaitsBrieflyAfterPinnedRendezvous_AndPushesDirectPathsViaRoot` +- Live result in `.codex-debug/direct-only-post-rendezvous-settle-trace.log`: + - right after `RX RENDEZVOUS: 0x9e072772f9 endpoints: 176.66.90.119:62120`, the runtime now immediately emits: + - `TX PUSH_DIRECT_PATHS via root for 0x9e072772f9: 212.241.85.84:53972 ... :53979` + - direct-only payload remained pinned to `176.66.90.119:62120@2` + - alternate hinted endpoints still stayed skipped once payload was pinned +- Issue update: + - this removes one more plausible local race, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer continued to return only relayed `OK(HELLO)`, relayed `PushDirectPaths`, and relayed `MulticastLike` traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - the blocker is now even more explicit: + - root introduction happens + - our public surfaces are relayed back immediately + - direct bootstrap and payload stay pinned to the rendezvous endpoint + - the remote peer still never sends a confirmed direct packet to this runtime + +## 2026-03-16T15:20:46.9796150+01:00 + +- Checkpoint in progress: + - the immediate relayed `PUSH_DIRECT_PATHS` triggered by root `RENDEZVOUS` now uses the full local advertisement set instead of the peer-aware filtered subset + - that means the peer sees both current public surfaces and local private surfaces immediately after rendezvous, not only the public ones +- Why it was tried: + - in the previous trace, the first relayed advertisement right after rendezvous contained only public `212.241.85.84:*` surfaces + - if the peer needed our private/local surfaces early for the shared/private hints it later advertised (`100.85.196.109:*`, `10.22.10.94:*`), filtering them out in that first post-rendezvous push was still a plausible local gap +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - regression coverage for the rendezvous-triggered root push now asserts both: + - public surfaces are included + - local private surfaces are included +- Live result in `.codex-debug/direct-only-post-rendezvous-full-local-ads.log`: + - the first two relayed pushes after `RX RENDEZVOUS` now show: + - one public-only push from the existing bootstrap path already in flight + - then the new full local push including `10.0.0.60:58487 ... :58494` + - later relayed pushes consistently included both public and local private surfaces + - direct-only payload still stayed pinned to `176.66.90.119:62120@6` +- Issue update: + - this closed another plausible local advertisement gap, but it still did not produce `RX direct raw`, `RX OK(ECHO)`, or any `hop=0` packet + - the peer still returned only relayed `OK(HELLO)`, relayed `PushDirectPaths`, and relayed `MulticastLike` traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + +## 2026-03-16T15:24:03.5623456+01:00 + +- Checkpoint in progress: + - root `RENDEZVOUS` handling now sends the full local relayed `PUSH_DIRECT_PATHS` before starting the direct rendezvous bootstrap, not after +- Why it was tried: + - the previous trace still showed the direct bootstrap packet going out before the full local advertisement round + - if the peer needed the relayed advertisement first, the remaining local race was the order inside the rendezvous handler itself +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` +- Live result in `.codex-debug/direct-only-post-rendezvous-full-first.log`: + - this run first received relayed `PUSH_DIRECT_PATHS` from the peer and spent most of the connect budget probing the broad hinted set + - root `RENDEZVOUS` arrived later in the run for `176.66.90.119:62120` + - after that, direct-only behavior narrowed again to the pinned rendezvous path and the relayed local path pushes were still the full local set +- Issue update: + - reordering the rendezvous handler did not change the outcome + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet + - the peer still only returned relayed traffic via `84.17.53.155:9993` + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - the active blocker remains the same: + - even after late root introduction and full local advertisement replay, the remote peer never establishes or answers a direct path to this runtime + +## 2026-03-16T15:35:37.1697952+01:00 + +- Checkpoint in progress: + - direct-only payload is now gated harder and no longer uses ordinary hinted endpoints before there is either a pinned root `RENDEZVOUS` endpoint or a confirmed hop-0 path + - focused tests were updated to reflect the new contract: ordinary hints remain bootstrap-only in direct-only mode +- Why it was tried: + - the previous live runs still spent part of the connect budget on broad ordinary hinted payload fanout before root introduced the public rendezvous endpoint + - if that early hinted payload was confusing the peer or simply wasting the narrow connect budget, the runtime needed to separate control bootstrap from payload eligibility more strictly +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` passed +- Live result in `.codex-debug/direct-only-post-rendezvous-required.log`: + - root supplied `RENDEZVOUS` for `176.66.90.119:62120` early in the run + - after that, direct-only payload stayed pinned to exactly `176.66.90.119:62120@3` + - alternate hinted endpoints were skipped once payload was pinned + - the runtime still emitted relayed `PUSH_DIRECT_PATHS` with both public and local private surfaces +- Issue update: + - this removed the earlier pre-rendezvous direct-only payload spray, but it still did not produce any direct return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet at all + - every successful control reply remained relayed `OK(HELLO)` via `84.17.53.155:9993` + - the peer also continued sending relayed `PushDirectPaths` and relayed `MulticastLike` traffic during the pinned rendezvous phase + - the HTTPS call still ended with `Timed out waiting for TCP SYN-ACK` + - current blocker update: + - the local runtime now cleanly waits for root rendezvous before direct-only payload + - root rendezvous arrives and payload clamps correctly to one endpoint/socket + - the remote peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T15:47:59.8613883+01:00 + +- Checkpoint in progress: + - single-socket direct hint bootstrap now truly honors the receiving socket instead of silently falling back to a planner-preferred socket + - direct-only payload still remains confirmation-gated from the previous checkpoint +- Why it was tried: + - the previous stricter direct-only run exposed a concrete local mismatch in the trace: + - `Direct hint bootstrap ... socket=7` + - followed by `TX ECHO bootstrap ... (socket=0)` + - that meant the runtime was logging one socket choice and sending on another for ordinary single-socket hint bootstrap, which could break NAT state and invalidate the bootstrap attempt before root rendezvous or on similar hint paths +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_DirectOnly_PushDirectPaths_BootstrapsOnReceivingSocket` +- Live result in `.codex-debug/direct-only-post-hint-socket-fix.log`: + - this particular rerun still got root `RENDEZVOUS` before the broad ordinary-hint bootstrap could dominate, so the old pre-rendezvous mismatch did not reappear in the same form + - the trace did confirm the pinned rendezvous path remained socket-consistent: + - `Direct hint bootstrap ... socket=7` + - `TX ECHO bootstrap to 176.66.90.119:62120 (socket=7)` + - `TX HELLO bootstrap to 176.66.90.119:62120 (socket=7, reported=176.66.90.119:62120)` +- Issue update: + - this closes another real local transport bug, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` packet + - all successful control replies remained relayed `OK(HELLO)` via `84.17.53.155:9993` + - the active blocker is unchanged: + - bootstrap is now cleaner and more socket-consistent + - the remote peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T15:56:07.0106456+01:00 + +- Checkpoint in progress: + - root `RENDEZVOUS` handling now immediately records peer-root socket affinity from the receiving socket + - on the same transition, the runtime now sends a relayed root `HELLO` refresh on that exact socket before replaying local `PUSH_DIRECT_PATHS` +- Why it was tried: + - the current live traces still show root introducing a rendezvous endpoint and the runtime then probing it directly on one socket, but there is still no hop-0 return traffic at all + - one remaining stale-mapping risk was that root introduction and our direct bootstrap were happening without an immediate same-socket root mapping refresh at the handoff point + - if root or the peer was still operating on a slightly stale observed mapping for that socket, refreshing root `HELLO` first is the narrowest remaining protocol-consistent fix +- Verification: + - added regression coverage: + - `DataplaneRuntime_Rendezvous_RefreshesRootHello_OnReceivingSocket` +- Issue update: + - this is intended to remove one more potential stale-public-surface race at the exact moment root introduces the peer + - expected next observation: + - either the peer starts returning hop-0 control after the refreshed introduction window + - or the traces stay unchanged, which would narrow the blocker further away from root-side surface freshness and toward the remote peer never attempting direct return at all + +## 2026-03-16T15:58:34.9946276+01:00 + +- Checkpoint in progress: + - the same-socket root `HELLO` refresh on `RENDEZVOUS` is now live, tested, and rerun against Jaeger with full tracing +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-root-refresh-on-rendezvous.log`: + - root again introduced `176.66.90.119:62120` + - immediately around that transition the runtime now does the intended same-socket root refresh behavior on socket `4`: + - relayed root `HELLO` + - relayed `PUSH_DIRECT_PATHS` + - direct TTL-2 hole-punch + - direct `HELLO` / `ECHO` bootstrap + - repeated relayed peer control still came back only through `84.17.53.155:9993` +- Issue update: + - this removed the remaining suspected stale-root-mapping race at rendezvous handoff, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the direct-only call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` +- blocker update: + - root-side rendezvous handoff is now cleaner and same-socket refreshed + - the peer still never attempts or never succeeds at any direct return packet to this runtime + +## 2026-03-16T16:03:23.8257280+01:00 + +- Checkpoint in progress: + - direct-only relayed root control now uses peer-root socket affinity as soon as rendezvous is pinned, instead of waiting for a confirmed hop-0 path + - this keeps post-rendezvous root `HELLO` and root `NETWORK_CREDENTIALS` on one socket in direct-only mode +- Why it was tried: + - the previous live rerun proved rendezvous handoff was same-socket refreshed, but subsequent relayed root maintenance still fanned out across all sockets + - that left one remaining local instability source: + - root and peer could keep seeing all eight public socket surfaces while direct-only payload and direct bootstrap were already pinned to one rendezvous socket + - the narrowest next fix was to keep relayed root control aligned with the pinned rendezvous/root-affine socket too +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_SendHelloViaRoot_DirectOnlyWithPinnedRendezvous_UsesObservedRootSocket` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-pinned-root-affinity.log`: + - root introduced `176.66.90.119:17569` + - after that, relayed root control stayed pinned to socket `7`: + - `TX HELLO bootstrap to 84.17.53.155:9993 (socket=7, reported=)` + - `TX NETWORK_CREDENTIALS via root for 0x9e072772f9` + - direct control and payload bootstrap also stayed pinned to the same rendezvous path and socket: + - `TX hole-punch to 176.66.90.119:17569 (socket=7, hopLimit=2)` + - `TX HELLO bootstrap to 176.66.90.119:17569 (socket=7, reported=176.66.90.119:17569)` + - later repeated `TX ECHO bootstrap to 176.66.90.119:17569 (socket=7)` +- Issue update: + - this removed the last obvious local root-control fanout after rendezvous, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - every successful control reply still came back relayed `hop=1` via `84.17.53.155:9993` + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + - blocker update: + - direct-only behavior is now pinned not just for direct bootstrap and payload, but also for relayed root control after rendezvous + - the peer still never returns a direct packet to this runtime even under that stricter single-socket regime + +## 2026-03-16T16:11:24.0584259+01:00 + +- Checkpoint in progress: + - direct-only pinned-rendezvous bootstrap now waits briefly for relayed `OK(HELLO)` peer-version learning before probing the pinned rendezvous endpoint + - once peer version is known, the first pinned-rendezvous probe now starts with direct `ECHO` instead of immediately sending full `HELLO` +- Why it was tried: + - the previous live run showed the rendezvous path and relayed root control were finally pinned to one socket, but the first post-rendezvous direct probe could still be a full `HELLO` + - upstream modern peers prefer `ECHO`-first direct probing once version is known, so a narrow grace window after rendezvous was the next parity fix + - if the peer was discarding or deprioritizing the first post-rendezvous direct `HELLO`, waiting a moment for relayed version learning was the narrowest remaining sequencing change +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-version-grace.log`: + - root introduced `176.66.90.119:17569` + - after the grace window, the first pinned-rendezvous direct probe was now: + - `Direct hint bootstrap: peer=0x9e072772f9 endpoint=176.66.90.119:17569 socket=4 fullHello=False allSockets=False` + - `TX ECHO bootstrap to 176.66.90.119:17569 (socket=4)` + - later maintenance still escalated to same-socket direct `HELLO` as expected + - relayed root control remained pinned to the same socket and continued to return only relayed `OK(HELLO)` via `84.17.53.155:9993` +- Issue update: + - this removed the remaining local sequencing mismatch at the start of pinned-rendezvous probing + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + - blocker update: + - the pinned rendezvous path is now `ECHO`-first on the receiving socket, which matches the intended modern-peer bootstrap shape more closely + - the peer still never returns any confirmed direct packet to this runtime even after that sequencing fix + +## 2026-03-16T16:22:18.3431425+01:00 + +- Checkpoint in progress: + - pinned-rendezvous version-grace is now enforced on the immediate root-rendezvous callback too, not just inside the later bootstrap loop + - direct-only bootstrap also now suppresses relayed root `HELLO` / `NETWORK_CREDENTIALS` / periodic `PUSH_DIRECT_PATHS` for a short quiet window right after pinned rendezvous is observed +- Why it was tried: + - the previous live rerun exposed a real ordering bug: + - root `RENDEZVOUS` was still able to trigger an immediate same-socket direct probe before `_pinnedRendezvousObservedMs` was recorded + - that bypassed the new version-grace gate and could still send a first direct `HELLO` instead of waiting for peer-version learning + - even after fixing the grace in the bootstrap loop, relayed root maintenance was still active during the same rendezvous window and could keep reinforcing the relay path while direct probing was trying to form a hop-0 path +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - added regression coverage: + - `DataplaneRuntime_Rendezvous_DelaysInitialDirectProbeUntilPeerVersionKnown` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-callback-delay.log`: + - root introduced `176.66.90.119:13234` on socket `4` + - the first pinned-rendezvous direct probe is now the intended same-socket `ECHO`: + - `Direct hint bootstrap: peer=0x9e072772f9 endpoint=176.66.90.119:13234 socket=4 fullHello=False allSockets=False` + - `TX ECHO bootstrap to 176.66.90.119:13234 (socket=4)` + - later maintenance still escalates to same-socket direct `HELLO` on the pinned rendezvous path as expected +- Issue update: + - the root-rendezvous callback no longer bypasses the version-grace gate, and the first pinned-rendezvous direct packet shape is now correct on the wire + - there is still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - all observed successful control replies still come back relayed `OK(HELLO)` via `84.17.53.155:9993` + - the underlying blocker therefore shifted again: + - it is no longer the immediate rendezvous callback ordering + - it is now the absence of any direct return traffic even after the pinned-rendezvous path starts with same-socket `ECHO` + +## 2026-03-16T16:27:33.0239107+01:00 + +- Checkpoint in progress: + - direct-only root `RENDEZVOUS` no longer sends an immediate relayed root `HELLO` or relayed `PUSH_DIRECT_PATHS` before the same-socket direct probe + - the direct-only rendezvous handoff is now closer to vendored upstream: + - record receiving/root-affine socket + - send low-TTL hole-punch + - attempt direct contact on that same socket +- Why it was tried: + - the prior run showed that even after fixing the version-grace ordering, the root-rendezvous handoff still had extra relayed control around it + - vendored upstream `_doRENDEZVOUS()` does not refresh relay control there; it only hole-punches and attempts direct contact + - so the next parity fix was removing that relay reinforcement from the direct-only rendezvous callback itself +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - updated regression coverage now asserts: + - direct-only rendezvous does not relay root `HELLO` on the receiving socket + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-no-root-relay.log`: + - before root introduction, ordinary peer hints still triggered direct `ECHO` bootstrap on the observed relayed socket + - when root later introduced `176.66.90.119:13234` on socket `6`, the immediate handoff was cleaner: + - `RX RENDEZVOUS ... via 84.17.53.155:9993` + - `TX hole-punch to 176.66.90.119:13234 (socket=6, hopLimit=2)` + - `TX ECHO bootstrap to 176.66.90.119:13234 (socket=6)` + - no immediate relayed root `HELLO` preceded that direct probe +- Issue update: + - this removed another local relay-side handoff difference, but it still did not produce any hop-0 return traffic + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returned successful control via relayed `OK(HELLO)` from `84.17.53.155:9993` + - blocker update: + - the remaining relay reinforcement now appears to be the later periodic root maintenance that resumes after the rendezvous handoff + - the next transport lever is suppressing relayed root maintenance for the full lifetime of an unresolved pinned rendezvous path, not just at the immediate callback + +## 2026-03-16T16:43:41.7467823+01:00 + +- Checkpoints in progress: + - relayed root `HELLO`, relayed `NETWORK_CREDENTIALS`, and periodic relayed `PUSH_DIRECT_PATHS` are now suppressed for the full lifetime of an unresolved pinned-rendezvous path in direct-only mode + - ordinary peer `PUSH_DIRECT_PATHS` hints now request all eligible local sockets again instead of being forced onto one receiving socket +- Why they were tried: + - the prior runs had converged on a cleaner same-socket rendezvous handoff, but the runtime still resumed relay maintenance afterward and could keep reinforcing the relay path while direct bootstrap was unresolved + - vendored upstream also treats ordinary pushed hints differently from root rendezvous: + - root rendezvous is same-socket and explicit + - ordinary peer `PUSH_DIRECT_PATHS` is not a strict one-socket path pin + - so the next two parity steps were: + - suppress relay maintenance for the whole unresolved pinned-rendezvous window + - restore broader ordinary-hint probing before rendezvous takes over +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-pinned-rendezvous-full-relay-suppression-rerun.log`: + - after root introduced `176.66.90.119:13234`, the runtime stayed on: + - same-socket TTL-2 hole-punch + - same-socket direct `ECHO` / later direct `HELLO` + - there was no post-rendezvous relayed root `HELLO`, no relayed `NETWORK_CREDENTIALS`, and no relayed `PUSH_DIRECT_PATHS` + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` +- Live result in `.codex-debug/direct-only-post-ordinary-hint-fanout-rerun.log`: + - the ordinary-hint fanout change is built and active, but this run received root `RENDEZVOUS` for `176.66.90.119:13234` almost immediately + - direct-only behavior therefore collapsed back into a pure pinned-rendezvous run: + - same-socket hole-punch on socket `7` + - same-socket delayed `ECHO` + - later same-socket direct `HELLO` + - root even repeated `RENDEZVOUS` for the same endpoint on socket `2`, but the peer still never returned any confirmed direct traffic +- Issue update: + - the relay-maintenance suppression is now confirmed on the wire, so the remaining blocker is no longer post-rendezvous relay reinforcement from this runtime + - the ordinary-hint fanout parity change did not help this live run because root rendezvous arrived immediately and took over the direct-only path selection + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + - blocker update: + - the managed runtime is now very close to the vendored rendezvous bootstrap shape for this case + - the peer still never returns any confirmed hop-0 packet to this runtime, even when relay maintenance is fully suppressed and direct probing is confined to the pinned rendezvous path + +## 2026-03-16T16:55:47.4026110+01:00 + +- Checkpoints in progress: + - full direct `HELLO` on unresolved pinned rendezvous now has a managed-stack parity hook to escape the strict sticky socket when a physical endpoint would otherwise need a full `HELLO` + - direct-only rendezvous clamping now distinguishes payload from control more carefully: + - payload still stays pinned to the root rendezvous endpoint + - sibling public hints on the same peer IP are allowed back into control-plane probing instead of being discarded purely because the exact port differs +- Why they were tried: + - the previous traces showed the peer often advertising many public ports on `176.66.90.119`, while direct-only mode was clamping everything to one exact root rendezvous port + - that is a risky assumption if the rendezvous port is only one candidate and the live direct port has shifted to a sibling public endpoint + - at the same time, vendored upstream does not keep full physical-address `HELLO` socket-pinned the same way this managed stack had been doing +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-hello-fanout.log`: + - root introduced `176.66.90.119:7395` + - this run did receive peer `PUSH_DIRECT_PATHS` later, but the peer version had already been learned via relayed root `OK(HELLO)`, so the new full-`HELLO` fanout hook was not the limiting factor + - direct-only still stayed on: + - same-socket `ECHO` to `176.66.90.119:7395` + - later same-socket direct `HELLO` + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` +- Live result in `.codex-debug/direct-only-post-pinned-sibling-hints.log`: + - root again introduced `176.66.90.119:7395` early + - this run never reached a post-rendezvous peer `PUSH_DIRECT_PATHS` phase at all, so the sibling-public-hint widening did not get exercised on the wire + - the runtime stayed on repeated same-socket hole-punch plus direct `HELLO` to `176.66.90.119:7395` + - the peer still only returned relayed traffic via `84.17.53.155:9993` +- Issue update: + - the blocker changed slightly again: + - it is no longer just “we might be over-clamping same-IP sibling ports,” because the latest live run did not even reach the sibling-hint path + - it is now also clear that some runs never progress past root rendezvous plus relayed peer traffic, with no usable peer `PUSH_DIRECT_PATHS` follow-up at all + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + +## 2026-03-16T17:01:54.9472340+01:00 + +- Checkpoint in progress: + - unresolved pinned-rendezvous maintenance for modern peers now stays on same-socket direct `ECHO` instead of periodically flipping back to full `HELLO` + - sibling public peer hints on the same IP are still allowed for control-plane probing after rendezvous, but payload remains pinned to the root rendezvous endpoint +- Why it was tried: + - the previous live runs were still dominated by repeated full `HELLO` on the pinned rendezvous endpoint after peer version was known + - vendored modern-peer bootstrap is much more `ECHO`-driven there, and repeated full `HELLO` on an unresolved path was another remaining behavioral difference + - at the same time, the sibling-public-hint widening needed one live run where peer `PUSH_DIRECT_PATHS` actually arrived after rendezvous so we could see whether those alternate same-IP ports were really being probed +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-pinned-rendezvous-echo-maintenance.log`: + - root introduced `176.66.90.119:7395` on socket `7` + - after peer version was learned from relayed root `OK(HELLO)`, pinned-rendezvous maintenance stayed on: + - same-socket direct `ECHO` to `176.66.90.119:7395` + - same-socket TTL-2 hole-punch + - no fallback back to repeated full `HELLO` on the pinned endpoint + - this same run also exercised the sibling-public-hint widening: + - peer `PUSH_DIRECT_PATHS` arrived after rendezvous + - sibling public endpoints like `176.66.90.119:19665`, `:11880`, `:51607`, `:8995`, `:26547` were probed with direct `ECHO` across sockets `0..7` + - payload still remained pinned to `176.66.90.119:7395` +- Issue update: + - the behavioral gap narrowed again: + - pinned-rendezvous maintenance is now `ECHO`-based as intended + - same-IP sibling public peer hints are no longer silently discarded after rendezvous + - even with both of those fixes active in the same run, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returned relayed traffic via `84.17.53.155:9993` + - the direct-only Jaeger call still failed with: + - `No direct ZeroTier payload path is available ... direct bootstrap did not discover a usable peer path` + +## 2026-03-16T17:13:47.9510439+01:00 + +- Checkpoint in progress: + - stabilized `DataplaneRuntime_DirectOnly_MaintenanceStaysOnPinnedRendezvousSocket` so it matches the current direct-only contract after the pinned-rendezvous `ECHO` maintenance change +- Why it was tried: + - the focused direct-only suite had one remaining failure after the recent rendezvous changes + - the failure was not a transport regression: + - the immediate maintenance pass can now be legitimately silent because the pinned-rendezvous hole-punch and `ECHO` are still inside their rate-limit windows right after bootstrap + - the old test still required immediate post-bootstrap maintenance traffic, which is no longer guaranteed by the current implementation +- Verification: + - focused suites passed again: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` +- Issue update: + - the issue changed from a possible maintenance-path regression to a test-contract mismatch + - the transport-side blocker itself did not change: + - the live problem is still that the peer never returns any confirmed hop-0 packet to this runtime after root rendezvous + +## 2026-03-16T17:25:05.5885963+01:00 + +- Checkpoints in progress: + - admissible non-public alternate hints are now allowed to stay in control-plane probing after root rendezvous instead of being skipped purely because payload is pinned + - relayed root control is no longer suppressed during pinned-rendezvous bootstrap while peer version is still unknown +- Why they were tried: + - a full redirected live trace showed the peer advertising private/shared hints like `100.85.196.109:*` and `10.22.10.94:*`, but the managed runtime was skipping them after rendezvous + - another live trace showed a tighter regression: + - rendezvous arrived before peer version was known + - relayed root `HELLO` then went quiet + - the runtime got stuck repeatedly sending full direct `HELLO` to the pinned rendezvous endpoint because it never learned that `ECHO` was legal +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-root-version-gate-fix.log`: + - relayed root `HELLO` and relayed `OK(HELLO)` are restored before pinned-rendezvous suppression takes over + - post-rendezvous peer `PUSH_DIRECT_PATHS` is again received + - the new alternate-hint policy is now clearly active on the wire: + - direct `ECHO` is sent to `100.85.196.109:9993`, `:49161`, `:21376` + - direct `ECHO` is sent to `10.22.10.94:9993` and `:49161` + - despite that, there is still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returns relayed traffic via `84.17.53.155:9993` +- Issue update: + - the new suppression/version-learning regression is fixed + - the non-public alternate-hint path is now exercised live + - the remaining blocker is again narrowed back to the wire: + - even after probing public sibling ports plus admissible private/shared hints, the peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T17:39:00.8464207+01:00 + +- Checkpoint in progress: + - direct-only maintenance no longer collapses back to the pinned rendezvous endpoint only + - allowed alternate hints are now revisited during maintenance, not just probed once at `PUSH_DIRECT_PATHS` receipt time +- Why it was tried: + - the previous live trace showed a gap in the maintenance selector: + - public sibling ports and admissible private/shared hints were only getting the initial bootstrap `ECHO` + - once the maintenance loop took over, it went back to the pinned rendezvous endpoint only + - that meant alternate hints did not stay warm long enough to test whether the peer might answer later in the connect window +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-maintenance-alt-revisit.log`: + - post-rendezvous peer `PUSH_DIRECT_PATHS` was received as before + - the managed runtime revisited alternate hints during maintenance and escalated them to `HELLO`, not only `ECHO` + - this happened for: + - public sibling endpoints like `176.66.90.119:19665`, `:63878`, `:15196`, `:23523`, `:16476`, `:31761` + - shared/private endpoints like `100.85.196.109:*` and `10.22.10.94:*` + - additional private Docker/LAN endpoints like `172.17.0.1:*` and `172.18.0.1:*` + - pinned rendezvous maintenance on `176.66.90.119:7395` still stayed active with same-socket `ECHO` plus TTL-2 hole-punch +- Issue update: + - the maintenance-selector gap is fixed + - the wire-level blocker remains: + - even with repeated alternate-hint `HELLO` and pinned-rendezvous `ECHO` in the same run, there is still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - the peer still only returns relayed traffic via `84.17.53.155:9993` + +## 2026-03-16T17:49:49.6485658+01:00 + +- Checkpoint in progress: + - sticky direct-hint maintenance selection was extracted into a dedicated helper: + - [ZeroTierStickyHintMaintenanceSelector.cs](/C:/Users/Jonas/repos/private/JKamsker/libzt-dotnet/ZTSharp/ZeroTier/Internal/ZeroTierStickyHintMaintenanceSelector.cs) + - pinned rendezvous maintenance still stays sticky + - alternate hinted maintenance now rotates across admissible sockets in the new focused helper path instead of always collapsing to the first fallback socket +- Why it was tried: + - the previous live trace showed a more specific socket-selection issue after the maintenance-selector fix: + - pinned rendezvous maintenance stayed on the correct root-affine socket + - but repeated alternate-hint `HELLO` still always went out on socket `0` + - that meant the selector had to distinguish: + - pinned rendezvous: stay sticky + - alternates: rotate across admissible sockets +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-maintenance-socket-rotation.log`: + - root again introduced pinned rendezvous on `176.66.90.119:7395` + - pinned rendezvous maintenance stayed on socket `7` with same-socket `ECHO` + - peer `PUSH_DIRECT_PATHS` again included: + - public sibling endpoints like `176.66.90.119:19665`, `:63878`, `:49134`, `:30840`, `:63656` + - shared/private endpoints like `100.85.196.109:*` and `10.22.10.94:*` + - repeated alternate-hint `HELLO` still went out on socket `0` in the live run + - there was still no `RX OK(ECHO)`, no `RX direct raw`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the new helper fixed and isolated one maintenance-path selector locally + - the live blocker changed shape again: + - the repeated post-rendezvous alternate-hint `HELLO` traffic is still not coming from the selector that was fixed here, because it remains pinned to socket `0` on the wire + - the next remaining local target is therefore the alternate-hint retry path outside the sticky maintenance selector, not the pinned-rendezvous maintenance loop + +## 2026-03-16T18:03:26.4107889+01:00 + +- Checkpoint in progress: + - direct-only ordinary hinted control is now held until rendezvous exists + - the maintenance loop now preserves rotated alternate-hint socket selection instead of overwriting it with the sticky selector +- Why it was tried: + - the previous live trace showed two distinct problems: + - pre-rendezvous direct-only control was still spending time on ordinary `PUSH_DIRECT_PATHS` + - once post-rendezvous alternate-hint `HELLO` retries started, they were all collapsing back to socket `0` + - that meant the runtime was still wasting bootstrap budget before rendezvous and not actually exercising multi-socket alternate retries after rendezvous +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `ZeroTierDirectEndpointManagerPushFlagsTests` + - `UserSpaceTcpClientConnectTests` + - `ZeroTierPinnedRendezvousAlternateHintPolicyTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-rendezvous-first-control.log`: + - root introduced pinned rendezvous first: + - `176.66.90.119:7395` + - initial rendezvous probe was correctly delayed only for peer-version learning: + - `Direct hint bootstrap delayed ... reason=await-peer-version` + - after `PUSH_DIRECT_PATHS`, pinned rendezvous maintenance stayed on socket `7` with repeated same-socket `ECHO` + - the alternate-hint `HELLO` retry path changed materially on the wire: + - public sibling endpoints like `176.66.90.119:19665`, `:2918`, `:7751`, `:52406`, `:53184`, `:35228`, `:22362` + - shared/private endpoints like `100.85.196.109:*` and `10.22.10.94:*` + - all rotated across sockets `0`, then `1`, then `2`, then `3`, then `4` + - despite that, there was still no `RX OK(ECHO)`, no `RX direct raw`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the alternate-hint retry path is no longer artificially stuck on one local socket + - the remaining blocker narrowed again to the wire: + - even with pinned rendezvous on one socket and alternate sibling/private retries rotating across multiple sockets, the peer still never returns a confirmed direct packet to this runtime + +## 2026-03-16T18:25:14.6073355+01:00 + +- Checkpoint in progress: + - relayed local direct-path advertisements now include conservative adjacent public-port predictions around each root-observed public surface + - pinned public rendezvous maintenance now also probes conservative adjacent sibling ports around the root-introduced rendezvous port +- Why it was tried: + - the previous live traces still showed zero hop-0 return traffic, which means the peer either still did not know a reachable return port for this runtime or the exact root-provided rendezvous port was not the only viable public candidate + - two transport-side levers were still available without using unrelated services: + - advertise likely adjacent public return ports for our own side + - directly probe likely adjacent public sibling ports around the pinned peer rendezvous port +- Verification: + - focused suites passed: + - `ZeroTierPredictedPublicSurfaceGeneratorTests` + - `ZeroTierPinnedRendezvousSiblingPortPredictorTests` + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-predicted-public-ads.log`: + - root-surface seeding and relayed `PUSH_DIRECT_PATHS` now clearly include predicted public return ports like: + - `212.241.85.84:50150..50161` + - despite that wider advertised local return set, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` +- Live result in `.codex-debug/direct-only-post-sticky-sibling-prediction.log`: + - after root pinned rendezvous at `176.66.90.119:7395`, the bootstrap loop now really probes adjacent sibling ports too: + - `176.66.90.119:7394` + - `176.66.90.119:7396` + - `176.66.90.119:7393` + - `176.66.90.119:7397` + - payload still stayed pinned to the rendezvous path + - even with both levers active together, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the blocker changed again in a useful way: + - predicted local public return ports are now definitely being advertised to the peer + - predicted sibling public ports around the pinned rendezvous port are now definitely being probed on the wire + - the remaining blocker is still not local selection or missing probe coverage: + - even after both local-return prediction and pinned-rendezvous sibling-port probing are active, the peer still never returns a single confirmed hop-0 packet to this runtime + +## 2026-03-16T18:31:41.0651393+01:00 + +- Checkpoint in progress: + - same-IP public sibling probes around the pinned rendezvous endpoint now inherit the pinned rendezvous socket instead of falling back to planner-selected sockets like `0` + - the sticky maintenance selector now uses the pinned socket for public alternates that share the pinned rendezvous IP +- Why it was tried: + - the previous live trace changed the issue again: + - sibling public ports like `176.66.90.119:7394..7397` were finally being probed + - but they were still going out on socket `0` while the root-introduced rendezvous path itself was on socket `7` + - that left a remaining local NAT-asymmetry risk: the peer might see alternate same-IP probes from a different public mapping than the one root had just introduced +- Verification: + - focused suites passed: + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierPredictedPublicSurfaceGeneratorTests` + - `ZeroTierPinnedRendezvousSiblingPortPredictorTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-sibling-socket-affinity.log`: + - root introduced pinned rendezvous on `176.66.90.119:7395` via socket `7` + - the sibling probes now really stay on that same socket: + - `TX HELLO bootstrap to 176.66.90.119:7394 (socket=7, ...)` + - `TX HELLO bootstrap to 176.66.90.119:7396 (socket=7, ...)` + - `TX HELLO bootstrap to 176.66.90.119:7393 (socket=7, ...)` + - `TX HELLO bootstrap to 176.66.90.119:7397 (socket=7, ...)` + - the same-socket public-sibling behavior also applied to other public hints on the same peer IP, such as: + - `176.66.90.119:19665` + - `176.66.90.119:50163` + - `176.66.90.119:26412` + - `176.66.90.119:11674` + - `176.66.90.119:46770` + - `176.66.90.119:5395` + - despite that, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the local public-sibling socket-affinity bug is fixed + - the remaining blocker narrowed again: + - this is no longer missing sibling-port probing + - and no longer the wrong local socket for those sibling probes + - even with root-rendezvous, sibling public probes, and sibling public socket affinity all aligned on the same socket, the peer still never returns a confirmed hop-0 packet + +## 2026-03-16T18:37:40.8390788+01:00 +- Change: + - same-IP public sibling hints around a pinned rendezvous no longer get periodically promoted to full `HELLO` + - the sticky hinted maintenance rule now keeps those sibling probes on `ECHO` while still allowing unrelated alternates to escalate to `HELLO` +- Why it was tried: + - the previous live trace changed the issue again: + - sibling public ports around the pinned rendezvous were finally using the correct local socket + - but they were still being retried as `HELLO`, not `ECHO` + - that left a remaining protocol-shape mismatch versus the intended modern-peer direct bootstrap: + - pinned rendezvous itself was already `ECHO`-first + - same-IP sibling ports around that rendezvous should follow the same direct `ECHO` path shape instead of periodically switching to `HELLO` +- Expected verification: + - focused suites should still pass: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - live trace should change from sibling `TX HELLO bootstrap to 176.66.90.119:7394..7397` to sibling `TX ECHO bootstrap` on the pinned rendezvous socket +- Issue update: + - the remaining local mismatch is now specifically the verb choice on same-IP sibling public retries after rendezvous + - if this patch behaves as intended and the peer still does not respond hop-0, the blocker will narrow further to the remote side ignoring both pinned-rendezvous and sibling same-IP `ECHO` traffic + +## 2026-03-16T18:42:22.0606954+01:00 +- Change: + - same-IP public sibling retries around a pinned rendezvous now stay on `ECHO` instead of periodically switching to `HELLO` + - the focused pinned-rendezvous maintenance tests were tightened to reflect that newer direct-bootstrap contract +- Why it was tried: + - the previous live trace changed the issue again: + - sibling public ports around the pinned rendezvous were finally using the correct local socket + - but they were still being retried as `HELLO`, which did not match the intended modern-peer `ECHO`-first path shape +- Verification: + - focused suites passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDirectBootstrapControlPolicyTests` + - `UserSpaceTcpClientConnectTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-sibling-echo.log`: + - root introduced pinned rendezvous on `176.66.90.119:7395` via socket `6` + - same-IP sibling public retries now really stay on `ECHO` on that same socket: + - `TX ECHO bootstrap to 176.66.90.119:7394 (socket=6)` + - `TX ECHO bootstrap to 176.66.90.119:7396 (socket=6)` + - `TX ECHO bootstrap to 176.66.90.119:7393 (socket=6)` + - `TX ECHO bootstrap to 176.66.90.119:7397 (socket=6)` + - the same `ECHO`-only shape also applied to other public same-IP hints like: + - `176.66.90.119:19665` + - `176.66.90.119:37463` + - `176.66.90.119:27299` + - `176.66.90.119:44556` + - `176.66.90.119:5395` + - despite that, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the local same-IP sibling verb-selection bug is fixed + - the remaining blocker narrowed again: + - this is no longer wrong local socket for sibling public retries + - and no longer wrong control verb for sibling public retries + - even with pinned-rendezvous `ECHO` plus sibling same-IP `ECHO` on the same socket, the peer still never returns a confirmed hop-0 packet + +## 2026-03-16T18:52:18.4803247+01:00 +- Change: + - the initial direct-hint callback path now stops forcing same-IP public sibling hints around a pinned rendezvous into `allSockets=True` + - if the planner does not yet have a preferred socket for the pinned rendezvous, pinned-socket selection now falls back to the observed root/rendezvous socket +- Why it was tried: + - the previous live trace changed the issue again: + - later maintenance had already become same-socket `ECHO` + - but the first post-rendezvous callback path was still spraying sibling public hints like `176.66.90.119:19665` across sockets `0..7` + - that meant the initial rendezvous follow-up was still using a broader local NAT surface than the later sticky maintenance path +- Verification: + - focused direct-path tests passed: + - `ZeroTierDataplaneRuntimeDirectPathTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-initial-sibling-clamp.log`: + - root-introduced rendezvous and same-IP public sibling hints now all start same-socket on `4`: + - `TX ECHO bootstrap to 176.66.90.119:7395 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:19665 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:37463 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:27299 (socket=4)` + - `TX ECHO bootstrap to 176.66.90.119:44556 (socket=4)` + - public same-IP alternates no longer show `allSockets=True` + - non-public alternates still do, for example: + - `100.85.196.109:9993` + - `100.85.196.109:49161` + - `172.17.0.1:9993` + - `172.18.0.1:9993` + - there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the initial same-IP public sibling spray bug is fixed + - the remaining blocker narrowed again: + - public rendezvous plus public same-IP sibling probes now both start same-socket and `ECHO`-first + - the peer still never returns a confirmed hop-0 packet + - the remaining broad fanout is now on non-public alternate hints, not on the root-introduced public path + +## 2026-03-16T19:10:13.9849570+01:00 +- Change: + - non-public alternate hints after pinned rendezvous now use one constrained socket instead of `allSockets=True` + - non-public alternate maintenance is now prioritized ahead of public same-IP alternates once payload is already pinned to the public rendezvous path + - when a constrained non-public alternate has no better sticky path yet, it now falls back to the observed root/rendezvous socket +- Why it was tried: + - the previous live trace changed the issue again: + - public rendezvous and public same-IP siblings were already same-socket and `ECHO`-first + - but non-public alternates were either still broad-fanout or got starved behind the public alternates during maintenance +- Verification: + - focused suites passed: + - `ZeroTierStickyHintMaintenanceSelectorTests` + - `ZeroTierDataplaneRuntimeDirectPathTests` + - CLI build passed: + - `dotnet build samples/ZTSharp.Cli/ZTSharp.Cli.csproj -c Release --no-restore` +- Live result in `.codex-debug/direct-only-post-nonpublic-priority.log`: + - non-public alternates are now revisited after rendezvous and are same-socket: + - `TX ECHO bootstrap to 100.85.196.109:9993 (socket=7)` + - `TX ECHO bootstrap to 100.85.196.109:49161 (socket=7)` + - `TX ECHO bootstrap to 100.85.196.109:21376 (socket=7)` + - `TX ECHO bootstrap to 172.17.0.1:9993 (socket=7)` + - `TX ECHO bootstrap to 172.18.0.1:9993 (socket=7)` + - `TX ECHO bootstrap to 10.22.10.94:9993 (socket=7)` + - `TX ECHO bootstrap to 172.17.0.1:49161 (socket=7)` + - `TX ECHO bootstrap to 172.18.0.1:49161 (socket=7)` + - `TX ECHO bootstrap to 10.22.10.94:49161 (socket=7)` + - `TX ECHO bootstrap to 172.17.0.1:21376 (socket=7)` + - `TX ECHO bootstrap to 172.18.0.1:21376 (socket=7)` + - despite that, there was still no `RX direct raw`, no `RX OK(ECHO)`, and no `hop=0` + - final failure remained: + - `No direct ZeroTier payload path is available for peer 0x9e072772f9; relay fallback is disabled (direct bootstrap did not discover a usable peer path)` +- Issue update: + - the non-public alternate fanout/starvation issue is fixed + - the remaining blocker narrowed again: + - public rendezvous, public same-IP siblings, and non-public alternates are all now same-socket and `ECHO`-first + - the peer still never returns a confirmed hop-0 packet even after all of those local transport-side corrections + +## 2026-03-16T20:14:37.1731390+01:00 +- Change: + - performed read-only analysis against the production Jaeger host using stock `zerotier-cli` and non-mutating `ssh` commands + - no config or service changes were made locally or remotely +- Why it was tried: + - the managed runtime still fails to form hop-0 P2P to `jaeger.pdcs.kamsker.at` + - the user confirmed that stock ZeroTier is now directly connected, so the next question was whether the remote host and stock path reveal why the managed stack still fails +- Verification: + - local stock ZeroTier: + - `zerotier-cli info` + - `zerotier-cli listnetworks` + - `zerotier-cli listpeers` + - remote read-only commands over `ssh jaeger.pdcs.kamsker.at`: + - `sudo -n zerotier-cli info` + - `sudo -n zerotier-cli listnetworks` + - `sudo -n zerotier-cli listpeers` + - `ip -br addr` + - `ip route` + - `ip route get 10.121.15.246` + - `ss -tln '( sport = :443 )'` +- Live result: + - local stock node is `b621d170ad` on `9ad07d01093a69e3` with ZeroTier IP `10.121.15.246` + - remote Jaeger host is: + - hostname `dev-build-server` + - ZeroTier node `9e072772f9` + - ZeroTier version `1.16.0` + - ZeroTier interface `zttpmuo2hk` + - ZeroTier IP `10.121.15.120/24` + - stock path is confirmed direct on both ends: + - local `listpeers` shows `9e072772f9 176.66.90.119/46827 ... LEAF` + - remote `listpeers` shows `b621d170ad 212.241.85.84/49072 ... LEAF` + - remote interface inventory explains the alternate hints seen in managed-stack traces: + - LAN `10.22.10.94/24` + - ZeroTier `10.121.15.120/24` + - Tailscale `100.85.196.109/32` + - Docker bridges `172.17.0.1/16` and `172.18.0.1/16` + - remote routing confirms `10.121.15.246` is reached over `zttpmuo2hk` + - remote HTTPS listener is bound on `0.0.0.0:443` and `[::]:443` +- Issue update: + - stock ZeroTier proves that this peer pair can form a real direct path on the current networks: + - local public tuple seen remotely: `212.241.85.84:49072` + - remote public tuple seen locally: `176.66.90.119:46827` + - the remaining blocker is therefore inside the managed runtime, not the environment: + - the remote host is reachable directly + - the host’s extra LAN/Tailscale/Docker interfaces explain the non-public hints that kept appearing in traces + - the managed stack still needs to match stock behavior for path introduction, surface authority, reply correlation, or ingress handling diff --git a/samples/ZTSharp.Cli/CliHelp.cs b/samples/ZTSharp.Cli/CliHelp.cs index 04ee08e..e8ddb52 100644 --- a/samples/ZTSharp.Cli/CliHelp.cs +++ b/samples/ZTSharp.Cli/CliHelp.cs @@ -29,6 +29,7 @@ libzt call --network --url [options] --map-ip Map IP to node id for overlay HTTP (repeatable) --multipath Enable experimental multipath direct-path selection and bonding (managed stack only) + --direct-only For 'call': require negotiated direct peer paths for payloads and disallow root relay fallback --mp-bond Bond policy: off|active-backup|broadcast|balance-rr|balance-xor|balance-aware (default: off) --mp-udp-sockets Number of UDP sockets to open when multipath is enabled (default: 1) --mp-udp-ports Explicit local UDP ports for multipath sockets (0 = ephemeral; length must match --mp-udp-sockets) diff --git a/samples/ZTSharp.Cli/CliParsing.cs b/samples/ZTSharp.Cli/CliParsing.cs index c1aef46..611446e 100644 --- a/samples/ZTSharp.Cli/CliParsing.cs +++ b/samples/ZTSharp.Cli/CliParsing.cs @@ -187,17 +187,45 @@ public static ZeroTierBondPolicy ParseBondPolicy(string value) throw new InvalidOperationException("Invalid bond policy."); } - value = value.Trim(); - return value switch + var normalized = value.Trim(); + if (normalized.Equals("off", StringComparison.OrdinalIgnoreCase)) { - "off" => ZeroTierBondPolicy.Off, - "active-backup" or "activebackup" => ZeroTierBondPolicy.ActiveBackup, - "broadcast" => ZeroTierBondPolicy.Broadcast, - "balance-rr" or "balance-roundrobin" or "balance-round-robin" or "roundrobin" or "rr" => ZeroTierBondPolicy.BalanceRoundRobin, - "balance-xor" or "xor" => ZeroTierBondPolicy.BalanceXor, - "balance-aware" or "aware" => ZeroTierBondPolicy.BalanceAware, - _ => throw new InvalidOperationException("Invalid bond policy (expected off|active-backup|broadcast|balance-rr|balance-xor|balance-aware).") - }; + return ZeroTierBondPolicy.Off; + } + + if (normalized.Equals("active-backup", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("activebackup", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.ActiveBackup; + } + + if (normalized.Equals("broadcast", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.Broadcast; + } + + if (normalized.Equals("balance-rr", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("balance-roundrobin", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("balance-round-robin", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("roundrobin", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("rr", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.BalanceRoundRobin; + } + + if (normalized.Equals("balance-xor", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("xor", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.BalanceXor; + } + + if (normalized.Equals("balance-aware", StringComparison.OrdinalIgnoreCase) || + normalized.Equals("aware", StringComparison.OrdinalIgnoreCase)) + { + return ZeroTierBondPolicy.BalanceAware; + } + + throw new InvalidOperationException("Invalid bond policy (expected off|active-backup|broadcast|balance-rr|balance-xor|balance-aware)."); } public static IReadOnlyList ParsePortList(string value, string name) diff --git a/samples/ZTSharp.Cli/Commands/CallCommand.cs b/samples/ZTSharp.Cli/Commands/CallCommand.cs index cd85653..6fa76aa 100644 --- a/samples/ZTSharp.Cli/Commands/CallCommand.cs +++ b/samples/ZTSharp.Cli/Commands/CallCommand.cs @@ -26,6 +26,7 @@ public static async Task RunAsync(string[] commandArgs) int? mpUdpSockets = null; IReadOnlyList? mpUdpPorts = null; var mpWarmupRoot = true; + var directOnly = false; for (var i = 0; i < commandArgs.Length; i++) { @@ -87,6 +88,11 @@ public static async Task RunAsync(string[] commandArgs) case "--multipath": mpEnabled = true; break; + case "--direct-only": + directOnly = true; + mpEnabled = true; + mpWarmupRoot = false; + break; case "--mp-bond": { var value = CliParsing.ReadOptionValue(commandArgs, ref i, "--mp-bond"); @@ -147,13 +153,20 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, - WarmupDuplicateToRoot = mpWarmupRoot + WarmupDuplicateToRoot = mpWarmupRoot, + AllowRootRelayFallback = !directOnly }; await RunCallZeroTierAsync(statePath, networkId, multipath, url, cancellation.Token).ConfigureAwait(false); diff --git a/samples/ZTSharp.Cli/Commands/ExposeCommand.cs b/samples/ZTSharp.Cli/Commands/ExposeCommand.cs index 0cbc41e..9cc53bb 100644 --- a/samples/ZTSharp.Cli/Commands/ExposeCommand.cs +++ b/samples/ZTSharp.Cli/Commands/ExposeCommand.cs @@ -146,11 +146,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/JoinCommand.cs b/samples/ZTSharp.Cli/Commands/JoinCommand.cs index 385cd40..a3e4e55 100644 --- a/samples/ZTSharp.Cli/Commands/JoinCommand.cs +++ b/samples/ZTSharp.Cli/Commands/JoinCommand.cs @@ -123,11 +123,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/ListenCommand.cs b/samples/ZTSharp.Cli/Commands/ListenCommand.cs index 891ae58..9a0e052 100644 --- a/samples/ZTSharp.Cli/Commands/ListenCommand.cs +++ b/samples/ZTSharp.Cli/Commands/ListenCommand.cs @@ -99,11 +99,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs b/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs index 51c04ec..840d219 100644 --- a/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs +++ b/samples/ZTSharp.Cli/Commands/UdpListenCommand.cs @@ -93,11 +93,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs b/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs index e3d159e..7127991 100644 --- a/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs +++ b/samples/ZTSharp.Cli/Commands/UdpSendCommand.cs @@ -107,11 +107,17 @@ public static async Task RunAsync(string[] commandArgs) if (string.Equals(stack, "managed", StringComparison.OrdinalIgnoreCase)) { + var resolvedUdpSocketCount = mpUdpSockets ?? mpUdpPorts?.Count ?? 1; + if (mpUdpPorts is not null && mpUdpPorts.Count != resolvedUdpSocketCount) + { + throw new InvalidOperationException("Invalid multipath config: --mp-udp-ports count must match --mp-udp-sockets."); + } + var multipath = new ZeroTierMultipathOptions { Enabled = mpEnabled, BondPolicy = mpBondPolicy, - UdpSocketCount = mpUdpSockets ?? 1, + UdpSocketCount = resolvedUdpSocketCount, LocalUdpPorts = mpUdpPorts, WarmupDuplicateToRoot = mpWarmupRoot }; diff --git a/scripts/zt_join_accept_fetch.ps1 b/scripts/zt_join_accept_fetch.ps1 index fa369a6..4e2efaa 100644 --- a/scripts/zt_join_accept_fetch.ps1 +++ b/scripts/zt_join_accept_fetch.ps1 @@ -3,9 +3,7 @@ param( [string]$Configuration = "Release", [string]$Org = "cmlpb7m960006np01jgwg4dr7", [string]$Network = "9ad07d01093a69e3", - [string]$Url = "http://10.121.15.99:5380/", - [int]$UdpPort = 0, - [string]$HttpMode = "os", + [string]$Url = "https://jaeger.pdcs.kamsker.at/", [string]$StateRootPath = "", [switch]$SkipBuild ) @@ -16,26 +14,30 @@ function Write-Step([string]$Message) { Write-Host ("== " + $Message) } -function Get-ZtNodeIdFromState([string]$StateRoot) { - $secretPath = Join-Path $StateRoot "identity.secret" - $bytes = [System.IO.File]::ReadAllBytes($secretPath) - if ($bytes.Length -lt 32) { - throw "Invalid identity secret file: $secretPath" +function Resolve-ZtNetCommand() { + foreach ($candidate in @("ztnet-cli", "ztnet")) { + $command = Get-Command $candidate -ErrorAction SilentlyContinue + if ($null -ne $command) { + return $command.Source + } } - $secret = New-Object byte[] 32 - [Array]::Copy($bytes, 0, $secret, 0, 32) - $sha = [System.Security.Cryptography.SHA256]::Create() - try { - $hash = $sha.ComputeHash($secret) + throw "Neither 'ztnet-cli' nor 'ztnet' was found on PATH." +} + +function Get-ZtNodeIdFromState([string]$StateRoot) { + $identityPath = Join-Path $StateRoot "zerotier\\identity.bin" + $bytes = [System.IO.File]::ReadAllBytes($identityPath) + if ($bytes.Length -lt 13) { + throw "Invalid identity file: $identityPath" } - finally { - $sha.Dispose() + + $magic = [System.Text.Encoding]::ASCII.GetString($bytes, 0, 4) + if ($magic -ne "ZTID") { + throw "Unexpected identity file format: $identityPath" } - $value = [System.BitConverter]::ToUInt64($hash, 0) - $mask = [UInt64]0xFFFFFFFFFF - $value = $value -band $mask + $value = [System.BitConverter]::ToUInt64($bytes, 5) return ("0x{0:x10}" -f $value) } @@ -52,24 +54,28 @@ if ([string]::IsNullOrWhiteSpace($StateRootPath)) { New-Item -ItemType Directory -Path $StateRootPath -Force | Out-Null +$ztnetCommand = Resolve-ZtNetCommand + Write-Step "Checking ztnet auth..." -ztnet auth test | Out-Host +& $ztnetCommand auth test | Out-Host if (-not $SkipBuild) { Write-Step "Building ($Configuration)..." dotnet build -c $Configuration | Out-Host } -Write-Step "Joining network $Network (state: $StateRootPath)..." -& dotnet run -c $Configuration --no-build ` +Write-Step "Bootstrapping managed ZeroTier identity for $Network (state: $StateRootPath)..." +$bootstrapOutput = & dotnet run -c $Configuration --no-build ` --project samples/ZTSharp.Cli/ZTSharp.Cli.csproj -- ` join ` --once ` --network $Network ` - --stack overlay ` - --transport osudp ` - --udp-port $UdpPort ` - --state $StateRootPath | Out-Host + --stack managed ` + --state $StateRootPath 2>&1 +$bootstrapOutput | Out-Host +if ($LASTEXITCODE -ne 0) { + Write-Step "Initial join did not complete before authorization; continuing with generated node identity." +} $nodeId = Get-ZtNodeIdFromState -StateRoot $StateRootPath Write-Step "NodeId: $nodeId" @@ -79,17 +85,18 @@ if ($nodeIdForZtnet.StartsWith("0x")) { } Write-Step "Adding + authorizing member in ZTNet (org: $Org)..." -ztnet --org $Org --yes network member add $Network $nodeIdForZtnet | Out-Host -ztnet --org $Org --yes network member authorize $Network $nodeIdForZtnet | Out-Host +& $ztnetCommand --org $Org --yes network member add $Network $nodeIdForZtnet | Out-Host +& $ztnetCommand --org $Org --yes network member authorize $Network $nodeIdForZtnet | Out-Host -Write-Step "Fetching $Url (call --http $HttpMode)..." -& dotnet run -c $Configuration --no-build ` +Write-Step "Fetching $Url over managed ZeroTier HTTP..." +$callOutput = & dotnet run -c $Configuration --no-build ` --project samples/ZTSharp.Cli/ZTSharp.Cli.csproj -- ` call ` - --http $HttpMode ` --network $Network ` - --stack overlay ` - --transport osudp ` - --udp-port 0 ` + --stack managed ` --state $StateRootPath ` - --url $Url | Out-Host + --url $Url 2>&1 +$callOutput | Out-Host +if ($LASTEXITCODE -ne 0) { + throw "Managed ZeroTier HTTP call failed (exit code $LASTEXITCODE)." +}