diff --git a/CLAUDE.md b/CLAUDE.md index e6b59523..5cae0d54 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,14 +35,14 @@ dotnet build MaxMind.Db.sln ```bash # Run all tests -dotnet test MaxMind.Db.Test/MaxMind.Db.Test.csproj +dotnet test --project MaxMind.Db.Test/MaxMind.Db.Test.csproj # Run specific test class -dotnet test --filter "FullyQualifiedName~ReaderTest" -dotnet test --filter "FullyQualifiedName~DecoderTest" +dotnet test --project MaxMind.Db.Test/MaxMind.Db.Test.csproj -- --filter-class "*ReaderTest" +dotnet test --project MaxMind.Db.Test/MaxMind.Db.Test.csproj -- --filter-class "*DecoderTest" # Run specific test method -dotnet test --filter "FullyQualifiedName~ReaderTest.TestMany" +dotnet test --project MaxMind.Db.Test/MaxMind.Db.Test.csproj -- --filter-method "*ReaderTest.TestPointerHeavyValueCountDecodes" ``` ### Running Benchmarks diff --git a/MaxMind.Db.Benchmark/Program.cs b/MaxMind.Db.Benchmark/Program.cs index ab49286d..0a0eb71f 100644 --- a/MaxMind.Db.Benchmark/Program.cs +++ b/MaxMind.Db.Benchmark/Program.cs @@ -7,7 +7,7 @@ using System.Linq; using System.Net; -BenchmarkRunner.Run(); +BenchmarkRunner.Run(args: args); [MemoryDiagnoser] public class CityBenchmark diff --git a/MaxMind.Db.Test/CollectionActivatorTest.cs b/MaxMind.Db.Test/CollectionActivatorTest.cs new file mode 100644 index 00000000..723a1410 --- /dev/null +++ b/MaxMind.Db.Test/CollectionActivatorTest.cs @@ -0,0 +1,68 @@ +using System.Collections.Generic; +using Xunit; + +namespace MaxMind.Db.Test +{ + public static class CollectionActivatorTest + { + [Fact] + public static void ListFactoryUsesRequestedCapacity() + { + var factory = new ListActivatorCreator().GetActivator(typeof(ICollection)); + var list = Assert.IsType>(factory(123)); + Assert.Empty(list); + Assert.Equal(123, list.Capacity); + } + + [Fact] + public static void ListFactoryPreservesCustomDefaultConstructor() + { + var factory = new ListActivatorCreator().GetActivator(typeof(DefaultList)); + var list = Assert.IsType>(factory(123)); + Assert.True(list.WasConstructed); + Assert.Equal(0, list.Capacity); + } + + [Fact] + public static void DictionaryFactoryCreatesRequestedInterface() + { + var factory = new DictionaryActivatorCreator().GetActivator(typeof(IDictionary)); + var dictionary = Assert.IsType>(factory(123)); + Assert.Empty(dictionary); +#if NET8_0_OR_GREATER + Assert.True(dictionary.EnsureCapacity(0) >= 123); +#endif + dictionary.Add("value", 7); + Assert.Equal(7, dictionary["value"]); + } + + [Fact] + public static void DictionaryFactoryPreservesCustomDefaultConstructor() + { + var factory = new DictionaryActivatorCreator().GetActivator(typeof(DefaultDictionary)); + var dictionary = Assert.IsType>(factory(123)); + Assert.True(dictionary.WasConstructed); + Assert.Empty(dictionary); + } + + private sealed class DefaultList : List + { + public DefaultList() + { + WasConstructed = true; + } + + public bool WasConstructed { get; } + } + + private sealed class DefaultDictionary : Dictionary where TKey : notnull + { + public DefaultDictionary() + { + WasConstructed = true; + } + + public bool WasConstructed { get; } + } + } +} diff --git a/MaxMind.Db.Test/DecoderTest.cs b/MaxMind.Db.Test/DecoderTest.cs index 69b89dbf..44403025 100644 --- a/MaxMind.Db.Test/DecoderTest.cs +++ b/MaxMind.Db.Test/DecoderTest.cs @@ -4,7 +4,9 @@ using System.Collections.Generic; using System.IO; using System.Numerics; +using System.Runtime.ExceptionServices; using System.Text; +using System.Threading; using Xunit; #endregion @@ -41,6 +43,861 @@ public static void TestTypeDecoding(Dictionary tests) where T : cl } } + private static void WritePointer1(List bytes, int target) + { + // One-byte-payload pointer (type 1, pointer_size 1) with base 0. + bytes.Add((byte)((1 << 5) | ((target >> 8) & 0x7))); + bytes.Add((byte)(target & 0xFF)); + } + + private static byte[] NestedContainers(int count) + { + var bytes = new List(count * 3 + 1); + for (var i = 0; i < count; i++) + { + if (i % 2 == 0) + { + bytes.Add(0x01); // array with one element + bytes.Add(0x04); + } + else + { + bytes.Add(0xE1); // map with one entry + bytes.Add(0x41); // one-byte string key + bytes.Add((byte)'x'); + } + } + bytes.Add(0xA0); // leaf: uint16 with value 0 + return [.. bytes]; + } + + [Fact] + public static void TestMapPointerFanOutIsBounded() + { + // Both map values point to the same child. Each visit must + // charge the child map again. + const int depth = 100; + var bytes = new List { 0xA0 }; // leaf: uint16 with value 0 + var prev = 0; + for (var i = 0; i < depth; i++) + { + var offset = bytes.Count; + bytes.Add(0xE2); + bytes.Add(0x41); + bytes.Add((byte)'a'); + WritePointer1(bytes, prev); + bytes.Add(0x41); + bytes.Add((byte)'b'); + WritePointer1(bytes, prev); + prev = offset; + } + + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(prev, out _)); + Assert.Contains("maximum number of values", ex.Message); + } + + // 256 bytes of string payload per pointer target. 8,192 occurrences + // reach exactly the 2 MiB payload budget and 8,193 cross it. + private const int FlatFanOutTargetSize = 256; + + // An array of pointers to one string isolates repeated payload + // charges from container fan-out and depth limits. + private static byte[] FlatScalarPointerTargets(int pointerCount, out int arrayOffset) + { + var encodedSize = pointerCount - 285; + var bytes = new List(pointerCount * 2 + FlatFanOutTargetSize + 8) + { + 0x5D, // target: UTF-8 string with a one-byte encoded size + (byte)(FlatFanOutTargetSize - 29), + }; + bytes.AddRange(new byte[FlatFanOutTargetSize]); + arrayOffset = bytes.Count; + bytes.Add(0x1E); + bytes.Add(0x04); // array with a two-byte encoded size + bytes.Add((byte)(encodedSize >> 8)); + bytes.Add((byte)encodedSize); + for (var i = 0; i < pointerCount; i++) + { + WritePointer1(bytes, 0); + } + + return [.. bytes]; + } + + [Theory] + [InlineData(8_192, false)] + [InlineData(8_193, true)] + public static void TestFlatScalarPointerTargetsConsumePayloadBudget(int pointerCount, bool exceedsLimit) + { + // This is intentionally flat so neither depth nor exponential + // container fan-out can hide incorrect payload accounting. + var bytes = FlatScalarPointerTargets(pointerCount, out var arrayOffset); + + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + if (exceedsLimit) + { + var ex = Assert.Throws(() => decoder.Decode(arrayOffset, out _)); + Assert.Equal( + "The MaxMind DB file's data section exceeds the maximum payload size.", + ex.Message); + } + else + { + for (var i = 0; i < 3; i++) + { + var decoded = Assert.IsType>(decoder.Decode(arrayOffset, out var offset)); + Assert.Equal(pointerCount, decoded.Count); + Assert.Equal(bytes.Length, offset); + } + } + } + + [Theory] + [InlineData(8_192, false)] + [InlineData(8_193, true)] + public static void TestFlatModelKeyPointerTargetsConsumePayloadBudget(int pointerCount, bool exceedsLimit) + { + // DecodeKey must charge each visit to the shared key. Unknown + // fields have false values, so skipping them adds no payload charge. + var encodedSize = pointerCount - 285; + var bytes = new List(pointerCount * 4 + FlatFanOutTargetSize + 8) + { + 0x5D, // target: UTF-8 string with a one-byte encoded size + (byte)(FlatFanOutTargetSize - 29), + }; + bytes.AddRange(new byte[FlatFanOutTargetSize]); + var mapOffset = bytes.Count; + bytes.Add(0xFE); // map with a two-byte encoded size + bytes.Add((byte)(encodedSize >> 8)); + bytes.Add((byte)encodedSize); + for (var i = 0; i < pointerCount; i++) + { + WritePointer1(bytes, 0); + bytes.Add(0x00); // extended boolean + bytes.Add(0x07); // false + } + + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + if (exceedsLimit) + { + var ex = Assert.Throws(() => decoder.Decode(mapOffset, out _)); + Assert.Equal( + "The MaxMind DB file's data section exceeds the maximum payload size.", + ex.Message); + } + else + { + var decoded = decoder.Decode(mapOffset, out var offset); + Assert.Null(decoded.Name); + Assert.Equal(bytes.Count, offset); + } + } + + // The root and 65,535 booleans use the entire value budget + // without consuming payload bytes. + private static byte[] ValueBudgetArray(int childCount = 65_535) + { + var encodedSize = childCount - 285; + var bytes = new List(childCount * 2 + 4) + { + 0x1E, // array with a two-byte encoded size + 0x04, + (byte)(encodedSize >> 8), + (byte)encodedSize, + }; + for (var i = 0; i < childCount; i++) + { + bytes.Add(0x00); // extended boolean + bytes.Add(0x07); // false + } + + return [.. bytes]; + } + + [Fact] + public static void TestConcurrentSuccessfulAndRejectedLookupsHaveSeparateBudgets() + { + // Exercise both budgets through one shared decoder. The repeated + // lookup tests check budget reset without relying on thread overlap. + const int pointerCount = 8_192; + const int childCount = 65_535; + var payloadBytes = FlatScalarPointerTargets(pointerCount, out var arrayOffset); + var valueBytes = ValueBudgetArray(); + var valueOffset = payloadBytes.Length; + var excessiveValues = ValueBudgetArray(childCount + 1); + var excessiveValueOffset = payloadBytes.Length + valueBytes.Length; + var excessivePayload = FlatScalarPointerTargets(pointerCount + 1, out var excessiveArrayOffset); + var excessivePayloadOffset = excessiveValueOffset + excessiveValues.Length; + var bytes = new byte[excessivePayloadOffset + excessivePayload.Length]; + payloadBytes.CopyTo(bytes, 0); + valueBytes.CopyTo(bytes, valueOffset); + excessiveValues.CopyTo(bytes, excessiveValueOffset); + excessivePayload.CopyTo(bytes, excessivePayloadOffset); + + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + + System.Threading.Tasks.Parallel.For(0, 16, i => + { + if (i % 4 == 0) + { + var decoded = Assert.IsType>(decoder.Decode(arrayOffset, out _)); + Assert.Equal(pointerCount, decoded.Count); + } + else if (i % 4 == 1) + { + var decoded = Assert.IsType>(decoder.Decode(valueOffset, out _)); + Assert.Equal(childCount, decoded.Count); + } + else if (i % 4 == 2) + { + var error = Assert.Throws(() => decoder.Decode(excessiveValueOffset, out _)); + Assert.Contains("maximum number of values", error.Message); + } + else + { + var error = Assert.Throws(() => decoder.Decode(excessivePayloadOffset + excessiveArrayOffset, out _)); + Assert.Contains("maximum payload size", error.Message); + } + }); + } + + [Theory] + [InlineData(32)] + [InlineData(33)] + public static void TestContainerDepthAroundStackProbeDecodes(int containerCount) + { + // Exercise alternating maps and arrays on both sides of the + // runtime stack-probe threshold. + var bytes = NestedContainers(containerCount); + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + + decoder.Decode(0, out var offset); + Assert.Equal(bytes.Length, offset); + } + + [Fact] + public static void TestContainerDepthBoundaryRejectsOneOverTheLimit() + { + // One container beyond the depth limit must be rejected. + var bytes = NestedContainers(513); + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Equal("The MaxMind DB file's data section exceeds the maximum depth.", ex.Message); + } + + [Fact] + public static void TestContainerDepthAtLimitSucceedsGivenSufficientStack() + { + // Give the thread enough stack to verify that exactly 512 + // container levels decode. + var bytes = NestedContainers(512); + Exception? failure = null; + + var thread = new Thread(() => + { + try + { + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + decoder.Decode(0, out var offset); + Assert.Equal(bytes.Length, offset); + } + catch (Exception ex) + { + failure = ex; + } + }, maxStackSize: 16 << 20); + thread.Start(); + thread.Join(); + + if (failure != null) + { + ExceptionDispatchInfo.Capture(failure).Throw(); + } + } + + [Fact] + public static void TestContainerDepthAtLimitDoesNotCrashTheHostOnADefaultStack() + { + // A default stack may be too small for 512 levels. Require success + // or a catchable depth error, rather than host termination. + var bytes = NestedContainers(512); + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + + try + { + decoder.Decode(0, out var offset); + Assert.Equal(bytes.Length, offset); + } + catch (InvalidDatabaseException ex) + { + Assert.Equal("The MaxMind DB file's data section exceeds the maximum depth.", ex.Message); + } + } + + [Fact] + public static void TestCyclicPointerThrows() + { + // Reject a self-pointer before following it again. + using var database = new MemoryMapBuffer(new MemoryStream([0x20, 0x00], writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("pointer to another pointer", ex.Message); + } + + private sealed class KeyOnlyModel + { + [Constructor] + public KeyOnlyModel([MapKey("name")] string? name = null) => Name = name; + + public string? Name { get; } + } + + [Fact] + public static void TestOversizedMapIsBounded() + { + // Declare 32,769 map entries without a body. Charging both keys + // and values must exceed the budget before the first read. + using var database = new MemoryMapBuffer(new MemoryStream([0xfe, 0x7e, 0xe4], writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum number of values", ex.Message); + } + + [Theory] + [InlineData(32_767, false)] + [InlineData(32_768, true)] + public static void TestMapValueBudgetBoundary(int entryCount, bool exceedsLimit) + { + var size = entryCount - 285; + var bytes = new List { 0xFE, (byte)(size >> 8), (byte)size }; + if (!exceedsLimit) + { + for (var i = 0; i < entryCount; i++) + { + bytes.Add(0x44); + bytes.AddRange(Encoding.ASCII.GetBytes(i.ToString("X4", System.Globalization.CultureInfo.InvariantCulture))); + bytes.AddRange([0x00, 0x07]); // false + } + } + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + if (exceedsLimit) + { + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum number of values", error.Message); + } + else + { + var record = decoder.Decode>(0, out var offset); + Assert.Equal(entryCount, record.Count); + Assert.False(Assert.IsType(record["7FFE"])); + Assert.Equal(bytes.Count, offset); + } + } + + [Fact] + public static void TestUnknownFieldValueCountIsBounded() + { + // The root map already charges its key and value. The unknown value + // is a complete array whose 65,535 children exceed the remaining + // budget. Skipping it must enforce the same limit as decoding it. + const int childCount = 65_535; + var bytes = new List(childCount * 2 + 16) + { + 0xE1, + 0x47, + (byte)'u', (byte)'n', (byte)'k', (byte)'n', (byte)'o', (byte)'w', (byte)'n', + 0x1E, 0x04, 0xFE, 0xE2, + }; + for (var i = 0; i < childCount; i++) + { + bytes.Add(0x00); // extended boolean with value false + bytes.Add(0x07); + } + + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum number of values", ex.Message); + } + + [Fact] + public static void TestUnknownFieldDepthIsBounded() + { + // The unknown map value begins at depth one. Its 512th nested + // container therefore exceeds the maximum depth while being + // skipped, without any pointers in the data. + var nested = NestedContainers(513); + var bytes = new List(nested.Length + 9) + { + 0xE1, + 0x47, + (byte)'u', (byte)'n', (byte)'k', (byte)'n', (byte)'o', (byte)'w', (byte)'n', + }; + bytes.AddRange(nested); + + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum depth", ex.Message); + } + + [Theory] + [InlineData(1, false)] + [InlineData(2, true)] + [InlineData(511, true)] + [InlineData(512, true)] + public static void TestMapKeyPointerChainPreservesValueOffset(int pointerCount, bool exceedsLimit) + { + // Place the pointer targets before the map so that its following + // value is only found by retaining the first pointer's end offset. + var bytes = new List { 0x44, (byte)'n', (byte)'a', (byte)'m', (byte)'e' }; + var target = 0; + for (var i = 1; i < pointerCount; i++) + { + var pointerOffset = bytes.Count; + WritePointer1(bytes, target); + target = pointerOffset; + } + var mapOffset = bytes.Count; + bytes.Add(0xE1); + WritePointer1(bytes, target); + bytes.AddRange([0x43, (byte)'v', (byte)'a', (byte)'l']); + + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + if (exceedsLimit) + { + var error = Assert.Throws(() => decoder.Decode(mapOffset, out _)); + Assert.Contains("pointer to another pointer", error.Message); + } + else + { + var record = decoder.Decode(mapOffset, out var offset); + Assert.Equal("val", record.Name); + Assert.Equal(bytes.Count, offset); + } + } + + [Fact] + public static void TestCyclicPointerAsMapKeyThrows() + { + // Model keys use DecodeKey rather than the dictionary path. + // The key pointer at offset 1 targets itself. + using var database = new MemoryMapBuffer(new MemoryStream([0xe1, 0x20, 0x01], writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("pointer to another pointer", ex.Message); + } + + // These headers declare oversized values without a body. Expect + // a limit error, proving rejection occurs before a payload read. + + [Fact] + public static void TestOversizedArrayIsRejectedBeforeFirstChild() + { + // Declare 65,536 children, one more than the root leaves available. + using var database = new MemoryMapBuffer(new MemoryStream([0x1e, 0x04, 0xfe, 0xe3], writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum number of values", ex.Message); + } + + [Fact] + public static void TestOversizedStringIsRejectedBeforeItIsRead() + { + // Declare 2 MiB + 1 string bytes without a payload. + using var database = new MemoryMapBuffer( + new MemoryStream([0x5f, 0x1e, 0xfe, 0xe4], writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum payload size", ex.Message); + } + + [Fact] + public static void TestOversizedBytesIsRejectedBeforeItIsRead() + { + // Declare 2 MiB + 1 bytes without a payload. + using var database = new MemoryMapBuffer( + new MemoryStream([0x9f, 0x1e, 0xfe, 0xe4], writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum payload size", ex.Message); + } + + [Fact] + public static void TestTruncatedPayloadThrowsDatabaseException() + { + // A string header declaring four bytes at the end of the buffer. + // Truncated data is malformed input, so it must surface as the + // reader's database exception rather than an argument exception. + using var database = new MemoryMapBuffer(new MemoryStream([0x44], writable: false)); + var decoder = new Decoder(database, 0); + var ex = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("beyond the end", ex.Message); + } + + [Theory] + [InlineData(new byte[] { 0xC4 })] + [InlineData(new byte[] { 0x08, 0x02 })] + public static void TestTruncatedIntegerThrowsDatabaseException(byte[] bytes) + { + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("beyond the end", error.Message); + } + + [Fact] + public static void TestTruncatedModelKeyThrowsDatabaseException() + { + using var database = new MemoryMapBuffer(new MemoryStream([0xE1, 0x44], writable: false)); + var decoder = new Decoder(database, 0); + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("beyond the end", error.Message); + } + + [Fact] + public static void TestBufferReadRejectsOutOfBoundsOffsets() + { + // Exercise the shared bounds check through the buffer API. + using var database = new MemoryMapBuffer(new MemoryStream([0x01, 0x02, 0x03, 0x04], writable: false)); + + // A read that ends exactly at Length is accepted. + Assert.Equal(new byte[] { 0x01, 0x02, 0x03, 0x04 }, database.Read(0, 4)); + + // A read that ends one byte past Length is rejected. + var pastEnd = Assert.Throws(() => database.Read(0, 5)); + Assert.Contains("beyond the end", pastEnd.Message); + + // A negative offset is rejected. + var negativeOffset = Assert.Throws(() => database.Read(-1, 1)); + Assert.Contains("beyond the end", negativeOffset.Message); + } + + private static byte[] PointerChain(int length) + { + // Each pointer targets the next two-byte link. The chain ends + // in a uint16 zero at offset 2 * length. + var bytes = new List(length * 2 + 1); + for (var i = 0; i < length; i++) + { + WritePointer1(bytes, 2 * (i + 1)); + } + bytes.Add(0xA0); // leaf: uint16 with value 0 + return [.. bytes]; + } + + [Theory] + [InlineData(1, false)] + [InlineData(2, true)] + [InlineData(512, true)] + public static void TestPointerToPointerIsRejected(int chainLength, bool invalid) + { + var bytes = PointerChain(chainLength); + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + if (invalid) + { + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("pointer to another pointer", error.Message); + } + else + { + Assert.Equal(0, Assert.IsType(decoder.Decode(0, out var offset))); + Assert.Equal(2, offset); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public static void TestTwoNodePointerCycleIsRejected(bool mapKey) + { + var bytes = new List { 0x20, 0x02, 0x20, 0x00, 0xE1, 0x20, 0x00 }; + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + var error = Assert.Throws(() => + { + if (mapKey) + { + decoder.Decode(4, out _); + } + else + { + decoder.Decode(0, out _); + } + }); + Assert.Contains("pointer to another pointer", error.Message); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public static void TestPointerTargetIsRejectedBeforeReadingItsPayload(bool mapKey) + { + byte[] bytes; + if (mapKey) + { + bytes = [0xE1, 0x20, 0x03, 0x20]; + } + else + { + bytes = [0x20, 0x02, 0x20]; + } + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + var error = Assert.Throws(() => + { + if (mapKey) + { + decoder.Decode(0, out _); + } + else + { + decoder.Decode(0, out _); + } + }); + Assert.Contains("pointer to another pointer", error.Message); + } + + [Theory] + [InlineData(255, false)] + [InlineData(256, false)] + [InlineData(257, true)] + public static void TestDepthAccumulatesAcrossPointersToContainers(int count, bool exceedsLimit) + { + // Every pointer targets a one-element array. Each pair costs two + // levels, and no pointer directly targets another pointer. + var bytes = new List(); + for (var i = 0; i < count; i++) + { + WritePointer1(bytes, bytes.Count + 2); + bytes.AddRange([0x01, 0x04]); + } + bytes.Add(0xA0); + Exception? failure = null; + var thread = new Thread(() => + { + try + { + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + if (exceedsLimit) + { + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum depth", error.Message); + } + else + { + object value = decoder.Decode(0, out var offset); + Assert.Equal(2, offset); + for (var i = 0; i < count; i++) + { + value = Assert.Single(Assert.IsType>(value)); + } + Assert.Equal(0, Assert.IsType(value)); + } + } + catch (Exception ex) + { + failure = ex; + } + }, maxStackSize: 16 << 20); + thread.Start(); + thread.Join(); + if (failure != null) + { + ExceptionDispatchInfo.Capture(failure).Throw(); + } + } + + // Each array slot follows the shared chain again. This exercises many + // pointer follows within the depth and value limits, without a timing assertion. + private static byte[] ManySlotsEachFollowingALongPointerChain(int slotCount, int chainLength, out int arrayOffset) + { + var chain = PointerChain(chainLength); + var bytes = new List(chain.Length + slotCount * 2 + 8); + bytes.AddRange(chain); + arrayOffset = bytes.Count; + var encodedSize = slotCount - 285; + bytes.Add(0x1E); // array with a two-byte encoded size + bytes.Add(0x04); + bytes.Add((byte)(encodedSize >> 8)); + bytes.Add((byte)encodedSize); + for (var i = 0; i < slotCount; i++) + { + WritePointer1(bytes, 0); + } + + return [.. bytes]; + } + + [Fact] + public static void TestManySlotsEachFollowingALongPointerChainIsRejected() + { + // A shared chain could previously amplify one lookup into tens + // of millions of pointer follows while staying within the limits. + const int slotCount = 65_534; + const int chainLength = 509; + var bytes = ManySlotsEachFollowingALongPointerChain(slotCount, chainLength, out var arrayOffset); + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + + var error = Assert.Throws(() => decoder.Decode(arrayOffset, out _)); + Assert.Contains("pointer to another pointer", error.Message); + } + + [Theory] + [InlineData(6, 4)] + [InlineData(9, 8)] + [InlineData(10, 16)] + public static void TestIntegerWidthBoundaries(int type, int maximumSize) + { + foreach (var size in new[] { 0, maximumSize }) + { + var bytes = EncodedInteger(type, size, size); + bytes.Add(0xA0); // next value: uint16 zero + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + var value = decoder.Decode(0, out var offset); + var expected = (BigInteger.One << (size * 8)) - 1; + if (type == 6) + { + Assert.Equal((long)expected, Assert.IsType(value)); + } + else if (type == 9) + { + Assert.Equal((ulong)expected, Assert.IsType(value)); + } + else + { + Assert.Equal(expected, Assert.IsType(value)); + } + Assert.Equal(bytes.Count - 1, offset); + Assert.Equal(0, Assert.IsType(decoder.Decode(offset, out offset))); + Assert.Equal(bytes.Count, offset); + } + } + + [Theory] + [InlineData(6, 4)] + [InlineData(9, 8)] + [InlineData(10, 16)] + public static void TestOversizedIntegerRejectsBeforeReading(int type, int maximumSize) + { + foreach (var payloadSize in new[] { 0, maximumSize + 1 }) + { + var bytes = EncodedInteger(type, maximumSize + 1, payloadSize); + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains($"larger than {maximumSize} bytes", error.Message); + } + } + + [Theory] + [InlineData(6, 4)] + [InlineData(9, 8)] + [InlineData(10, 16)] + public static void TestIntegerSizeLimitPrecedesPayloadLimit(int type, int maximumSize) + { + // Declare 2 MiB + 1 bytes without a body. Reject the width before + // charging payload or attempting a read. + var bytes = new List(); + if (type == 6) + { + bytes.Add(0xDF); + } + else + { + bytes.Add(0x1F); + bytes.Add((byte)(type - 7)); + } + bytes.AddRange([0x1E, 0xFE, 0xE4]); + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains($"larger than {maximumSize} bytes", error.Message); + } + + [Theory] + [InlineData(6, 4)] + [InlineData(9, 8)] + [InlineData(10, 16)] + public static void TestValidIntegerSharesPayloadBudgetWithString(int type, int maximumSize) + { + foreach (var exceedsLimit in new[] { false, true }) + { + foreach (var stringFirst in new[] { false, true }) + { + var stringSize = (1 << 21) - maximumSize; + if (exceedsLimit) + { + stringSize++; + } + var encodedSize = stringSize - 65821; + var text = new List { 0x5F, (byte)(encodedSize >> 16), (byte)(encodedSize >> 8), (byte)encodedSize }; + text.AddRange(Encoding.UTF8.GetBytes(new string('a', stringSize))); + var integer = EncodedInteger(type, maximumSize, maximumSize); + var bytes = new List { 0x02, 0x04 }; // array of two values + if (stringFirst) + { + bytes.AddRange(text); + bytes.AddRange(integer); + } + else + { + bytes.AddRange(integer); + bytes.AddRange(text); + } + using var database = new MemoryMapBuffer(new MemoryStream(bytes.ToArray(), writable: false)); + var decoder = new Decoder(database, 0); + if (exceedsLimit) + { + var error = Assert.Throws(() => decoder.Decode(0, out _)); + Assert.Contains("maximum payload size", error.Message); + } + else + { + var values = Assert.IsType>(decoder.Decode(0, out var offset)); + Assert.Equal(2, values.Count); + Assert.Contains(values, value => value is string textValue && textValue.Length == stringSize); + Assert.Equal(bytes.Count, offset); + } + } + } + } + + private static List EncodedInteger(int type, int size, int payloadSize) + { + var bytes = new List(); + if (type == 6) + { + bytes.Add((byte)(0xC0 | size)); + } + else + { + bytes.Add((byte)size); + bytes.Add((byte)(type - 7)); + } + for (var i = 0; i < payloadSize; i++) + { + bytes.Add(0xFF); + } + return bytes; + } + public static IEnumerable TestUInt16() { var uint16s = new Dictionary @@ -194,6 +1051,168 @@ public static IEnumerable TestFloats() yield return [floats]; } + [Theory] + [InlineData(1, false)] + [InlineData(2, false)] + [InlineData(3, false)] + [InlineData(4, false)] + [InlineData(1, true)] + [InlineData(2, true)] + [InlineData(3, true)] + [InlineData(4, true)] + public static void TestPointerWidthsPreserveValueAndOffset(int width, bool mapKey) + { + // Use the smallest encoded target that leaves room for the record. + // Two- and three-byte pointers have an implicit offset added. + var target = 16; + if (width == 2) + { + target += 1 << 11; + } + else if (width == 3) + { + target += (1 << 19) + (1 << 11); + } + var bytes = new byte[target + 5]; + bytes[0] = 0xA0; // scalar target for the pointer inside the array + var start = 1; + if (mapKey) + { + bytes[start++] = 0xE1; + } + bytes[start] = (byte)(0x20 | ((width - 1) << 3)); + bytes[start + width] = 16; + var end = start + width + 1; + if (mapKey) + { + bytes[end++] = 0x43; + bytes[end++] = (byte)'v'; + bytes[end++] = (byte)'a'; + bytes[end++] = (byte)'l'; + new byte[] { 0x44, (byte)'n', (byte)'a', (byte)'m', (byte)'e' }.CopyTo(bytes, target); + } + else + { + // A pointer to a container containing a pointer remains valid. + new byte[] { 0x01, 0x04, 0x20, 0x00 }.CopyTo(bytes, target); + } + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + if (mapKey) + { + var record = decoder.Decode(1, out var offset); + Assert.Equal("val", record.Name); + Assert.Equal(end, offset); + } + else + { + var record = decoder.Decode>(1, out var offset); + Assert.Equal(0, Assert.IsType(Assert.Single(record))); + Assert.Equal(end, offset); + } + var rawDecoder = new Decoder(database, 0, false); + Assert.Equal((long)target, Assert.IsType(rawDecoder.Decode(start, out var rawOffset))); + Assert.Equal(start + width + 1, rawOffset); + } + + [Fact] + public static void TestSkippedPointerDoesNotValidateItsTarget() + { + byte[] bytes = [0xE2, 0x44, (byte)'s', (byte)'k', (byte)'i', (byte)'p', + 0x20, 17, 0x44, (byte)'n', (byte)'a', (byte)'m', (byte)'e', + 0x43, (byte)'v', (byte)'a', (byte)'l', 0x20]; + // The unknown field points at a pointer header with no payload. + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + var record = decoder.Decode(0, out var offset); + Assert.Equal("val", record.Name); + Assert.Equal(17, offset); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(7)] + public static void TestFourBytePointerIgnoresLowControlBits(int lowBits) + { + byte[] bytes = [(byte)(0x38 | lowBits), 0, 0, 0, 5, 0xA1, 7]; + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + Assert.Equal(7, Assert.IsType(decoder.Decode(0, out var offset))); + Assert.Equal(5, offset); + var rawDecoder = new Decoder(database, 0, false); + Assert.Equal(5L, Assert.IsType(rawDecoder.Decode(0, out offset))); + Assert.Equal(5, offset); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(7)] + public static void TestFourByteMapKeyPointerPreservesValueOffset(int lowBits) + { + byte[] bytes = [0x44, (byte)'n', (byte)'a', (byte)'m', (byte)'e', + 0xE1, (byte)(0x38 | lowBits), 0, 0, 0, 0, + 0x43, (byte)'v', (byte)'a', (byte)'l']; + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + var model = decoder.Decode(5, out var offset); + Assert.Equal("val", model.Name); + Assert.Equal(bytes.Length, offset); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(7)] + public static void TestSkippedFourBytePointerPreservesNextField(int lowBits) + { + byte[] bytes = [0xE2, 0x41, (byte)'x', (byte)(0x38 | lowBits), 0, 0, 0, 0, + 0x44, (byte)'n', (byte)'a', (byte)'m', (byte)'e', + 0x43, (byte)'v', (byte)'a', (byte)'l']; + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0); + var model = decoder.Decode(0, out var offset); + Assert.Equal("val", model.Name); + Assert.Equal(bytes.Length, offset); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(7)] + public static void TestTruncatedFourBytePointerThrows(int lowBits) + { + for (var payloadSize = 0; payloadSize < 4; payloadSize++) + { + var bytes = new byte[payloadSize + 1]; + bytes[0] = (byte)(0x38 | lowBits); + using var database = new MemoryMapBuffer(new MemoryStream(bytes, writable: false)); + var decoder = new Decoder(database, 0, false); + Assert.Throws(() => decoder.Decode(0, out _)); + } + } + public static IEnumerable TestPointers() { var pointers = new Dictionary diff --git a/MaxMind.Db.Test/MemoryMapBufferTest.cs b/MaxMind.Db.Test/MemoryMapBufferTest.cs new file mode 100644 index 00000000..caa519fa --- /dev/null +++ b/MaxMind.Db.Test/MemoryMapBufferTest.cs @@ -0,0 +1,39 @@ +using System.IO; +using Xunit; + +namespace MaxMind.Db.Test +{ + public static class MemoryMapBufferTest + { + [Theory] + [InlineData(0, 5)] + [InlineData(3, 2)] + [InlineData(-1, 1)] + [InlineData(long.MaxValue, 1)] + public static void ReadsRejectOutOfBoundsRanges(long offset, int count) + { + using var buffer = new MemoryMapBuffer(new MemoryStream(new byte[4], writable: false)); + using var other = new MemoryMapBuffer(new MemoryStream(new byte[8], writable: false)); + Assert.Throws(() => buffer.HashBytes(offset, count)); + Assert.Throws(() => buffer.ReadLong(offset, count)); + Assert.Throws(() => buffer.ReadULong(offset, count)); + Assert.Throws(() => buffer.EqualsBytes(offset, new byte[8], 0, count)); + // Both operands must be checked, even when all bytes compare equal. + Assert.Throws(() => buffer.EqualsBytes(offset, other, 0, count)); + Assert.Throws(() => other.EqualsBytes(0, buffer, offset, count)); + } + + [Fact] + public static void ReadsAcceptRangesEndingAtTheBufferBoundary() + { + using var buffer = new MemoryMapBuffer(new MemoryStream([0, 1, 2, 3], writable: false)); + using var other = new MemoryMapBuffer(new MemoryStream([1, 2, 3], writable: false)); + Assert.Equal(0x010203, buffer.ReadLong(1, 3)); + Assert.Equal(0x010203UL, buffer.ReadULong(1, 3)); + Assert.Equal(other.HashBytes(0, 3), buffer.HashBytes(1, 3)); + Assert.True(buffer.EqualsBytes(1, other, 0, 3)); + Assert.True(buffer.EqualsBytes(1, new byte[] { 1, 2, 3 }, 0, 3)); + Assert.False(buffer.EqualsBytes(1, new byte[] { 1, 2, 4 }, 0, 3)); + } + } +} diff --git a/MaxMind.Db.Test/ReaderTest.cs b/MaxMind.Db.Test/ReaderTest.cs index 7ff24c15..92fd6fe8 100644 --- a/MaxMind.Db.Test/ReaderTest.cs +++ b/MaxMind.Db.Test/ReaderTest.cs @@ -681,6 +681,244 @@ private static void TestAddresses(Reader reader, string file, IEnumerable( + () => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains(expected, ex.Message); + } + + [Fact] + public void TestPayloadAtLimitDecodes() + { + // Exactly 2 MiB of bytes payload must decode. + using var reader = new Reader( + Path.Combine(_testDataRoot, "MaxMind-DB-test-decoder-payload-limit.mmdb")); + var result = reader.Find(IPAddress.Parse("1.1.1.1")); + var list = Assert.IsType>(result); + Assert.Equal(33, list.Count); + } + + [Fact] + public void TestPayloadOverLimitIsRejected() + { + // One byte over 2 MiB must be rejected. + using var reader = new Reader( + Path.Combine(_testDataRoot, "MaxMind-DB-test-decoder-payload-limit-over.mmdb")); + var ex = Assert.Throws( + () => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains("maximum payload size", ex.Message); + } + + [Fact] + public void TestMetadataPayloadLimitIsRejectedOnOpen() + { + // The metadata languages array alone exceeds the payload limit. + var ex = Assert.Throws( + () => new Reader( + Path.Combine(_testDataRoot, "MaxMind-DB-test-metadata-payload-limit.mmdb"))); + Assert.Contains("maximum payload size", ex.Message); + } + + // The shared array targets must exhaust the value budget through + // the real tree walk and pointer base. + [Theory] + [InlineData("MaxMind-DB-test-pointer-decoder-dos.mmdb", "1.1.1.1")] + [InlineData("MaxMind-DB-test-pointer-decoder-dos-ipv6.mmdb", "::1")] + public void TestPointerFanOutFixtureIsRejected(string fixture, string address) + { + using var reader = new Reader(Path.Combine(_testDataRoot, fixture)); + var ex = Assert.Throws( + () => reader.Find(IPAddress.Parse(address))); + Assert.Contains("maximum number of values", ex.Message); + } + + [Fact] + public void TestValueCountAtLimitDecodes() + { + // Exactly 65,536 decoded values, the boundary the limit allows. + using var reader = new Reader( + Path.Combine(_testDataRoot, "MaxMind-DB-test-decoder-value-limit.mmdb")); + for (var i = 0; i < 3; i++) + { + var values = Assert.IsType>(reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Equal(65_535, values.Count); + } + } + + [Fact] + public void TestValueCountOverLimitIsRejected() + { + // One value past 65,536. Catches an off-by-one in the comparison. + using var reader = new Reader( + Path.Combine(_testDataRoot, "MaxMind-DB-test-decoder-value-limit-over.mmdb")); + var ex = Assert.Throws( + () => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains("maximum number of values", ex.Message); + } + + [Fact] + public void TestPointerHeavyValueCountDecodes() + { + // 65,535 values reached through a depth-15 pointer fan-out, one + // under the limit. A reader that over-counts a followed pointer + // would reject this conformant database. + using var reader = new Reader( + Path.Combine(_testDataRoot, "MaxMind-DB-test-decoder-value-limit-pointer-heavy.mmdb")); + Assert.NotNull(reader.Find(IPAddress.Parse("1.1.1.1"))); + } + + // Exercise the memory-loading path with the same hostile fixture. + [Fact] + public void TestPayloadLimitAppliesToMemoryMode() + { + var path = Path.Combine(_testDataRoot, "MaxMind-DB-test-payload-amplification-dos.mmdb"); + using var reader = new Reader(path, FileAccessMode.Memory); + var ex = Assert.Throws( + () => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains("maximum payload size", ex.Message); + } + + [Fact] + public void TestPayloadLimitAppliesToStreamConstruction() + { + var path = Path.Combine(_testDataRoot, "MaxMind-DB-test-payload-amplification-dos.mmdb"); + using var stream = File.OpenRead(path); + using var reader = new Reader(stream); + var ex = Assert.Throws( + () => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains("maximum payload size", ex.Message); + } + + [Fact] + public async Task TestPayloadLimitAppliesToAsyncConstruction() + { + var path = Path.Combine(_testDataRoot, "MaxMind-DB-test-payload-amplification-dos.mmdb"); + using var reader = await Reader.CreateAsync(path); + var ex = Assert.Throws( + () => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains("maximum payload size", ex.Message); + } + + [Fact] + public void TestPayloadLimitAppliesToFindAll() + { + // Enumeration must reject the hostile record when it is decoded. + var path = Path.Combine(_testDataRoot, "MaxMind-DB-test-payload-amplification-dos.mmdb"); + using var reader = new Reader(path); + var ex = Assert.Throws( + () => + { + foreach (var node in reader.FindAll()) + { + _ = node; + } + }); + Assert.Contains("maximum payload size", ex.Message); + } + + [Theory] + [InlineData(FileAccessMode.MemoryMapped)] + [InlineData(FileAccessMode.Memory)] + public void TestTruncatedRecordThrowsDatabaseException(FileAccessMode mode) + { + // The string declares 4,096 bytes, extending past the metadata and + // the end of this small database. Its header is inside the file. + var bytes = DatabaseWithRootRecords([0x5E, 0x0E, 0xE3], [0xA0]); + var path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, bytes); + using var reader = new Reader(path, mode); + var error = Assert.Throws(() => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains("beyond the end", error.Message); + } + finally + { + File.Delete(path); + } + } + + [Theory] + [InlineData(FileAccessMode.MemoryMapped)] + [InlineData(FileAccessMode.Memory)] + public void TestReaderRecoversAfterModelLimitFailure(FileAccessMode mode) + { + // Populate one constructor argument before the next field exceeds + // the value limit. The successful record omits that first field. + byte[] hostile = [0xE2, 0x44, (byte)'n', (byte)'a', (byte)'m', (byte)'e', + 0x43, (byte)'b', (byte)'a', (byte)'d', + 0x46, (byte)'v', (byte)'a', (byte)'l', (byte)'u', (byte)'e', (byte)'s', + 0x1E, 0x04, 0xFE, 0xE2]; + byte[] valid = [0xE1, 0x46, (byte)'v', (byte)'a', (byte)'l', (byte)'u', (byte)'e', (byte)'s', + 0x01, 0x04, 0x01, 0x07]; // one-element array containing true + var path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, DatabaseWithRootRecords(hostile, valid)); + using var reader = new Reader(path, mode); + for (var i = 0; i < 3; i++) + { + var error = Assert.Throws(() => reader.Find(IPAddress.Parse("1.1.1.1"))); + Assert.Contains("maximum number of values", error.Message); + var record = reader.Find(IPAddress.Parse("129.1.1.1")); + Assert.NotNull(record); + Assert.Null(record.Name); + Assert.NotNull(record.Values); + Assert.True(Assert.IsType(Assert.Single(record.Values))); + } + } + finally + { + File.Delete(path); + } + } + + private sealed class RecoveryRecord + { + [Constructor] + public RecoveryRecord(string? name = null, List? values = null) + { + Name = name; + Values = values; + } + + public string? Name { get; } + public List? Values { get; } + } + + private byte[] DatabaseWithRootRecords(byte[] left, byte[] right) + { + var path = Path.Combine(_testDataRoot, "MaxMind-DB-test-ipv4-24.mmdb"); + using var source = new Reader(path); + var original = File.ReadAllBytes(path); + byte[] marker = [0xAB, 0xCD, 0xEF, (byte)'M', (byte)'a', (byte)'x', (byte)'M', (byte)'i', (byte)'n', (byte)'d', (byte)'.', (byte)'c', (byte)'o', (byte)'m']; + var metadataOffset = Enumerable.Range(0, original.Length - marker.Length + 1) + .Last(i => original.Skip(i).Take(marker.Length).SequenceEqual(marker)); + var bytes = new List(original.Take(metadataOffset)); + var leftPointer = bytes.Count - source.Metadata.SearchTreeSize + source.Metadata.NodeCount; + bytes.AddRange(left); + var rightPointer = bytes.Count - source.Metadata.SearchTreeSize + source.Metadata.NodeCount; + bytes.AddRange(right); + bytes.AddRange(original.Skip(metadataOffset)); + // Root children select the two records by the address's first bit. + for (var i = 0; i < 3; i++) + { + bytes[i] = (byte)(leftPointer >> (16 - 8 * i)); + bytes[i + 3] = (byte)(rightPointer >> (16 - 8 * i)); + } + return [.. bytes]; + } + private static void TestMetadata(Reader reader, int ipVersion) { var metadata = reader.Metadata; diff --git a/MaxMind.Db.Test/TestData/MaxMind-DB b/MaxMind.Db.Test/TestData/MaxMind-DB index b2a3df13..363086b7 160000 --- a/MaxMind.Db.Test/TestData/MaxMind-DB +++ b/MaxMind.Db.Test/TestData/MaxMind-DB @@ -1 +1 @@ -Subproject commit b2a3df13c0e274d7a2dca3d5415465a3a9670e23 +Subproject commit 363086b7d90650100e91f954937794c6a090c2a0 diff --git a/MaxMind.Db/Decoder.cs b/MaxMind.Db/Decoder.cs index 1e2e6814..cc0d884b 100644 --- a/MaxMind.Db/Decoder.cs +++ b/MaxMind.Db/Decoder.cs @@ -7,6 +7,7 @@ using System.Collections; using System.Collections.Generic; using System.Numerics; +using System.Runtime.CompilerServices; #endregion @@ -45,6 +46,84 @@ internal sealed class Decoder private readonly bool _followPointers; private readonly int[] _pointerValueOffset = [0, 0, 1 << 11, (1 << 19) + (1 << 11), 0]; + // Per-lookup limits recommended by the MaxMind DB specification. + // The root costs one value. Containers charge their declared children + // on every visit, including visits through shared pointers. Pointer + // follows add depth but no value charge of their own. + private const int MaxDepth = 512; + private const int MaxDecodedValues = 1 << 16; + // Limit encoded string, bytes, uint32, uint64, and uint128 payload + // per lookup. This also rejects single values larger than 2 MiB, + // even when their size is permitted by the file format. + private const int MaxPayloadBytes = 1 << 21; + // The runtime stack can run out before the depth limit. Probe only + // at deeper levels to avoid the cost on shallow records. + private const int RuntimeStackCheckDepth = 32; + + private static bool HasSufficientExecutionStack() + { +#if NETSTANDARD2_0 + // netstandard2.0 exposes only the throwing stack probe. + try + { + RuntimeHelpers.EnsureSufficientExecutionStack(); + return true; + } + catch (InsufficientExecutionStackException) + { + return false; + } +#else + return RuntimeHelpers.TryEnsureSufficientExecutionStack(); +#endif + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CheckDepth(int depth) + { + // Delay the stack probe beyond ordinary record depths. Rejection + // can mean excessive nesting or insufficient runtime stack space. + if (depth < RuntimeStackCheckDepth) + { + return; + } + + if (depth >= MaxDepth || !HasSufficientExecutionStack()) + { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum depth."); + } + } + + // Check depth and charge declared children before reading or allocating. + // The budget is nonnegative on entry. CtrlData limits valueCount + // to less than 34 million, so subtraction cannot overflow. A negative + // result throws before another subtraction can occur. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CheckContainer(int depth, int valueCount, ref int budget) + { + CheckDepth(depth); + budget -= valueCount; + if (budget < 0) + { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum number of values."); + } + } + + // Charge encoded payload before reading it, including each visit + // to a shared target and each map key examined. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ConsumePayload(int size, ref int payloadBudget) + { + if (size > payloadBudget) + { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum payload size."); + } + payloadBudget -= size; + } + private readonly DictionaryActivatorCreator _dictionaryActivatorCreator; private readonly ListActivatorCreator _listActivatorCreator; @@ -74,19 +153,38 @@ internal Decoder(MemoryMapBuffer database, long pointerBase, bool followPointers /// An object containing the data read from the stream internal T Decode(long offset, out long outOffset, InjectableValues? injectables = null, Network? network = default) where T : class { - if (Decode(typeof(T), offset, out outOffset, injectables, network) is not T decoded) + // Budgets are local to each lookup, including concurrent lookups. + // Charge the root value before decoding its children. + var budget = MaxDecodedValues - 1; + var payloadBudget = MaxPayloadBytes; + return DecodeNested(offset, out outOffset, 0, ref budget, ref payloadBudget, injectables, network); + } + + private T DecodeNested(long offset, out long outOffset, int depth, ref int budget, ref int payloadBudget, InjectableValues? injectables, Network? network) where T : class + { + if (Decode(typeof(T), offset, out outOffset, depth, ref budget, ref payloadBudget, injectables, network) is not T decoded) { throw new InvalidDatabaseException("The value cannot be decoded as " + typeof(T)); } return decoded; } - private object Decode(Type expectedType, long offset, out long outOffset, InjectableValues? injectables = null, Network? network = null) + private object Decode(Type expectedType, long offset, out long outOffset, int depth, ref int budget, ref int payloadBudget, InjectableValues? injectables = null, Network? network = null) { + // Depth and value checks apply at containers and pointers. + // Scalars charge payload where applicable. var type = CtrlData(offset, out var size, out offset); - return DecodeByType(expectedType, type, offset, size, out outOffset, injectables, network); + if (type == ObjectType.Pointer || type == ObjectType.Map || type == ObjectType.Array) + { + return DecodeContainer(expectedType, type, offset, size, out outOffset, depth, ref budget, ref payloadBudget, injectables, network); + } + return DecodeScalar(expectedType, type, offset, size, out outOffset, ref payloadBudget); } + /// + /// Reads the type and size. For pointers, size contains the five + /// raw control bits. For other types, it contains the expanded size. + /// private ObjectType CtrlData(long offset, out int size, out long outOffset) { if (offset >= _database.Length) @@ -118,6 +216,12 @@ private ObjectType CtrlData(long offset, out int size, out long outOffset) // The size calculation is inlined as it is hot code size = ctrlByte & 0x1f; + // Pointer control bits encode the pointer width and value, not a payload size. + if (type == ObjectType.Pointer) + { + outOffset = offset; + return type; + } if (size >= 29) { var bytesToRead = size - 28; @@ -134,23 +238,29 @@ private ObjectType CtrlData(long offset, out int size, out long outOffset) } /// - /// Decodes the value by type. + /// Decodes a pointer or container. /// /// /// The type. /// The offset. /// The size. /// The out offset + /// The current nesting depth. + /// The remaining number of values that may be decoded. + /// The remaining payload budget in bytes. /// /// /// - /// Unable to handle type! - private object DecodeByType( + /// The data is invalid or exceeds a decoding limit. + private object DecodeContainer( Type expectedType, ObjectType type, long offset, int size, out long outOffset, + int depth, + ref int budget, + ref int payloadBudget, InjectableValues? injectables, Network? network ) @@ -167,21 +277,55 @@ private object DecodeByType( return pointer; } - var result = Decode(expectedType, pointer, out _, injectables, network); - return result; + // The logical slot is already charged. Following a pointer + // adds depth. Containers charge their children, and strings, + // bytes, uint32, uint64, and uint128 charge their payload. + CheckDepth(depth); + return DecodePointerTarget(expectedType, pointer, depth + 1, ref budget, ref payloadBudget, injectables, network); case ObjectType.Map: - return DecodeMap(expectedType, offset, size, out outOffset, injectables, network); + // A map entry decodes a key and a value, so it costs two values. + CheckContainer(depth, size * 2, ref budget); + return DecodeMap(expectedType, offset, size, out outOffset, depth, ref budget, ref payloadBudget, injectables, network); case ObjectType.Array: - return DecodeArray(expectedType, size, offset, out outOffset, injectables, network); + CheckContainer(depth, size, ref budget); + return DecodeArray(expectedType, size, offset, out outOffset, depth, ref budget, ref payloadBudget, injectables, network); + default: + throw new InvalidDatabaseException("Unable to handle type: " + type); + } + } + + private object DecodePointerTarget(Type expectedType, long offset, int depth, + ref int budget, ref int payloadBudget, InjectableValues? injectables, Network? network) + { + var type = CtrlData(offset, out var size, out offset); + if (type == ObjectType.Pointer) + { + throw new InvalidDatabaseException("The MaxMind DB file contains a pointer to another pointer."); + } + if (type == ObjectType.Map || type == ObjectType.Array) + { + return DecodeContainer(expectedType, type, offset, size, out _, depth, ref budget, ref payloadBudget, injectables, network); + } + return DecodeScalar(expectedType, type, offset, size, out _, ref payloadBudget); + } + + // Keep scalars out of the container dispatch frame. They need no depth, + // value budget, injectables, or network, and dominate ordinary records. + private object DecodeScalar(Type expectedType, ObjectType type, long offset, int size, + out long outOffset, ref int payloadBudget) + { + outOffset = offset + size; + switch (type) + { case ObjectType.Boolean: outOffset = offset; return DecodeBoolean(expectedType, size); case ObjectType.Utf8String: - return DecodeString(expectedType, offset, size); + return DecodeString(expectedType, offset, size, ref payloadBudget); case ObjectType.Double: return DecodeDouble(expectedType, offset, size); @@ -190,22 +334,22 @@ private object DecodeByType( return DecodeFloat(expectedType, offset, size); case ObjectType.Bytes: - return DecodeBytes(expectedType, offset, size); + return DecodeBytes(expectedType, offset, size, ref payloadBudget); case ObjectType.Uint16: return DecodeInteger(expectedType, offset, size); case ObjectType.Uint32: - return DecodeLong(expectedType, offset, size); + return DecodeLong(expectedType, offset, size, ref payloadBudget); case ObjectType.Int32: return DecodeInteger(expectedType, offset, size); case ObjectType.Uint64: - return DecodeUInt64(expectedType, offset, size); + return DecodeUInt64(expectedType, offset, size, ref payloadBudget); case ObjectType.Uint128: - return DecodeBigInteger(expectedType, offset, size); + return DecodeBigInteger(expectedType, offset, size, ref payloadBudget); default: throw new InvalidDatabaseException("Unable to handle type: " + type); @@ -278,17 +422,19 @@ private float DecodeFloat(Type expectedType, long offset, int size) /// Decodes the string. /// /// - private string DecodeString(Type expectedType, long offset, int size) + private string DecodeString(Type expectedType, long offset, int size, ref int payloadBudget) { ReflectionUtil.CheckType(expectedType, typeof(string)); + ConsumePayload(size, ref payloadBudget); return _database.ReadString(offset, size); } - private byte[] DecodeBytes(Type expectedType, long offset, int size) + private byte[] DecodeBytes(Type expectedType, long offset, int size, ref int payloadBudget) { ReflectionUtil.CheckType(expectedType, typeof(byte[])); + ConsumePayload(size, ref payloadBudget); return _database.Read(offset, size); } @@ -299,6 +445,9 @@ private byte[] DecodeBytes(Type expectedType, long offset, int size) /// The offset. /// The size. /// The out offset. + /// The current nesting depth. + /// The remaining number of values that may be decoded. + /// The remaining payload budget in bytes. /// /// /// @@ -307,6 +456,9 @@ private object DecodeMap( long offset, int size, out long outOffset, + int depth, + ref int budget, + ref int payloadBudget, InjectableValues? injectables, Network? network ) @@ -323,14 +475,14 @@ private object DecodeMap( (SourceGeneratorSupport.HasNonGenericDictionaryRegistration && SourceGeneratorSupport.TryGetDictionaryRegistration(expectedType, out _))) { - return DecodeMapToDictionary(expectedType, offset, size, out outOffset, injectables, network); + return DecodeMapToDictionary(expectedType, offset, size, out outOffset, depth, ref budget, ref payloadBudget, injectables, network); } - return DecodeMapToType(expectedType, offset, size, out outOffset, injectables, network); + return DecodeMapToType(expectedType, offset, size, out outOffset, depth, ref budget, ref payloadBudget, injectables, network); } private object DecodeMapToDictionary(Type expectedType, long offset, int size, out long outOffset, - InjectableValues? injectables, Network? network) + int depth, ref int budget, ref int payloadBudget, InjectableValues? injectables, Network? network) { // Fast path for Dictionary (and parents). if (expectedType.IsAssignableFrom(typeof(Dictionary))) @@ -338,8 +490,8 @@ private object DecodeMapToDictionary(Type expectedType, long offset, int size, o Dictionary dic = new(size); for (var i = 0; i < size; i++) { - var key = Decode(offset, out offset); - var value = Decode(offset, out offset, injectables, network); + var key = DecodeNested(offset, out offset, depth + 1, ref budget, ref payloadBudget, null, null); + var value = DecodeNested(offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); dic.Add(key, value); } @@ -353,8 +505,8 @@ private object DecodeMapToDictionary(Type expectedType, long offset, int size, o Dictionary dic = new(size); for (var i = 0; i < size; i++) { - var key = Decode(offset, out offset); - var value = Decode(offset, out offset, injectables, network); + var key = DecodeNested(offset, out offset, depth + 1, ref budget, ref payloadBudget, null, null); + var value = DecodeNested(offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); dic.Add(key, value); } @@ -368,9 +520,9 @@ private object DecodeMapToDictionary(Type expectedType, long offset, int size, o var generatedDictionary = registration.Factory(size); for (var i = 0; i < size; i++) { - var key = Decode(registration.KeyType, offset, out offset); + var key = Decode(registration.KeyType, offset, out offset, depth + 1, ref budget, ref payloadBudget); var value = Decode( - registration.ValueType, offset, out offset, injectables, network); + registration.ValueType, offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); registration.Add(generatedDictionary, key, value); } @@ -388,8 +540,8 @@ private object DecodeMapToDictionary(Type expectedType, long offset, int size, o var obj = (IDictionary)_dictionaryActivatorCreator.GetActivator(expectedType)(size); for (var i = 0; i < size; i++) { - var key = Decode(genericArgs[0], offset, out offset); - var value = Decode(genericArgs[1], offset, out offset, injectables, network); + var key = Decode(genericArgs[0], offset, out offset, depth + 1, ref budget, ref payloadBudget); + var value = Decode(genericArgs[1], offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); obj.Add(key, value); } @@ -402,6 +554,9 @@ private object DecodeMapToType( long offset, int size, out long outOffset, + int depth, + ref int budget, + ref int payloadBudget, InjectableValues? injectables, Network? network ) @@ -421,17 +576,17 @@ private object DecodeMapToType( for (var i = 0; i < size; i++) { - var key = DecodeKey(offset, out offset); + var key = DecodeKey(offset, out offset, depth + 1, ref payloadBudget); if (constructor.DeserializationParameters.TryGetValue(key, out var v)) { var param = v; var paramType = param.MemberType; - var value = Decode(paramType, offset, out offset, injectables, network); + var value = Decode(paramType, offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); parameters[param.Position] = value; } else { - offset = NextValueOffset(offset, 1); + offset = NextValueOffset(offset, 1, depth + 1, ref budget); } } @@ -513,33 +668,41 @@ private static void SetNetwork(TypeActivator constructor, object?[] parameters, private readonly TypeActivatorCreator _typeActivatorCreator; - private Key DecodeKey(long offset, out long outOffset) + private Key DecodeKey(long offset, out long outOffset, int depth, ref int payloadBudget) { var type = CtrlData(offset, out var size, out offset); - switch (type) + if (type == ObjectType.Pointer) { - case ObjectType.Pointer: - offset = DecodePointer(offset, size, out outOffset); - return DecodeKey(offset, out _); - - case ObjectType.Utf8String: - outOffset = offset + size; - return new Key(_database, offset, size); + CheckDepth(depth); + offset = DecodePointer(offset, size, out outOffset); + type = CtrlData(offset, out size, out offset); + if (type == ObjectType.Pointer) + { + throw new InvalidDatabaseException("The MaxMind DB file contains a pointer to another pointer."); + } + } + else + { + outOffset = offset + size; + } - default: - throw new InvalidDatabaseException($"Database contains a non-string as map key: {type}"); + if (type != ObjectType.Utf8String) + { + throw new InvalidDatabaseException($"Database contains a non-string as map key: {type}"); } + + // Preserve the offset after the first pointer and charge the final + // string once. + ConsumePayload(size, ref payloadBudget); + return new Key(_database, offset, size); } - private long NextValueOffset(long offset, int numberToSkip) + // The enclosing container charged numberToSkip. Skipped containers + // still charge their children and undergo structural depth checks. + private long NextValueOffset(long offset, int numberToSkip, int depth, ref int budget) { - while (true) + while (numberToSkip > 0) { - if (numberToSkip == 0) - { - return offset; - } - var type = CtrlData(offset, out var size, out offset); switch (type) { @@ -549,11 +712,13 @@ private long NextValueOffset(long offset, int numberToSkip) break; case ObjectType.Map: - numberToSkip += 2 * size; + CheckContainer(depth, 2 * size, ref budget); + offset = NextValueOffset(offset, 2 * size, depth + 1, ref budget); break; case ObjectType.Array: - numberToSkip += size; + CheckContainer(depth, size, ref budget); + offset = NextValueOffset(offset, size, depth + 1, ref budget); break; case ObjectType.Boolean: @@ -566,18 +731,25 @@ private long NextValueOffset(long offset, int numberToSkip) numberToSkip--; } + + return offset; } /// /// Decodes the long. /// /// - private long DecodeLong(Type expectedType, long offset, int size) + private long DecodeLong(Type expectedType, long offset, int size, ref int payloadBudget) { if (expectedType != typeof(long) && expectedType != typeof(long?)) { ReflectionUtil.CheckType(expectedType, typeof(long)); } + if (size > 4) + { + throw new InvalidDatabaseException("The MaxMind DB file contains a uint32 larger than 4 bytes."); + } + ConsumePayload(size, ref payloadBudget); return _database.ReadLong(offset, size); } @@ -588,6 +760,9 @@ private long DecodeLong(Type expectedType, long offset, int size) /// The size. /// The offset. /// The out offset. + /// The current nesting depth. + /// The remaining number of values that may be decoded. + /// The remaining payload budget in bytes. /// /// /// @@ -598,7 +773,7 @@ private long DecodeLong(Type expectedType, long offset, int size) Justification = "Generated collection registrations return before this runtime generic construction path. This path serves only the documented fallback for unregistered collection types, which is unsupported in NativeAOT applications.")] #endif private object DecodeArray(Type expectedType, int size, long offset, out long outOffset, - InjectableValues? injectables, Network? network) + int depth, ref int budget, ref int payloadBudget, InjectableValues? injectables, Network? network) { // Fast path for List (and parents). if (expectedType != typeof(object) && expectedType.IsAssignableFrom(typeof(List))) @@ -606,7 +781,7 @@ private object DecodeArray(Type expectedType, int size, long offset, out long ou List list = new(size); for (var i = 0; i < size; i++) { - var r = Decode(offset, out offset, injectables, network); + var r = DecodeNested(offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); list.Add(r); } @@ -620,7 +795,7 @@ private object DecodeArray(Type expectedType, int size, long offset, out long ou List list = new(size); for (var i = 0; i < size; i++) { - var value = Decode(offset, out offset, injectables, network); + var value = DecodeNested(offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); list.Add(value); } @@ -635,7 +810,7 @@ private object DecodeArray(Type expectedType, int size, long offset, out long ou for (var i = 0; i < size; i++) { var value = Decode( - registration.ElementType, offset, out offset, injectables, network); + registration.ElementType, offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); registration.Add(generatedCollection, value); } @@ -655,7 +830,7 @@ private object DecodeArray(Type expectedType, int size, long offset, out long ou var array = _listActivatorCreator.GetActivator(expectedType)(size); for (var i = 0; i < size; i++) { - var value = Decode(argType, offset, out offset, injectables, network); + var value = Decode(argType, offset, out offset, depth + 1, ref budget, ref payloadBudget, injectables, network); addMethod.Invoke(array, [value]); } @@ -667,12 +842,17 @@ private object DecodeArray(Type expectedType, int size, long offset, out long ou /// Decodes the uint64. /// /// - private ulong DecodeUInt64(Type expectedType, long offset, int size) + private ulong DecodeUInt64(Type expectedType, long offset, int size, ref int payloadBudget) { if (expectedType != typeof(ulong) && expectedType != typeof(ulong?)) { ReflectionUtil.CheckType(expectedType, typeof(ulong)); } + if (size > 8) + { + throw new InvalidDatabaseException("The MaxMind DB file contains a uint64 larger than 8 bytes."); + } + ConsumePayload(size, ref payloadBudget); return _database.ReadULong(offset, size); } @@ -680,12 +860,17 @@ private ulong DecodeUInt64(Type expectedType, long offset, int size) /// Decodes the big integer. /// /// - private BigInteger DecodeBigInteger(Type expectedType, long offset, int size) + private BigInteger DecodeBigInteger(Type expectedType, long offset, int size, ref int payloadBudget) { if (expectedType != typeof(BigInteger) && expectedType != typeof(BigInteger?)) { ReflectionUtil.CheckType(expectedType, typeof(BigInteger)); } + if (size > 16) + { + throw new InvalidDatabaseException("The MaxMind DB file contains a uint128 larger than 16 bytes."); + } + ConsumePayload(size, ref payloadBudget); return _database.ReadBigInteger(offset, size); } diff --git a/MaxMind.Db/DictionaryActivatorCreator.cs b/MaxMind.Db/DictionaryActivatorCreator.cs index f3e8292e..367be213 100644 --- a/MaxMind.Db/DictionaryActivatorCreator.cs +++ b/MaxMind.Db/DictionaryActivatorCreator.cs @@ -12,10 +12,10 @@ namespace MaxMind.Db { internal sealed class DictionaryActivatorCreator { - private readonly ConcurrentDictionary _dictActivators = + private readonly ConcurrentDictionary> _dictActivators = new(); - internal ObjectActivator GetActivator(Type expectedType) + internal Func GetActivator(Type expectedType) => _dictActivators.GetOrAdd(expectedType, DictionaryActivator); #if NET8_0_OR_GREATER @@ -28,7 +28,7 @@ internal ObjectActivator GetActivator(Type expectedType) "IL2070", Justification = "Generated dictionary registrations return before this reflection path. This path serves only the documented fallback for unregistered dictionary types, which is unsupported in trimmed applications.")] #endif - private static ObjectActivator DictionaryActivator(Type expectedType) + private static Func DictionaryActivator(Type expectedType) { var genericArgs = expectedType.GetGenericArguments(); ConstructorInfo? constructor; @@ -45,7 +45,7 @@ private static ObjectActivator DictionaryActivator(Type expectedType) } if (constructor == null) throw new DeserializationException($"Unable to find default constructor for {expectedType}"); - return ReflectionUtil.CreateActivator(constructor); + return ReflectionUtil.CreateCapacityActivator(constructor); } } } diff --git a/MaxMind.Db/ListActivatorCreator.cs b/MaxMind.Db/ListActivatorCreator.cs index 7ae916e6..2332961a 100644 --- a/MaxMind.Db/ListActivatorCreator.cs +++ b/MaxMind.Db/ListActivatorCreator.cs @@ -11,10 +11,10 @@ namespace MaxMind.Db { internal sealed class ListActivatorCreator { - private readonly ConcurrentDictionary _listActivators = + private readonly ConcurrentDictionary> _listActivators = new(); - internal ObjectActivator GetActivator(Type expectedType) + internal Func GetActivator(Type expectedType) => _listActivators.GetOrAdd(expectedType, ListActivator); #if NET8_0_OR_GREATER @@ -27,7 +27,7 @@ internal ObjectActivator GetActivator(Type expectedType) "IL2070", Justification = "Generated collection registrations return before this reflection path. This path serves only the documented fallback for unregistered collection types, which is unsupported in trimmed applications.")] #endif - private static ObjectActivator ListActivator(Type expectedType) + private static Func ListActivator(Type expectedType) { var genericArgs = expectedType.GetGenericArguments(); var argType = genericArgs.Length switch @@ -51,7 +51,7 @@ private static ObjectActivator ListActivator(Type expectedType) } if (constructor == null) throw new DeserializationException($"Unable to find default constructor for {expectedType}"); - return ReflectionUtil.CreateActivator(constructor); + return ReflectionUtil.CreateCapacityActivator(constructor); } } } diff --git a/MaxMind.Db/MemoryMapBuffer.cs b/MaxMind.Db/MemoryMapBuffer.cs index 73b1053a..c880f8b8 100644 --- a/MaxMind.Db/MemoryMapBuffer.cs +++ b/MaxMind.Db/MemoryMapBuffer.cs @@ -260,15 +260,25 @@ private unsafe void AcquireRawPointer() // Uses a targeted slice rather than spanning the full buffer so // that databases larger than 2 GiB still work (Span length is int). private unsafe ReadOnlySpan GetSpan(long offset, int count) + { + CheckBounds(offset, count); + return new ReadOnlySpan((byte*)_ptr + offset, count); + } +#endif + + // Check the database length, since the view accessor can include + // padding beyond the file. GetSpan shares this check on other targets. + // Reject negative offsets before unsigned addition, which could wrap. + // For nonnegative offsets and counts, the unsigned sum cannot overflow + // and keeps offsets beyond long.MaxValue outside the database. + private void CheckBounds(long offset, int count) { if (offset < 0 || (ulong)offset + (ulong)count > (ulong)Length) { - throw new ArgumentOutOfRangeException(nameof(offset), - "Attempt to read beyond the end of the MemoryMappedFile."); + throw new InvalidDatabaseException( + "Attempt to read beyond the end of the database."); } - return new ReadOnlySpan((byte*)_ptr + offset, count); } -#endif internal byte[] Read(long offset, int count) { @@ -282,6 +292,7 @@ internal byte[] Read(long offset, int count) { return Array.Empty(); } + CheckBounds(offset, count); var bytes = new byte[count]; _view.ReadArray(offset, bytes, 0, count); return bytes; @@ -298,12 +309,14 @@ internal byte ReadOne(long offset) } #if NETSTANDARD2_0 + CheckBounds(offset, 1); return _view.ReadByte(offset); #else + // This single-byte check rejects negative offsets and needs no addition. if ((ulong)offset >= (ulong)Length) { - throw new ArgumentOutOfRangeException(nameof(offset), - "Attempt to read beyond the end of the MemoryMappedFile."); + throw new InvalidDatabaseException( + "Attempt to read beyond the end of the database."); } unsafe { @@ -324,11 +337,7 @@ internal string ReadString(long offset, int count) { return string.Empty; } - if (offset < 0 || (ulong)offset + (ulong)count > (ulong)Length) - { - throw new ArgumentOutOfRangeException(nameof(offset), - "Attempt to read beyond the end of the MemoryMappedFile."); - } + CheckBounds(offset, count); var bytes = new byte[count]; _view.ReadArray(offset, bytes, 0, count); return Encoding.UTF8.GetString(bytes); @@ -348,6 +357,7 @@ internal int ReadInt(long offset) } #if NETSTANDARD2_0 + CheckBounds(offset, 4); return _view.ReadByte(offset) << 24 | _view.ReadByte(offset + 1) << 16 | _view.ReadByte(offset + 2) << 8 | @@ -372,6 +382,11 @@ internal int ReadVarInt(long offset, int count) } #if NETSTANDARD2_0 + // Zero reads nothing. Four delegates to ReadInt, which checks bounds. + if (count == 1 || count == 2 || count == 3) + { + CheckBounds(offset, count); + } return count switch { 0 => 0, @@ -411,6 +426,7 @@ internal int ReadVarInt(long offset, int count) internal int HashBytes(long offset, int count) { #if NETSTANDARD2_0 + CheckBounds(offset, count); var code = 17; for (var i = 0; i < count; i++) { @@ -432,6 +448,8 @@ internal int HashBytes(long offset, int count) internal bool EqualsBytes(long offset, MemoryMapBuffer other, long otherOffset, int count) { #if NETSTANDARD2_0 + CheckBounds(offset, count); + other.CheckBounds(otherOffset, count); for (var i = 0; i < count; i++) { if (_view.ReadByte(offset + i) != other._view.ReadByte(otherOffset + i)) @@ -450,6 +468,7 @@ internal bool EqualsBytes(long offset, MemoryMapBuffer other, long otherOffset, internal bool EqualsBytes(long offset, byte[] other, int otherOffset, int count) { #if NETSTANDARD2_0 + CheckBounds(offset, count); for (var i = 0; i < count; i++) { if (_view.ReadByte(offset + i) != other[otherOffset + i]) @@ -512,6 +531,7 @@ internal long ReadLong(long offset, int size) } #if NETSTANDARD2_0 + CheckBounds(offset, size); long val = 0; for (var i = 0; i < size; i++) { @@ -540,6 +560,7 @@ internal ulong ReadULong(long offset, int size) } #if NETSTANDARD2_0 + CheckBounds(offset, size); ulong val = 0; for (var i = 0; i < size; i++) { diff --git a/MaxMind.Db/ReflectionUtil.cs b/MaxMind.Db/ReflectionUtil.cs index 6d12fc12..b929d9da 100644 --- a/MaxMind.Db/ReflectionUtil.cs +++ b/MaxMind.Db/ReflectionUtil.cs @@ -47,6 +47,29 @@ internal static ObjectActivator CreateActivator(ConstructorInfo constructor) return (ObjectActivator)lambda.Compile(); } + // Collection factories take a capacity directly, without a boxed + // integer and a temporary argument array on each decode. + internal static Func CreateCapacityActivator(ConstructorInfo constructor) + { + if (constructor == null) + { + throw new ArgumentNullException(nameof(constructor)); + } + + var capacity = Expression.Parameter(typeof(int), "capacity"); + NewExpression create; + if (constructor.GetParameters().Length == 0) + { + create = Expression.New(constructor); + } + else + { + create = Expression.New(constructor, capacity); + } + // The Compile limitation described in CreateActivator also applies here. + return Expression.Lambda>(create, capacity).Compile(); + } + /// /// Creates a compiled activator that uses MemberInit expressions /// to set properties on an object created via a parameterless constructor. @@ -93,8 +116,15 @@ internal static void CheckType(Type expected, Type from) } if (!expected.IsAssignableFrom(from)) { - throw new DeserializationException($"Could not convert '{from}' to '{expected}'."); + ThrowCannotConvert(expected, from); } } + + // Keep error-message construction out of callers that inline CheckType. + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowCannotConvert(Type expected, Type from) + { + throw new DeserializationException($"Could not convert '{from}' to '{expected}'."); + } } } \ No newline at end of file diff --git a/releasenotes.md b/releasenotes.md index 46906d89..4ef151c3 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -2,6 +2,8 @@ ## 5.2.0 (YYYY-MM-DD) +- Fixed decoding of some valid four-byte data pointers. +- Fixed truncated or out-of-range results when decoding oversized integers. - Added NativeAOT and trimming support for C# model deserialization. The NuGet package now includes a source generator for constructor-based and property-based models, including models with annotated properties inherited @@ -11,6 +13,8 @@ collection types used directly with `Find` and `FindAll`. - Reused immutable source-generated activation metadata across readers, reducing the time and allocation cost of the first model lookup on a new reader. +- Improved lookup performance and reduced allocations when creating collections + through reflection. - Enabled trim, AOT, and single-file compatibility analysis. - Added the `MMDBSG001` through `MMDBSG016` diagnostics, which report model shapes the generator cannot support so that they are caught at build time @@ -23,6 +27,20 @@ - Fixed a `[MapKey(..., true)]` member of a non-nullable value type throwing during property-based activation instead of keeping its default. This affected the reflection path before this release and is now consistent across both. +- Added decoding limits to prevent crafted databases from consuming excessive + time and memory. Each lookup and metadata read allows at most 65,536 decoded + values, 512 nesting levels (including pointer follows), and 2 MiB of combined + string, bytes, uint32, uint64, and uint128 payload. Exceeding a limit or + encountering a pointer cycle throws `InvalidDatabaseException`. These limits + reject some previously accepted databases, including those with a string or + bytes value larger than 2 MiB. Available stack space can impose a lower + nesting limit. +- Pointers that point directly to other pointers now throw + `InvalidDatabaseException`. +- Truncated or out-of-bounds data reads now throw `InvalidDatabaseException` + instead of `ArgumentOutOfRangeException`. This also prevents incorrect decoded + values on `netstandard2.0`. Update error handlers to catch + `InvalidDatabaseException`. ## 5.1.0 (2026-05-22)