From 263cd9e93984ba20b3d6c31c755c9ef962e40733 Mon Sep 17 00:00:00 2001 From: zxdev <24780673+zxdev@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:24:30 -0700 Subject: [PATCH] feat(mmdb): stdlib-only MaxMind DB reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read companion to the mmdb-write command. mmdb-write depends on the third-party maxmind/mmdbwriter and is isolated in its own module for that reason; the reader is implemented from scratch on the standard library and lives in the root package, so downstream consumers can read cidr-produced databases (ASN, country, or any custom schema) through a first-party API with ZERO third-party transitive dependencies — the root module's go.mod stays empty. - OpenMMDB / OpenMMDBBytes / Lookup / Metadata / Close. - Full MaxMind DB v2 format: metadata parse, 24/28/32-bit search-tree records, IPv4-in-IPv6 (::/96) lookups, and the complete data decoder (string, uint16/32/64, int32, double, float, bytes, bool, map, array, uint128, and data-section pointers). - Tests over mmdb-write-produced fixtures: ASN across all three record sizes, nested-map Country records, deduped records resolved via data-section pointers, IPv4/IPv6/not-found, and open-error paths. Fixtures are small and committed (the stdlib-only root can't synthesize MMDBs in-test the way the writer module does); .gitignore keeps large corpora ignored while tracking test-*.mmdb. --- .gitignore | 6 +- mmdb.go | 346 +++++++++++++++++++++++++++++++++++++ mmdb_test.go | 106 ++++++++++++ testdata/test-asn.mmdb | Bin 0 -> 1769 bytes testdata/test-asn28.mmdb | Bin 0 -> 2011 bytes testdata/test-asn32.mmdb | Bin 0 -> 2253 bytes testdata/test-country.mmdb | Bin 0 -> 2374 bytes 7 files changed, 457 insertions(+), 1 deletion(-) create mode 100644 mmdb.go create mode 100644 mmdb_test.go create mode 100644 testdata/test-asn.mmdb create mode 100644 testdata/test-asn28.mmdb create mode 100644 testdata/test-asn32.mmdb create mode 100644 testdata/test-country.mmdb diff --git a/.gitignore b/.gitignore index f0b086d..82ce28e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,11 @@ _bin/ _dev/ sandbox/ install/ -testdata/ +# ignore large downloaded corpora under testdata/, but track the small committed +# MMDB reader fixtures (built by mmdb-write; the stdlib-only root cannot synthesize +# them in-test the way the writer module does). +testdata/* +!testdata/test-*.mmdb dat/ # build outputs and deploy target for build/cidr diff --git a/mmdb.go b/mmdb.go new file mode 100644 index 0000000..1e55775 --- /dev/null +++ b/mmdb.go @@ -0,0 +1,346 @@ +package cidr + +// Stdlib-only MaxMind DB (.mmdb) reader — the read companion to the mmdb-write +// command (which uses the third-party maxmind/mmdbwriter, isolated in its own +// module). Keeping the reader here, dependency-free, lets downstream consumers +// read cidr-produced databases (ASN, country, or any custom schema) through a +// first-party API with zero third-party transitive dependencies. +// +// Format: MaxMind DB File Format Specification v2 +// (https://maxmind.github.io/MaxMind-DB/). A file is a binary search tree, a +// 16-byte separator, a data section, and — after the "\xab\xcd\xefMaxMind.com" +// marker — the metadata (itself encoded in the data format). + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "math/big" + "net" + "os" +) + +var metaMarker = []byte("\xab\xcd\xefMaxMind.com") + +// MMDB is an opened MaxMind DB held in memory, safe for concurrent lookups. +type MMDB struct { + data []byte + nodeCount uint + recordSize uint + ipVersion int + nodeSize uint // bytes per node = recordSize*2/8 + treeSize uint // bytes + dataStart int // treeSize + 16; pointer base for the data section + metadata map[string]any +} + +// OpenMMDB reads and parses the MaxMind DB at path. +func OpenMMDB(path string) (*MMDB, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return OpenMMDBBytes(data) +} + +// OpenMMDBBytes parses an in-memory MaxMind DB. The slice is retained (not +// copied) and must not be mutated while the MMDB is in use. +func OpenMMDBBytes(data []byte) (*MMDB, error) { + idx := bytes.LastIndex(data, metaMarker) + if idx < 0 { + return nil, errors.New("cidr: mmdb: metadata marker not found (not a MaxMind DB)") + } + meta, _, err := decodeMMDB(data[idx+len(metaMarker):], 0, 0) + if err != nil { + return nil, fmt.Errorf("cidr: mmdb metadata: %w", err) + } + mm, ok := meta.(map[string]any) + if !ok { + return nil, errors.New("cidr: mmdb: metadata is not a map") + } + recordSize := uintOf(mm["record_size"]) + if recordSize != 24 && recordSize != 28 && recordSize != 32 { + return nil, fmt.Errorf("cidr: mmdb: unsupported record_size %d", recordSize) + } + nodeCount := uintOf(mm["node_count"]) + nodeSize := recordSize * 2 / 8 + treeSize := nodeCount * nodeSize + if int(treeSize)+16 > len(data) { + return nil, errors.New("cidr: mmdb: truncated (search tree exceeds file)") + } + return &MMDB{ + data: data, + nodeCount: nodeCount, + recordSize: recordSize, + ipVersion: int(uintOf(mm["ip_version"])), + nodeSize: nodeSize, + treeSize: treeSize, + dataStart: int(treeSize) + 16, + metadata: mm, + }, nil +} + +// Metadata returns the decoded metadata map (node_count, record_size, +// ip_version, database_type, description, build_epoch, …). +func (m *MMDB) Metadata() map[string]any { return m.metadata } + +// Close releases the underlying buffer. Present for API symmetry / future mmap. +func (m *MMDB) Close() error { m.data = nil; return nil } + +// Lookup returns the record for ip as a decoded map (the shape depends on the +// database's schema: ASN, country, or any custom fields the producer packed). +// ok is false when ip has no record. +func (m *MMDB) Lookup(ip net.IP) (record map[string]any, ok bool, err error) { + v, found, err := m.lookupValue(ip) + if err != nil || !found { + return nil, found, err + } + mp, _ := v.(map[string]any) + return mp, true, nil +} + +// lookupValue walks the tree for ip and decodes the pointed-to data value. +func (m *MMDB) lookupValue(ip net.IP) (any, bool, error) { + addr := m.normalize(ip) + if addr == nil { + return nil, false, fmt.Errorf("cidr: mmdb: %v cannot be looked up in an IPv%d database", ip, m.ipVersion) + } + node := uint(0) + for i := 0; i < len(addr)*8; i++ { + if node >= m.nodeCount { + break + } + bit := (addr[i>>3] >> (7 - uint(i&7))) & 1 + rec, err := m.record(node, bit == 1) + if err != nil { + return nil, false, err + } + node = rec + } + switch { + case node == m.nodeCount: + return nil, false, nil // the empty record: no data + case node < m.nodeCount: + return nil, false, nil // ran out of address bits inside the tree + } + off := int(m.treeSize) + int(node-m.nodeCount) // == dataStart + (node - nodeCount - 16) + v, _, err := decodeMMDB(m.data, off, m.dataStart) + if err != nil { + return nil, false, err + } + return v, true, nil +} + +// normalize renders ip as the byte width the database indexes on. An IPv4 +// address in an IPv6 database is placed under ::/96 (12 zero bytes + v4), the +// location mmdbwriter/GeoLite2 use for the IPv4 subtree. +func (m *MMDB) normalize(ip net.IP) []byte { + if v4 := ip.To4(); v4 != nil { + if m.ipVersion == 4 { + return v4 + } + b := make([]byte, 16) + copy(b[12:], v4) + return b + } + if m.ipVersion == 4 { + return nil // an IPv6 address cannot be found in a v4-only database + } + return ip.To16() +} + +// record reads the left (right==false) or right record value of a tree node. +func (m *MMDB) record(node uint, right bool) (uint, error) { + base := node * m.nodeSize + if int(base)+int(m.nodeSize) > int(m.treeSize) { + return 0, errors.New("cidr: mmdb: node index out of range") + } + b := m.data[base:] + switch m.recordSize { + case 24: + if !right { + return uint(b[0])<<16 | uint(b[1])<<8 | uint(b[2]), nil + } + return uint(b[3])<<16 | uint(b[4])<<8 | uint(b[5]), nil + case 28: + if !right { + return uint(b[3]>>4)<<24 | uint(b[0])<<16 | uint(b[1])<<8 | uint(b[2]), nil + } + return uint(b[3]&0x0f)<<24 | uint(b[4])<<16 | uint(b[5])<<8 | uint(b[6]), nil + default: // 32 + if !right { + return uint(binary.BigEndian.Uint32(b[0:4])), nil + } + return uint(binary.BigEndian.Uint32(b[4:8])), nil + } +} + +// decodeMMDB decodes one value from buf at off. ptrBase is the offset pointers +// are relative to (the data-section start for tree data; 0 for the self-contained +// metadata section). Returns the value and the offset just past it. +func decodeMMDB(buf []byte, off, ptrBase int) (any, int, error) { + if off < 0 || off >= len(buf) { + return nil, 0, errors.New("cidr: mmdb: offset out of range") + } + ctrl := buf[off] + off++ + typ := ctrl >> 5 + if typ == 0 { // extended type + if off >= len(buf) { + return nil, 0, errors.New("cidr: mmdb: truncated extended type") + } + typ = buf[off] + 7 + off++ + } + + if typ == 1 { // pointer + ss := (ctrl >> 3) & 0x03 + var ptr int + switch ss { + case 0: + if off+1 > len(buf) { + return nil, 0, errTrunc + } + ptr = int(ctrl&0x07)<<8 | int(buf[off]) + off++ + case 1: + if off+2 > len(buf) { + return nil, 0, errTrunc + } + ptr = (int(ctrl&0x07)<<16 | int(buf[off])<<8 | int(buf[off+1])) + 2048 + off += 2 + case 2: + if off+3 > len(buf) { + return nil, 0, errTrunc + } + ptr = (int(ctrl&0x07)<<24 | int(buf[off])<<16 | int(buf[off+1])<<8 | int(buf[off+2])) + 526336 + off += 3 + default: // 3 + if off+4 > len(buf) { + return nil, 0, errTrunc + } + ptr = int(binary.BigEndian.Uint32(buf[off : off+4])) + off += 4 + } + v, _, err := decodeMMDB(buf, ptrBase+ptr, ptrBase) + return v, off, err + } + + size := int(ctrl & 0x1f) + switch { + case size < 29: + case size == 29: + if off+1 > len(buf) { + return nil, 0, errTrunc + } + size = 29 + int(buf[off]) + off++ + case size == 30: + if off+2 > len(buf) { + return nil, 0, errTrunc + } + size = 285 + int(buf[off])<<8 + int(buf[off+1]) + off += 2 + default: // 31 + if off+3 > len(buf) { + return nil, 0, errTrunc + } + size = 65821 + int(buf[off])<<16 + int(buf[off+1])<<8 + int(buf[off+2]) + off += 3 + } + + switch typ { + case 2: // UTF-8 string + if off+size > len(buf) { + return nil, 0, errTrunc + } + return string(buf[off : off+size]), off + size, nil + case 3: // double (IEEE-754 64-bit) + if off+8 > len(buf) { + return nil, 0, errTrunc + } + return math.Float64frombits(binary.BigEndian.Uint64(buf[off : off+8])), off + 8, nil + case 4: // bytes + if off+size > len(buf) { + return nil, 0, errTrunc + } + return append([]byte(nil), buf[off:off+size]...), off + size, nil + case 5, 6, 9: // uint16 / uint32 / uint64 + if off+size > len(buf) { + return nil, 0, errTrunc + } + return beUint(buf[off : off+size]), off + size, nil + case 7: // map + m := make(map[string]any, size) + o := off + for i := 0; i < size; i++ { + k, no, err := decodeMMDB(buf, o, ptrBase) + if err != nil { + return nil, 0, err + } + ks, ok := k.(string) + if !ok { + return nil, 0, errors.New("cidr: mmdb: non-string map key") + } + v, no2, err := decodeMMDB(buf, no, ptrBase) + if err != nil { + return nil, 0, err + } + m[ks] = v + o = no2 + } + return m, o, nil + case 8: // int32 (two's complement, up to 4 bytes) + if off+size > len(buf) { + return nil, 0, errTrunc + } + return int32(uint32(beUint(buf[off : off+size]))), off + size, nil + case 10: // uint128 + if off+size > len(buf) { + return nil, 0, errTrunc + } + return new(big.Int).SetBytes(buf[off : off+size]), off + size, nil + case 11: // array + arr := make([]any, size) + o := off + for i := 0; i < size; i++ { + v, no, err := decodeMMDB(buf, o, ptrBase) + if err != nil { + return nil, 0, err + } + arr[i] = v + o = no + } + return arr, o, nil + case 14: // boolean (size is 0 or 1) + return size != 0, off, nil + case 15: // float (IEEE-754 32-bit) + if off+4 > len(buf) { + return nil, 0, errTrunc + } + return math.Float32frombits(binary.BigEndian.Uint32(buf[off : off+4])), off + 4, nil + } + return nil, 0, fmt.Errorf("cidr: mmdb: unknown data type %d", typ) +} + +var errTrunc = errors.New("cidr: mmdb: truncated data section") + +// beUint reads up to 8 big-endian bytes as a uint64. +func beUint(b []byte) uint64 { + var v uint64 + for _, x := range b { + v = v<<8 | uint64(x) + } + return v +} + +// uintOf coerces a decoded metadata number (always a uint64 from the decoder) to +// a uint; anything else yields 0. +func uintOf(v any) uint { + if u, ok := v.(uint64); ok { + return uint(u) + } + return 0 +} diff --git a/mmdb_test.go b/mmdb_test.go new file mode 100644 index 0000000..769ff9a --- /dev/null +++ b/mmdb_test.go @@ -0,0 +1,106 @@ +package cidr + +import ( + "net" + "testing" +) + +// testdata/test-asn.mmdb is a GeoLite2-ASN-schema database built by the +// mmdb-write command from a fixed spec (see mmdb_test.go's companion in the +// commit message): 1.0.0.0/24 → AS13335, 8.8.8.0/24 → AS15169, +// 9.9.9.0/24 → AS19281, 2606:4700:4700::/48 → AS13335. Dual-stack (ip_version 6), +// 24-bit records. +func TestMMDBLookupASN(t *testing.T) { + cases := []struct { + ip string + asn uint64 + org string + want bool + }{ + {"1.0.0.1", 13335, "CLOUDFLARENET", true}, // IPv4-in-IPv6 (::/96) path + {"8.8.8.8", 15169, "GOOGLE", true}, + {"9.9.9.9", 19281, "QUAD9", true}, + {"2606:4700:4700::1111", 13335, "CLOUDFLARENET", true}, // native IPv6 + {"203.0.113.1", 0, "", false}, // not in any listed range + {"2001:db8::1", 0, "", false}, + } + // Every record size, because 24/28/32-bit node records use distinct encodings + // (28-bit splits the middle byte's nibbles) — the fixtures are the same spec. + for _, f := range []string{"test-asn.mmdb", "test-asn28.mmdb", "test-asn32.mmdb"} { + t.Run(f, func(t *testing.T) { + db, err := OpenMMDB("testdata/" + f) + if err != nil { + t.Fatalf("OpenMMDB: %v", err) + } + defer db.Close() + if got := db.Metadata()["database_type"]; got != "GeoLite2-ASN" { + t.Errorf("database_type = %v, want GeoLite2-ASN", got) + } + for _, c := range cases { + rec, ok, err := db.Lookup(net.ParseIP(c.ip)) + if err != nil { + t.Fatalf("Lookup(%s): %v", c.ip, err) + } + if ok != c.want { + t.Errorf("Lookup(%s) found=%v, want %v", c.ip, ok, c.want) + continue + } + if !ok { + continue + } + if got := rec["autonomous_system_number"]; got != c.asn { + t.Errorf("Lookup(%s) ASN = %v (%T), want %d", c.ip, got, got, c.asn) + } + if got := rec["autonomous_system_organization"]; got != c.org { + t.Errorf("Lookup(%s) org = %v, want %q", c.ip, got, c.org) + } + } + }) + } +} + +// testdata/test-country.mmdb is a GeoLite2-Country-schema database with nested +// maps (continent / country / registered_country). Two ranges share the "US" +// record, so the writer deduplicates it and the second lookup resolves through a +// data-section POINTER — exercising the pointer-follow path. +func TestMMDBLookupCountryNested(t *testing.T) { + db, err := OpenMMDB("testdata/test-country.mmdb") + if err != nil { + t.Fatalf("OpenMMDB: %v", err) + } + defer db.Close() + + cases := []struct{ ip, iso, continent string }{ + {"1.0.0.1", "US", "NA"}, + {"8.8.8.8", "US", "NA"}, // deduped → pointer-resolved record + {"81.2.69.1", "GB", "EU"}, + {"2001:67c:2e8::1", "DE", "EU"}, + } + for _, c := range cases { + rec, ok, err := db.Lookup(net.ParseIP(c.ip)) + if err != nil || !ok { + t.Fatalf("Lookup(%s) ok=%v err=%v", c.ip, ok, err) + } + country, _ := rec["country"].(map[string]any) + if got, _ := country["iso_code"].(string); got != c.iso { + t.Errorf("Lookup(%s) country.iso_code = %q, want %q", c.ip, got, c.iso) + } + continent, _ := rec["continent"].(map[string]any) + if got, _ := continent["code"].(string); got != c.continent { + t.Errorf("Lookup(%s) continent.code = %q, want %q", c.ip, got, c.continent) + } + // names is a nested map[lang]string — confirms recursive map decode. + if names, _ := country["names"].(map[string]any); len(names) == 0 { + t.Errorf("Lookup(%s) country.names empty — nested map not decoded", c.ip) + } + } +} + +func TestMMDBOpenErrors(t *testing.T) { + if _, err := OpenMMDBBytes([]byte("not an mmdb file at all")); err == nil { + t.Error("expected an error for a file without the metadata marker") + } + if _, err := OpenMMDB("testdata/does-not-exist.mmdb"); err == nil { + t.Error("expected an error opening a missing file") + } +} diff --git a/testdata/test-asn.mmdb b/testdata/test-asn.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..163bc4ab226a1d7868f0693e6df4e37bc09e1f1e GIT binary patch literal 1769 zcmZY7_j?m{9Ki8UY6UkSDmW?;5fNorK}AFeG>vVNhC++Qg2yGfrdON0cy}qJipq53 zV7LIO;D8EFRKP_QP!w0efdi-mP!QxV@bV6k2h9(!=Y5~=J$K*dn-lSf91$tN6GT>t z6w)VRC+v(T;mLRko{Fd8>39Zq!LHa1&&2L{7M_jgU=Qqx=i+&IK3;%Dcp>&eFJ6Sb z@nYBc7_TbX={pW!zA9KS&K&M%3tM7}24^YiT@JLqrl+x+}H;(OdF@&oZB{)9i{F5HcK zumgWV_r1T0{6_zdd-L;s#D4rkl-!5FWc+9vWx;4#Z;UHi%8b<)7Bob+wuj@YqefL*Q=Cjn3k5VYq&r&i0N?nEU|h3fmY&M) zcF$~P349e*OZuDR(NO}mS z!o%S0^AVA0q(@;o%z&9N3myY++~Xooke+P7ckaYyQ=9`&!(8yj%@dhVT97F&6j?-i z2A+lIGOs)@@&f6_^F^O~c;1{Z!OQRpyb4R823~`uund;N3RnrNU^SGAtYz7C@Vd)~ zB5#n^!v=U0Ho_*@3~#{}mk*eFtH?Hr+hGUngk7*3-iAG}7xqCdyaVsTd+p8KBxFaruU`DSER4uaHjVSVjy3QVtckU-HrBycu`brb*YI^r!20+Gy08H@#75W{ zn_yFHhHv6q*c@A6OKgR$u?@DxcGw;}U`OnPo$+n#f?e?)?1tU32lm8w@jY~7FYJwp z=)olPVjoP#6!f7V1DJ|w*ca1^^SK=yMjcaf%uEX`1Dcs2XEp8H?5ZO%s z4!_4Aa0_n5ZMYqG2#+)G6z*c)jeBq}?!*1~BObtmcnAyeFdo69cnmY2(Y}Y1BBxkC zjc4#I{)9iHeeM_LUxnwG&!fG6LF703?|2dI{Y%W3@rv*&^EJGVH}DVq6K`S>-a`BN zw?+P<-@(7p-oMLSjQ521nIGUk_%A-h|M343pM1}vmi>wb$uUdU^oSlaB{R>ol!&Co zB0{w^G^Z8BP$dWZRU$#_T3-tDr>xPu87^Zk?LEN*us#dnU z+!u_gIblhO>Y@)=4@;gA?X<&2g#PN^{_LIKvS#t&^W``14Db6q?>j@3LDWZ-2?r3h z5M?1>00+WB@IrVIyck{r2g6I@W$~3LQ$&NK=Jz((bou0dZZiS8~80e0Z$Trhwwf8 z0saVofOns1IUt6ds*wKk;}%AA%~Cb4S#DlL zSGd18HyGu*Cdav?DqWJ19q_W*=T~^BST_`LVW_iZKi7)Hr4?hj-ateQnBm-b*H$kx zg;z80F89werlv;c*qkZ0H>&unrR+~(G84T(&s}ot8jtEPFz6p_s9^bH&*`tpyR?)-0pOWkb6p zetM#6>JnNjD`6x1*y!*yZg5poqZVf+mMmLbV>v}-nrd;y8?d;^g2e&Tk`>)#V%+;v z5$akZZkgHQ+GKXw`Nj7%)b}+l?%B}6>V3mmW4@@j*=b-$^F=*kG;HvwYFc7ZMbv@P z*eRD~xNNXLQ05hBj6q@f0o%|MGRsK?&Vs(>tirdB*^}L5vzu&plRMpH(oL=s3oU1j zlbp3qtJsuI=djZvmOHy+i)f#n@ERXAej{`J1%6CC{ojsF7V6q3dKWtoiLspG{jCYxGptqUo?+BinKAbspYA;7M&g?=rxx}k6xro*G zKN6E!;}mBN-j??BO6uv{kCH~r+kX77B6)t1swrb5di>8Bm8vEiHPS3ytYt}Yxk@*r zx!f>CX-%2`zbp3Y+}*}drL9&g5>M!nSXO3L<*|+;MYiNh+2oQ{li*VdxIRg|W#h6Z zq+WxN!p(@GCM+@cY4Js|@}QbHx7ff!(RS39quk8MVzog{e4eQn=ty74;1Tg9Nv2xM Hn}+=Z44mzj literal 0 HcmV?d00001