perf(validators): replace tr46 with a generated UTS-46 table

package.json

@@ -28,13 +28,11 @@
     "bip32-path": "^0.4.2",
     "partisia-blockchain-applications-crypto": "^1.0.34",
     "partisia-blockchain-applications-rpc": "^1.0.13",
-    "partisia-blockchain-applications-sdk": "^0.1.4",
-    "tr46": "^4.1.1"
+    "partisia-blockchain-applications-sdk": "^0.1.4"
   },
   "devDependencies": {
     "@types/jest": "^29.5.14",
     "@types/node": "^20.19.9",
-    "@types/tr46": "^3.0.3",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
     "dotenv": "^16.6.1",

scripts/generate-idna-table.js

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+/**
+ * Regenerates src/validators/idna/table.ts from tr46.
+ *
+ * The full UTS-46 mapping table is ~225 KB because it stores an explicit
+ * mapping target for every code point. Most of those targets are exactly what
+ * you get from NFKC + context-free case folding, which every JS runtime already
+ * implements natively. So instead of shipping the table we ship only what
+ * cannot be derived:
+ *
+ *   - `DISALLOWED`: run-length encoded ranges of code points UTS-46 rejects
+ *     under useSTD3ASCIIRules. This is the bulk of what the native operations
+ *     get wrong -- they happily pass through control characters, unassigned
+ *     code points and symbols that IDNA forbids.
+ *   - `EXCEPTIONS`: the handful of code points whose UTS-46 mapping differs
+ *     from NFKC + case folding.
+ *
+ * Run with tr46 installed:  node scripts/generate-idna-table.js
+ */
+const fs = require('fs')
+const path = require('path')
+const { toUnicode } = require('tr46')
+
+function caseFold(s) {
+  // Per code point, so JS's contextual final-sigma rule (which UTS-46 does not
+  // apply) never triggers: 'ΣΣ'.toLowerCase() is 'σς', but UTS-46 wants 'σσ'.
+  let out = ''
+  for (const ch of s) out += ch.toLowerCase()
+  return out
+}
+
+function derive(cp) {
+  const mapped = caseFold(String.fromCodePoint(cp).normalize('NFKC')).normalize('NFC')
+  for (const ch of mapped) {
+    const c = ch.codePointAt(0)
+    if (c < 128 && !/[a-z0-9-]/.test(ch)) return null
+  }
+  return mapped
+}
+
+const run = s => {
+  const r = toUnicode(s, { useSTD3ASCIIRules: true })
+  return r.error ? null : r.domain
+}
+
+// Code-point status must be probed *in context*, not standalone. UTS-46 also
+// enforces a label-level rule -- "a label must not begin with a combining
+// mark" -- so testing a bare combining mark reports an error that belongs to
+// the label, not to the code point. Deriving from standalone probes therefore
+// wrongly marks every combining mark disallowed and rejects names like "Á".
+//
+// '0' is used as the neutral padding because no precomposed character exists
+// for digit-plus-mark, so NFC cannot merge the padding with the probe.
+const PAD = '0'
+
+const disallowed = []
+const leadingMarks = []
+const exceptions = []
+
+for (let cp = 0; cp <= 0x10ffff; cp++) {
+  if (cp >= 0xd800 && cp <= 0xdfff) continue // lone surrogates
+  const ch = String.fromCodePoint(cp)
+
+  const inContext = run(PAD + ch + PAD)
+  if (inContext === null) {
+    disallowed.push(cp)
+    continue
+  }
+
+  // Allowed in context but rejected at the start of a label => combining mark.
+  if (run(ch + PAD) === null) leadingMarks.push(cp)
+
+  const expected = inContext.slice(PAD.length, inContext.length - PAD.length)
+  if (expected !== derive(cp)) exceptions.push([cp, expected])
+}
+
+// Run-length encode a sorted code point set as [start, end] pairs.
+function toRanges (points) {
+  const ranges = []
+  let start = null
+  let prev = null
+  for (const cp of points) {
+    if (start === null) { start = cp; prev = cp } else if (cp === prev + 1) { prev = cp } else { ranges.push([start, prev]); start = cp; prev = cp }
+  }
+  if (start !== null) ranges.push([start, prev])
+  return ranges
+}
+
+// Delta-encode range starts and lengths in base 36 to keep the emitted source
+// compact; decoding happens once at module load.
+function encode (ranges) {
+  const parts = []
+  let cursor = 0
+  for (const [s, e] of ranges) {
+    parts.push((s - cursor).toString(36) + '+' + (e - s).toString(36))
+    cursor = e
+  }
+  return parts.join(',')
+}
+
+const ranges = toRanges(disallowed)
+const markRanges = toRanges(leadingMarks)
+const encodedRanges = encode(ranges)
+const encodedMarks = encode(markRanges)
+
+const banner = `// GENERATED FILE -- do not edit by hand.
+// Regenerate with: node scripts/generate-idna-table.js
+//
+// Derived from tr46 (UTS-46, useSTD3ASCIIRules). See the generator for why only
+// the disallowed ranges and mapping exceptions are stored rather than the full
+// ~225 KB mapping table.
+`
+
+const out = `${banner}
+/** Run-length encoded ranges of code points UTS-46 disallows. */
+export const DISALLOWED_RANGES = '${encodedRanges}'
+
+/**
+ * Run-length encoded ranges of combining marks. UTS-46 allows these inside a
+ * label but rejects any label that begins with one.
+ */
+export const LEADING_MARK_RANGES = '${encodedMarks}'
+
+/** Code points whose UTS-46 mapping differs from NFKC + case folding. */
+export const MAPPING_EXCEPTIONS: ReadonlyArray<readonly [number, string]> = ${JSON.stringify(exceptions)}
+`
+
+const target = path.join(__dirname, '..', 'src', 'validators', 'idna', 'table.ts')
+fs.mkdirSync(path.dirname(target), { recursive: true })
+fs.writeFileSync(target, out)
+
+console.log(`disallowed code points : ${disallowed.length}`)
+console.log(`disallowed ranges      : ${ranges.length}`)
+console.log(`leading-mark ranges    : ${markRanges.length}`)
+console.log(`mapping exceptions     : ${exceptions.length}`)
+console.log(`emitted                : ${target} (${out.length} bytes)`)

scripts/verify-idna.js

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+/**
+ * Exhaustively checks src/validators/idna against tr46 over every code point
+ * and a generated multi-label corpus. Kept out of the jest suite because the
+ * full sweep takes minutes; run it after regenerating the table.
+ *
+ * tr46 is no longer a dependency, so install it just for the run:
+ *
+ *   yarn add --dev tr46 @types/tr46 && node scripts/verify-idna.js
+ */
+require('ts-node').register({ compilerOptions: { module: 'CommonJS', esModuleInterop: true, target: 'ES2022' } })
+const { toUnicode: mine } = require('../src/validators/idna')
+const { toUnicode: ref } = require('tr46')
+
+const refRun = s => { const r = ref(s, { useSTD3ASCIIRules: true }); return r.error ? null : r.domain }
+const myRun = s => { const r = mine(s); return r.error ? null : r.domain }
+
+let tested = 0
+let mismatches = 0
+const samples = []
+
+function check (s) {
+  tested++
+  const a = refRun(s)
+  const b = myRun(s)
+  if (a !== b) {
+    mismatches++
+    if (samples.length < 20) samples.push(`${JSON.stringify(s)} tr46=${JSON.stringify(a)} mine=${JSON.stringify(b)}`)
+  }
+}
+
+console.log('sweeping every code point...')
+for (let cp = 0; cp <= 0x10ffff; cp++) {
+  if (cp >= 0xd800 && cp <= 0xdfff) continue
+  check(String.fromCodePoint(cp))
+}
+console.log(`  ${tested} single code points, ${mismatches} mismatches`)
+
+console.log('sweeping multi-code-point labels...')
+const before = tested
+// Combining marks, Hangul jamo and compatibility forms are where per-code-point
+// derivation is most likely to diverge from whole-string normalisation.
+const interesting = [
+  0x41, 0x61, 0x5a, 0x2d, 0x30, 0x300, 0x301, 0x308, 0x327, 0x1100, 0x1161, 0x11a8,
+  0x212b, 0x1e9b, 0x3a3, 0x3c2, 0x3c3, 0xdf, 0x130, 0x131, 0xff21, 0xff41, 0x2260,
+  0xfb00, 0xfb01, 0x24b6, 0x2460, 0x1f600, 0x1f1fa, 0x1f1f8, 0x5b57, 0x200d, 0x200c,
+  0x2e, 0x5f, 0x20, 0x2013, 0x3002, 0xff0e, 0xff61
+]
+for (const a of interesting) {
+  for (const b of interesting) {
+    check(String.fromCodePoint(a) + String.fromCodePoint(b))
+    for (const c of [0x61, 0x301, 0x2e, 0x1100]) {
+      check(String.fromCodePoint(a) + String.fromCodePoint(b) + String.fromCodePoint(c))
+    }
+  }
+}
+console.log(`  ${tested - before} multi-code-point strings, ${mismatches} cumulative mismatches`)
+
+console.log('checking real registry names and fixtures...')
+const real = [
+  'hölkj.mpc', 'hermès.mpc', 'nestlé.mpc', 'beyoncé.mpc', 'damgård.mpc', 'kénôse.mpc',
+  'pokémon.mpc', '👨‍💻.mpc', '🐳🐳🐳.mpc', '💎💎💎.mpc', '💲💲💲.mpc', '💵💵💵.mpc',
+  '🦄🦄🦄.mpc', 'виталик.mpc', 'ivanbjerredamgård.mpc', 'recently🔹registered.mpc',
+  'аауцуауцауца.mpc', 'fatmamıçokseviyorumbenege.mpc',
+  'name.mpc', 'NaME.mpc', 'the.name.mpc', '🌎.mpc', 'not_valid', 'not..valid', '.', '..',
+  'xn--ls8h', 'xn--80ak6aa92e', 'xn--fiq228c', 'café', 'CAFÉ', 'münchen', 'MÜNCHEN'
+]
+real.forEach(check)
+
+console.log('fuzzing random strings...')
+const fuzzBefore = tested
+// Mulberry32, seeded, so a failure is reproducible.
+let seed = 0x9e3779b9
+const rand = () => {
+  seed |= 0; seed = (seed + 0x6d2b79f5) | 0
+  let t = Math.imul(seed ^ (seed >>> 15), 1 | seed)
+  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
+  return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+}
+for (let i = 0; i < 2000000; i++) {
+  const len = 1 + Math.floor(rand() * 8)
+  let s = ''
+  for (let j = 0; j < len; j++) {
+    const roll = rand()
+    let cp
+    if (roll < 0.35) cp = interesting[Math.floor(rand() * interesting.length)]
+    else if (roll < 0.6) cp = 0x20 + Math.floor(rand() * 0x60)
+    else cp = Math.floor(rand() * 0x11000)
+    if (cp >= 0xd800 && cp <= 0xdfff) cp = 0x61
+    s += String.fromCodePoint(cp)
+  }
+  check(s)
+}
+console.log(`  ${tested - fuzzBefore} random strings, ${mismatches} cumulative mismatches`)
+
+console.log(`\ntotal: ${tested} inputs, ${mismatches} mismatches`)
+samples.forEach(s => console.log('  ' + s))
+process.exit(mismatches === 0 ? 0 : 1)

src/validators/domain-validator.ts

@@ -1,4 +1,4 @@
-import { toUnicode } from 'tr46'
+import { toUnicode } from './idna'
 import { IValidatorInterface, IValidatorOptions } from '../interface'
 import { BaseValidator } from './base-validator'
 
@@ -48,9 +48,7 @@ export class DomainValidator extends BaseValidator implements IValidatorInterfac
     if (reverse) name = name.split('.').reverse().join('.')
     if (name.includes('..')) name = name.replace(/\.\./g, '.')
 
-    // For some reason the toUnicode returns an object instead of a string
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const { domain, error } = toUnicode(name, { useSTD3ASCIIRules: true }) as any
+    const { domain, error } = toUnicode(name)
 
     return error ? '' : domain
   }