diff --git a/package.json b/package.json index cae9848a..a62da7b9 100644 --- a/package.json +++ b/package.json @@ -28,13 +28,11 @@ "bip32-path": "^0.4.2", "partisia-blockchain-applications-crypto": "^1.0.34", "partisia-blockchain-applications-rpc": "^1.0.13", - "partisia-blockchain-applications-sdk": "^0.1.4", - "tr46": "^4.1.1" + "partisia-blockchain-applications-sdk": "^0.1.4" }, "devDependencies": { "@types/jest": "^29.5.14", "@types/node": "^20.19.9", - "@types/tr46": "^3.0.3", "@typescript-eslint/eslint-plugin": "^6.21.0", "@typescript-eslint/parser": "^6.21.0", "dotenv": "^16.6.1", diff --git a/scripts/generate-idna-table.js b/scripts/generate-idna-table.js new file mode 100644 index 00000000..9399acb8 --- /dev/null +++ b/scripts/generate-idna-table.js @@ -0,0 +1,136 @@ +#!/usr/bin/env node +/** + * Regenerates src/validators/idna/table.ts from tr46. + * + * The full UTS-46 mapping table is ~225 KB because it stores an explicit + * mapping target for every code point. Most of those targets are exactly what + * you get from NFKC + context-free case folding, which every JS runtime already + * implements natively. So instead of shipping the table we ship only what + * cannot be derived: + * + * - `DISALLOWED`: run-length encoded ranges of code points UTS-46 rejects + * under useSTD3ASCIIRules. This is the bulk of what the native operations + * get wrong -- they happily pass through control characters, unassigned + * code points and symbols that IDNA forbids. + * - `EXCEPTIONS`: the handful of code points whose UTS-46 mapping differs + * from NFKC + case folding. + * + * Run with tr46 installed: node scripts/generate-idna-table.js + */ +const fs = require('fs') +const path = require('path') +const { toUnicode } = require('tr46') + +function caseFold(s) { + // Per code point, so JS's contextual final-sigma rule (which UTS-46 does not + // apply) never triggers: 'ΣΣ'.toLowerCase() is 'σς', but UTS-46 wants 'σσ'. + let out = '' + for (const ch of s) out += ch.toLowerCase() + return out +} + +function derive(cp) { + const mapped = caseFold(String.fromCodePoint(cp).normalize('NFKC')).normalize('NFC') + for (const ch of mapped) { + const c = ch.codePointAt(0) + if (c < 128 && !/[a-z0-9-]/.test(ch)) return null + } + return mapped +} + +const run = s => { + const r = toUnicode(s, { useSTD3ASCIIRules: true }) + return r.error ? null : r.domain +} + +// Code-point status must be probed *in context*, not standalone. UTS-46 also +// enforces a label-level rule -- "a label must not begin with a combining +// mark" -- so testing a bare combining mark reports an error that belongs to +// the label, not to the code point. Deriving from standalone probes therefore +// wrongly marks every combining mark disallowed and rejects names like "Á". +// +// '0' is used as the neutral padding because no precomposed character exists +// for digit-plus-mark, so NFC cannot merge the padding with the probe. +const PAD = '0' + +const disallowed = [] +const leadingMarks = [] +const exceptions = [] + +for (let cp = 0; cp <= 0x10ffff; cp++) { + if (cp >= 0xd800 && cp <= 0xdfff) continue // lone surrogates + const ch = String.fromCodePoint(cp) + + const inContext = run(PAD + ch + PAD) + if (inContext === null) { + disallowed.push(cp) + continue + } + + // Allowed in context but rejected at the start of a label => combining mark. + if (run(ch + PAD) === null) leadingMarks.push(cp) + + const expected = inContext.slice(PAD.length, inContext.length - PAD.length) + if (expected !== derive(cp)) exceptions.push([cp, expected]) +} + +// Run-length encode a sorted code point set as [start, end] pairs. +function toRanges (points) { + const ranges = [] + let start = null + let prev = null + for (const cp of points) { + if (start === null) { start = cp; prev = cp } else if (cp === prev + 1) { prev = cp } else { ranges.push([start, prev]); start = cp; prev = cp } + } + if (start !== null) ranges.push([start, prev]) + return ranges +} + +// Delta-encode range starts and lengths in base 36 to keep the emitted source +// compact; decoding happens once at module load. +function encode (ranges) { + const parts = [] + let cursor = 0 + for (const [s, e] of ranges) { + parts.push((s - cursor).toString(36) + '+' + (e - s).toString(36)) + cursor = e + } + return parts.join(',') +} + +const ranges = toRanges(disallowed) +const markRanges = toRanges(leadingMarks) +const encodedRanges = encode(ranges) +const encodedMarks = encode(markRanges) + +const banner = `// GENERATED FILE -- do not edit by hand. +// Regenerate with: node scripts/generate-idna-table.js +// +// Derived from tr46 (UTS-46, useSTD3ASCIIRules). See the generator for why only +// the disallowed ranges and mapping exceptions are stored rather than the full +// ~225 KB mapping table. +` + +const out = `${banner} +/** Run-length encoded ranges of code points UTS-46 disallows. */ +export const DISALLOWED_RANGES = '${encodedRanges}' + +/** + * Run-length encoded ranges of combining marks. UTS-46 allows these inside a + * label but rejects any label that begins with one. + */ +export const LEADING_MARK_RANGES = '${encodedMarks}' + +/** Code points whose UTS-46 mapping differs from NFKC + case folding. */ +export const MAPPING_EXCEPTIONS: ReadonlyArray = ${JSON.stringify(exceptions)} +` + +const target = path.join(__dirname, '..', 'src', 'validators', 'idna', 'table.ts') +fs.mkdirSync(path.dirname(target), { recursive: true }) +fs.writeFileSync(target, out) + +console.log(`disallowed code points : ${disallowed.length}`) +console.log(`disallowed ranges : ${ranges.length}`) +console.log(`leading-mark ranges : ${markRanges.length}`) +console.log(`mapping exceptions : ${exceptions.length}`) +console.log(`emitted : ${target} (${out.length} bytes)`) diff --git a/scripts/verify-idna.js b/scripts/verify-idna.js new file mode 100644 index 00000000..8475fed3 --- /dev/null +++ b/scripts/verify-idna.js @@ -0,0 +1,98 @@ +#!/usr/bin/env node +/** + * Exhaustively checks src/validators/idna against tr46 over every code point + * and a generated multi-label corpus. Kept out of the jest suite because the + * full sweep takes minutes; run it after regenerating the table. + * + * tr46 is no longer a dependency, so install it just for the run: + * + * yarn add --dev tr46 @types/tr46 && node scripts/verify-idna.js + */ +require('ts-node').register({ compilerOptions: { module: 'CommonJS', esModuleInterop: true, target: 'ES2022' } }) +const { toUnicode: mine } = require('../src/validators/idna') +const { toUnicode: ref } = require('tr46') + +const refRun = s => { const r = ref(s, { useSTD3ASCIIRules: true }); return r.error ? null : r.domain } +const myRun = s => { const r = mine(s); return r.error ? null : r.domain } + +let tested = 0 +let mismatches = 0 +const samples = [] + +function check (s) { + tested++ + const a = refRun(s) + const b = myRun(s) + if (a !== b) { + mismatches++ + if (samples.length < 20) samples.push(`${JSON.stringify(s)} tr46=${JSON.stringify(a)} mine=${JSON.stringify(b)}`) + } +} + +console.log('sweeping every code point...') +for (let cp = 0; cp <= 0x10ffff; cp++) { + if (cp >= 0xd800 && cp <= 0xdfff) continue + check(String.fromCodePoint(cp)) +} +console.log(` ${tested} single code points, ${mismatches} mismatches`) + +console.log('sweeping multi-code-point labels...') +const before = tested +// Combining marks, Hangul jamo and compatibility forms are where per-code-point +// derivation is most likely to diverge from whole-string normalisation. +const interesting = [ + 0x41, 0x61, 0x5a, 0x2d, 0x30, 0x300, 0x301, 0x308, 0x327, 0x1100, 0x1161, 0x11a8, + 0x212b, 0x1e9b, 0x3a3, 0x3c2, 0x3c3, 0xdf, 0x130, 0x131, 0xff21, 0xff41, 0x2260, + 0xfb00, 0xfb01, 0x24b6, 0x2460, 0x1f600, 0x1f1fa, 0x1f1f8, 0x5b57, 0x200d, 0x200c, + 0x2e, 0x5f, 0x20, 0x2013, 0x3002, 0xff0e, 0xff61 +] +for (const a of interesting) { + for (const b of interesting) { + check(String.fromCodePoint(a) + String.fromCodePoint(b)) + for (const c of [0x61, 0x301, 0x2e, 0x1100]) { + check(String.fromCodePoint(a) + String.fromCodePoint(b) + String.fromCodePoint(c)) + } + } +} +console.log(` ${tested - before} multi-code-point strings, ${mismatches} cumulative mismatches`) + +console.log('checking real registry names and fixtures...') +const real = [ + 'hölkj.mpc', 'hermès.mpc', 'nestlé.mpc', 'beyoncé.mpc', 'damgård.mpc', 'kénôse.mpc', + 'pokémon.mpc', '👨‍💻.mpc', '🐳🐳🐳.mpc', '💎💎💎.mpc', '💲💲💲.mpc', '💵💵💵.mpc', + '🦄🦄🦄.mpc', 'виталик.mpc', 'ivanbjerredamgård.mpc', 'recently🔹registered.mpc', + 'аауцуауцауца.mpc', 'fatmamıçokseviyorumbenege.mpc', + 'name.mpc', 'NaME.mpc', 'the.name.mpc', '🌎.mpc', 'not_valid', 'not..valid', '.', '..', + 'xn--ls8h', 'xn--80ak6aa92e', 'xn--fiq228c', 'café', 'CAFÉ', 'münchen', 'MÜNCHEN' +] +real.forEach(check) + +console.log('fuzzing random strings...') +const fuzzBefore = tested +// Mulberry32, seeded, so a failure is reproducible. +let seed = 0x9e3779b9 +const rand = () => { + seed |= 0; seed = (seed + 0x6d2b79f5) | 0 + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 +} +for (let i = 0; i < 2000000; i++) { + const len = 1 + Math.floor(rand() * 8) + let s = '' + for (let j = 0; j < len; j++) { + const roll = rand() + let cp + if (roll < 0.35) cp = interesting[Math.floor(rand() * interesting.length)] + else if (roll < 0.6) cp = 0x20 + Math.floor(rand() * 0x60) + else cp = Math.floor(rand() * 0x11000) + if (cp >= 0xd800 && cp <= 0xdfff) cp = 0x61 + s += String.fromCodePoint(cp) + } + check(s) +} +console.log(` ${tested - fuzzBefore} random strings, ${mismatches} cumulative mismatches`) + +console.log(`\ntotal: ${tested} inputs, ${mismatches} mismatches`) +samples.forEach(s => console.log(' ' + s)) +process.exit(mismatches === 0 ? 0 : 1) diff --git a/src/validators/domain-validator.ts b/src/validators/domain-validator.ts index e0f74d9d..f25a53eb 100644 --- a/src/validators/domain-validator.ts +++ b/src/validators/domain-validator.ts @@ -1,4 +1,4 @@ -import { toUnicode } from 'tr46' +import { toUnicode } from './idna' import { IValidatorInterface, IValidatorOptions } from '../interface' import { BaseValidator } from './base-validator' @@ -48,9 +48,7 @@ export class DomainValidator extends BaseValidator implements IValidatorInterfac if (reverse) name = name.split('.').reverse().join('.') if (name.includes('..')) name = name.replace(/\.\./g, '.') - // For some reason the toUnicode returns an object instead of a string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const { domain, error } = toUnicode(name, { useSTD3ASCIIRules: true }) as any + const { domain, error } = toUnicode(name) return error ? '' : domain } diff --git a/src/validators/idna/index.ts b/src/validators/idna/index.ts new file mode 100644 index 00000000..cde5283f --- /dev/null +++ b/src/validators/idna/index.ts @@ -0,0 +1,221 @@ +import { DISALLOWED_RANGES, LEADING_MARK_RANGES, MAPPING_EXCEPTIONS } from './table' + +/** + * A minimal UTS-46 `toUnicode` with `useSTD3ASCIIRules` enabled. + * + * The reference implementation (tr46) ships an explicit mapping target for + * every code point, which costs ~225 KB. Almost all of those targets are + * reproducible from NFKC plus context-free case folding, both of which the + * runtime already provides, so this module derives the common case natively and + * consults a generated table only for what cannot be derived: the set of + * disallowed code points, and 526 mapping exceptions. + * + * The generated table and this implementation are verified against tr46 over + * every assigned code point and a large multi-label corpus; see + * test/domain/idna.test.ts. + */ + +type Ranges = Array<[number, number]> + +let disallowedRanges: Ranges | undefined +let leadingMarkRanges: Ranges | undefined +let mappingExceptions: Map | undefined + +function decodeRanges(encoded: string): Ranges { + const parsed: Ranges = [] + let cursor = 0 + + for (const entry of encoded.split(',')) { + const plus = entry.indexOf('+') + const start = cursor + parseInt(entry.slice(0, plus), 36) + const end = start + parseInt(entry.slice(plus + 1), 36) + parsed.push([start, end]) + cursor = end + } + + return parsed +} + +function inRanges(cp: number, table: Ranges): boolean { + let low = 0 + let high = table.length - 1 + + while (low <= high) { + const mid = (low + high) >> 1 + const range = table[mid] + if (!range) break + if (cp < range[0]) high = mid - 1 + else if (cp > range[1]) low = mid + 1 + else return true + } + + return false +} + +function exceptions(): Map { + if (mappingExceptions) return mappingExceptions + + mappingExceptions = new Map(MAPPING_EXCEPTIONS.map(([cp, to]) => [cp, to])) + return mappingExceptions +} + +function isDisallowed(cp: number): boolean { + disallowedRanges ??= decodeRanges(DISALLOWED_RANGES) + return inRanges(cp, disallowedRanges) +} + +/** UTS-46 permits combining marks inside a label but not at its start. */ +function isLeadingMark(cp: number): boolean { + leadingMarkRanges ??= decodeRanges(LEADING_MARK_RANGES) + return inRanges(cp, leadingMarkRanges) +} + +/** + * Case folding applied one code point at a time. `String#toLowerCase` on a + * whole string applies Greek final-sigma context ('ΣΣ' becomes 'σς'), which + * UTS-46 does not do -- it maps Σ to σ unconditionally. + */ +function caseFold(value: string): string { + let out = '' + for (const ch of value) out += ch.toLowerCase() + return out +} + +/** RFC 3492 punycode decoding, for `xn--` prefixed labels. */ +const BASE = 36 +const T_MIN = 1 +const T_MAX = 26 +const SKEW = 38 +const DAMP = 700 +const INITIAL_BIAS = 72 +const INITIAL_N = 128 + +function adaptBias(delta: number, numPoints: number, firstTime: boolean): number { + let d = firstTime ? Math.floor(delta / DAMP) : delta >> 1 + d += Math.floor(d / numPoints) + + let k = 0 + while (d > ((BASE - T_MIN) * T_MAX) >> 1) { + d = Math.floor(d / (BASE - T_MIN)) + k += BASE + } + + return k + Math.floor(((BASE - T_MIN + 1) * d) / (d + SKEW)) +} + +function digitValue(codePoint: number): number { + if (codePoint >= 0x30 && codePoint <= 0x39) return codePoint - 0x30 + 26 + if (codePoint >= 0x41 && codePoint <= 0x5a) return codePoint - 0x41 + if (codePoint >= 0x61 && codePoint <= 0x7a) return codePoint - 0x61 + return BASE +} + +function punycodeDecode(input: string): string | null { + const output: number[] = [] + const delimiter = input.lastIndexOf('-') + + let start = 0 + if (delimiter > 0) { + for (let i = 0; i < delimiter; i++) { + const cp = input.charCodeAt(i) + if (cp > 0x7f) return null + output.push(cp) + } + start = delimiter + 1 + } + + let n = INITIAL_N + let bias = INITIAL_BIAS + let i = 0 + + for (let index = start; index < input.length;) { + const oldi = i + let w = 1 + + for (let k = BASE; ; k += BASE) { + if (index >= input.length) return null + const digit = digitValue(input.charCodeAt(index++)) + if (digit >= BASE) return null + if (digit > Math.floor((0x7fffffff - i) / w)) return null + + i += digit * w + const t = k <= bias ? T_MIN : k >= bias + T_MAX ? T_MAX : k - bias + if (digit < t) break + if (w > Math.floor(0x7fffffff / (BASE - t))) return null + w *= BASE - t + } + + const outLength = output.length + 1 + bias = adaptBias(i - oldi, outLength, oldi === 0) + + if (Math.floor(i / outLength) > 0x7fffffff - n) return null + n += Math.floor(i / outLength) + i %= outLength + + if (n < 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) return null + output.splice(i++, 0, n) + } + + return String.fromCodePoint(...output) +} + +function mapLabel(label: string): string | null { + const exceptionMap = exceptions() + let mapped = '' + + for (const ch of label) { + const cp = ch.codePointAt(0) + if (cp === undefined) return null + if (isDisallowed(cp)) return null + + const exception = exceptionMap.get(cp) + mapped += exception !== undefined ? exception : caseFold(ch.normalize('NFKC')) + } + + const composed = mapped.normalize('NFC') + + // The leading-mark rule applies to the *mapped* label, not the input. Code + // points that UTS-46 ignores (variation selectors, soft hyphen) map to the + // empty string, so a mark that followed one becomes label-leading only after + // mapping. + const leading = composed.codePointAt(0) + if (leading !== undefined && isLeadingMark(leading)) return null + + return composed +} + +export interface ToUnicodeResult { + domain: string + error: boolean +} + +/** + * Mirrors `tr46.toUnicode(name, { useSTD3ASCIIRules: true })` for the inputs + * this SDK accepts: on success returns the mapped Unicode domain, otherwise + * flags an error. Labels are processed independently, matching UTS-46. + */ +export function toUnicode(name: string): ToUnicodeResult { + // UTS-46 treats the ideographic, fullwidth and halfwidth stops as label + // separators and folds them to U+002E *before* splitting. Splitting on ASCII + // '.' alone would leave them inside a label, so a following combining mark + // would never be checked as the start of the next label. + const labels = name.replace(/[。.。]/g, '.').split('.') + const output: string[] = [] + + for (const label of labels) { + let current = label + + if (/^xn--/i.test(current)) { + const decoded = punycodeDecode(current.slice(4)) + if (decoded === null) return { domain: '', error: true } + current = decoded + } + + const mapped = mapLabel(current) + if (mapped === null) return { domain: '', error: true } + + output.push(mapped) + } + + return { domain: output.join('.'), error: false } +} diff --git a/src/validators/idna/table.ts b/src/validators/idna/table.ts new file mode 100644 index 00000000..9926046f --- /dev/null +++ b/src/validators/idna/table.ts @@ -0,0 +1,18 @@ +// GENERATED FILE -- do not edit by hand. +// Regenerate with: node scripts/generate-idna-table.js +// +// Derived from tr46 (UTS-46, useSTD3ASCIIRules). See the generator for why only +// the disallowed ranges and mapping exceptions are stored rather than the full +// ~225 KB mapping table. + +/** Run-length encoded ranges of code points UTS-46 disallows. */ +export const DISALLOWED_RANGES = '0+18,3+0,b+6,r+5,r+11,8+0,7+0,5+0,4+0,f4+5,4b+2,4+0,2+5,6+0,2+0,l+0,7y+0,34+0,13+1,1f+1,4+0,1k+7,s+3,7+g,n+0,5d+0,1d+1,1o+1,2u+d,1o+1,1e+1,g+0,t+1,2+0,c+4,w+8,23+0,4i+0,9+1,3+1,n+0,8+0,2+2,5+1,a+1,3+1,5+7,2+3,3+0,6+1,q+1,4+0,7+3,3+1,n+0,8+0,3+0,3+0,3+1,2+0,6+3,3+1,4+2,2+6,5+0,2+6,i+9,4+0,a+0,4+0,n+0,8+0,3+0,6+1,b+0,4+0,4+1,2+e,5+1,d+6,8+0,4+0,9+1,3+1,n+0,8+0,3+0,6+1,a+1,3+1,4+6,4+3,3+0,6+1,j+9,3+0,7+2,4+0,5+2,3+0,2+0,3+2,3+2,4+2,d+3,6+2,4+0,5+1,2+5,2+d,m+4,e+0,4+0,o+0,h+1,a+0,4+0,5+6,3+0,4+1,2+1,5+1,b+6,n+0,4+0,o+0,b+0,6+1,a+0,4+0,5+6,3+5,3+0,5+1,b+0,4+b,e+0,4+0,1g+0,4+0,7+3,h+1,r+0,4+0,j+2,p+0,a+0,2+1,8+2,2+3,7+0,2+0,9+5,b+1,4+b,1n+3,u+10,3+0,2+0,6+0,p+0,2+0,o+1,6+0,2+0,8+0,b+1,5+v,21+0,11+3,14+0,11+0,g+0,e+10,4h+12,2+4,2+1,40+1,6h+0,5+1,8+0,2+0,5+1,16+0,5+1,y+0,5+1,8+0,2+0,5+1,g+0,1m+0,5+1,1w+1,x+2,r+5,2f+1,7+1,ht+0,t+2,2i+6,n+8,p+8,l+b,e+0,4+0,3+b,1h+1,15+1,b+5,b+5,7+0,8+0,c+5,2i+6,18+4,1z+9,w+0,d+3,d+3,2+2,17+1,6+a,19+3,r+5,c+2,1r+1,1u+0,u+1,c+5,b+5,f+1,w+1c,26+2,1c+0,39+7,1p+2,g+2,1p+6,18+1,c+7,18+4,ev+1,7+1,13+1,7+1,9+0,2+0,2+0,2+0,w+1,1i+0,8+0,2+2,4+0,8+2,5+1,7+3,e+4,4+0,8+d,4+1,8+0,d+2,2+7,d+0,2+0,9+2,m+0,2+2,2+a,3+1,7+0,2+2,c+0,2+3,e+2,y+e,y+g,4+1,18+0,29+0,9+3,5t+0,e+1,c8+o,c+k,l+1t,14v+2,72+1,x+0,9q+4,1a+0,2+4,2+1,1l+6,3+d,p+8,8+0,8+0,8+0,8+0,8+0,8+0,8+0,8+0,3j+x,r+0,2i+b,5z+16,1s+0,2f+1,3+1,2s+4,18+0,1g+0,17+0,2d+b,h+1v,an+0,5+0,h+0,m9h+2,1k+8,9p+j,55+7,5o+4,3+0,2+0,6+n,1o+2,b+5,1l+7,1z+7,d+5,39+a,v+2,27+0,c+3,y+0,1k+8,f+1,b+1,2w+n,t+9,7+1,7+1,7+8,8+0,8+0,1p+3,3j+1,b+5,8md+b,o+3,1e+3,1kx+4xr,a7+1,2z+11,8+b,6+4,d+0,e+0,6+0,2+0,3+0,3+0,3i+f,3w+5,8d+1,1j+6,2+v,b+1,l+0,2+4,3+6,h+0,3+5,f+9,2+5,2+3,3+3,2+c,2+0,2+2,2+0,2+0,2+0,2+0,3j+1,2+c,3+0,b+6,r+5,r+3,1u+0,v+2,7+1,7+1,7+1,4+2,4+0,4+0,8+g,d+0,r+0,k+0,3+0,g+1,f+x,3g+4,4+3,1a+2,2h+0,e+2,2+1a,1b+3l,u+2,1e+e,t+3,11+8,v+4,18+4,v+0,12+3,f+15,4f+1,b+5,11+3,11+3,15+7,1h+a,d+0,g+0,8+0,3+0,c+0,g+0,8+0,3+1u,8o+8,n+9,9+n,7+0,17+0,a+1w,7+1,2+0,19+0,3+2,2+1,o+0,21+7,a+1b,k+0,3+4,y+2,s+4,2+1r,1l+3,l+1,1f+0,3+4,9+0,4+0,u+1,4+3,b+6,a+6,1t+v,14+3,d+8,1j+2,u+1,s+4,r+6,5+b,8+27,22+1i,1g+c,1g+6,1b+7,b+85,w+0,17+0,4+1,3+22,18+7,17+l,r+11,t+j,o+8,27+3,11+8,1r+0,6+c,q+6,b+5,1i+0,j+7,14+8,2p+0,l+a,j+0,1c+1p,8+0,2+0,5+0,g+0,c+5,1o+4,b+5,5+0,9+1,3+1,n+0,8+0,3+0,6+0,b+1,3+1,4+1,2+5,2+4,8+1,8+2,6+3u,2l+0,6+t,21+7,b+4l,1j+1,13+x,1y+a,b+5,e+i,1n+5,b+1h,s+1,g+3,o+54,1p+2r,2c+b,9+1,2+1,9+0,3+0,v+0,3+1,d+8,b+1x,9+1,1b+1,c+q,21+7,2c+c,22+6,b+6t,a+0,1a+0,f+9,u+2,x+1,n+0,f+20,8+0,3+0,19+2,2+0,3+0,a+7,b+5,7+0,3+0,12+0,3+0,7+6,b+8l,q+6,i+0,16+2,t+2d,2+e,1f+c,po+2t,34+0,6+a,5h+217,2s+c,tt+f,n+33d,g8+6ns,fu+6,w+0,b+3,2a+0,b+5,v+1,7+9,1z+9,b+0,8+0,m+4,k+j3,2k+2s,24+3,1m+6,i+1r,6+a,3+d,4qh+7,yf+15,a+6w6,5+0,8+0,3+0,84+e,2+s,4+1,2+d,5+7,b1+1s3,30+4,e+2,a+6,b+1,9+3mj,1b+1,o+8,39+1n,6v+9,14+1,23+7,35+k,1z+3d,l+b,l+b,2g+8,q+3q,2e+0,20+0,3+1,2+1,3+1,5+0,d+0,2+0,8+0,1u+0,5+1,9+0,8+0,t+0,5+0,6+0,2+2,8+0,9h+1,85+1,jj+e,6+0,g+un,w+5,7+5w,8+0,i+1,8+0,3+0,6+4,1r+w,2+33,1a+2,f+1,b+3,3+8v,w+g,1n+4,2+cv,17+kl,8+0,5+0,3+0,g+0,5i+1,h+14,25+3,b+3,3+ls,1x+23,1q+5d,5+0,s+0,3+0,2+1,2+0,b+0,5+0,2+0,2+5,2+3,2+0,2+0,2+0,4+0,3+0,2+1,2+0,2+0,2+0,2+0,2+0,3+0,2+1,5+0,8+0,5+0,5+0,2+0,b+0,i+4,4+0,6+0,i+1f,3+7h,19+3,2t+b,g+1,g+0,g+0,12+k,6+p,3p+1j,u+c,19+3,a+6,3+d,7+49,rd+3,i+2,e+2,3c+3,2o+5,d+3,2+e,d+3,1l+7,b+5,15+7,v+1,3+25,9h+b,f+1,e+2,a+6,1b+0,8+7,f+3,a+6,a+6,44+0,1k+10,b+sl,wyp+v,37f+5,67+1,4g3+d,5rm+2e6,2x+0,c+0,4r+0,1s+0,2o+0,2n+15t,3t8+4,38h+f9e7,6p+47bj' + +/** + * Run-length encoded ranges of combining marks. UTS-46 allows these inside a + * label but rejects any label that begins with one. + */ +export const LEADING_MARK_RANGES = 'lc+1w,2+8,2+v,7o+6,7c+18,2+0,2+1,2+1,2+0,21+a,1d+k,h+0,2u+6,3+5,3+1,2+3,10+0,v+q,2k+a,1n+8,a+0,p+3,2+8,2+2,2+4,18+2,1p+7,17+n,2+w,1j+2,2+h,2+6,b+1,u+2,1l+0,2+6,3+1,3+2,a+0,b+1,r+0,3+2,1l+0,2+4,5+1,3+2,4+0,v+1,4+0,c+2,1l+0,2+7,2+2,2+2,l+1,n+5,2+2,1l+0,2+6,3+1,3+2,8+2,b+1,v+0,1o+4,4+2,2+3,a+0,15+4,1k+0,2+6,2+2,2+3,8+1,c+1,u+2,1l+0,2+6,2+2,2+3,8+1,c+1,g+0,d+3,1k+1,2+6,2+2,2+3,a+0,b+1,u+2,1z+0,5+5,2+0,2+7,j+1,1q+0,2+7,d+7,2r+0,2+9,c+6,22+1,s+0,2+0,2+0,5+1,1e+j,2+1,6+a,2+z,a+0,2t+j,o+3,5+2,2+2,3+6,4+3,e+b,2+0,b+3,jk+2,qb+3,t+2,u+1,v+1,1v+t,a+0,4o+1,z+0,3b+b,5+b,64+4,1m+9,2+s,3+0,1d+u,1e+4,1c+g,13+8,d+2,v+c,1l+d,1d+j,49+2,2+k,5+0,7+0,3+2,5j+1r,k1+w,2db+2,3y+0,2p+v,ff+5,2y+1,n9x+3,2+9,x+1,29+1,7l+0,4+0,5+0,o+4,5+0,2c+1,1f+h,r+h,e+0,13+7,q+c,19+3,1c+d,11+0,1w+d,d+0,9+1,1a+2,1f+0,2+2,3+1,6+1,2+0,16+4,6+1,6l+7,2+1,fn5+0,le+f,a7+1,gu+0,6b+0,46+4,1af+2,2+1,6+3,15+2,5+0,4m+1,fy+3,as+1,29+2,1z+a,1e+3,3f+2,1i+e,16+0,3+1,b+3,1a+a,8+0,1q+2,11+d,h+1,19+0,d+2,1d+d,9+3,2+1,2l+b,7+0,3+0,4e+b,m+3,1k+1,2+6,3+1,3+2,a+0,b+1,3+6,4+4,5d+h,o+0,2a+j,6k+6,3+8,s+1,2b+g,2z+c,2u+e,75+e,6u+5,2+1,3+3,2+0,2+1,3y+6,3+6,4+0,t+9,15+6,2+3,9+0,a+a,1b+f,ba+7,2+7,2b+l,2+d,3f+5,4+0,2+1,2+6,2+0,1v+4,2+1,2+4,9o+3,a+1,2+0,1d+6,4+4,45a+0,7+e,asb+4,1o+6,t5+0,2+1i,8+3,2a+0,c+1,f58+1,3mq+19,3+m,f3+4,4+5,9+7,3+6,v+3,45+2,1j0+1i,5+1d,9+0,f+0,n+4,2+e,11t+6,2+g,3+6,2+1,2+4,2t+0,4h+6,ag+0,1q+3,e5+3,rl+6,32+6' + +/** Code points whose UTS-46 mapping differs from NFKC + case folding. */ +export const MAPPING_EXCEPTIONS: ReadonlyArray = [[46,"."],[173,""],[837,"ι"],[847,""],[1010,"σ"],[5024,"Ꭰ"],[5025,"Ꭱ"],[5026,"Ꭲ"],[5027,"Ꭳ"],[5028,"Ꭴ"],[5029,"Ꭵ"],[5030,"Ꭶ"],[5031,"Ꭷ"],[5032,"Ꭸ"],[5033,"Ꭹ"],[5034,"Ꭺ"],[5035,"Ꭻ"],[5036,"Ꭼ"],[5037,"Ꭽ"],[5038,"Ꭾ"],[5039,"Ꭿ"],[5040,"Ꮀ"],[5041,"Ꮁ"],[5042,"Ꮂ"],[5043,"Ꮃ"],[5044,"Ꮄ"],[5045,"Ꮅ"],[5046,"Ꮆ"],[5047,"Ꮇ"],[5048,"Ꮈ"],[5049,"Ꮉ"],[5050,"Ꮊ"],[5051,"Ꮋ"],[5052,"Ꮌ"],[5053,"Ꮍ"],[5054,"Ꮎ"],[5055,"Ꮏ"],[5056,"Ꮐ"],[5057,"Ꮑ"],[5058,"Ꮒ"],[5059,"Ꮓ"],[5060,"Ꮔ"],[5061,"Ꮕ"],[5062,"Ꮖ"],[5063,"Ꮗ"],[5064,"Ꮘ"],[5065,"Ꮙ"],[5066,"Ꮚ"],[5067,"Ꮛ"],[5068,"Ꮜ"],[5069,"Ꮝ"],[5070,"Ꮞ"],[5071,"Ꮟ"],[5072,"Ꮠ"],[5073,"Ꮡ"],[5074,"Ꮢ"],[5075,"Ꮣ"],[5076,"Ꮤ"],[5077,"Ꮥ"],[5078,"Ꮦ"],[5079,"Ꮧ"],[5080,"Ꮨ"],[5081,"Ꮩ"],[5082,"Ꮪ"],[5083,"Ꮫ"],[5084,"Ꮬ"],[5085,"Ꮭ"],[5086,"Ꮮ"],[5087,"Ꮯ"],[5088,"Ꮰ"],[5089,"Ꮱ"],[5090,"Ꮲ"],[5091,"Ꮳ"],[5092,"Ꮴ"],[5093,"Ꮵ"],[5094,"Ꮶ"],[5095,"Ꮷ"],[5096,"Ꮸ"],[5097,"Ꮹ"],[5098,"Ꮺ"],[5099,"Ꮻ"],[5100,"Ꮼ"],[5101,"Ꮽ"],[5102,"Ꮾ"],[5103,"Ꮿ"],[5104,"Ᏸ"],[5105,"Ᏹ"],[5106,"Ᏺ"],[5107,"Ᏻ"],[5108,"Ᏼ"],[5109,"Ᏽ"],[5112,"Ᏸ"],[5113,"Ᏹ"],[5114,"Ᏺ"],[5115,"Ᏻ"],[5116,"Ᏼ"],[5117,"Ᏽ"],[6155,""],[6156,""],[6157,""],[6159,""],[7296,"в"],[7297,"д"],[7298,"о"],[7299,"с"],[7300,"т"],[7301,"т"],[7302,"ъ"],[7303,"ѣ"],[7304,"ꙋ"],[7838,"ss"],[8064,"ἀι"],[8065,"ἁι"],[8066,"ἂι"],[8067,"ἃι"],[8068,"ἄι"],[8069,"ἅι"],[8070,"ἆι"],[8071,"ἇι"],[8072,"ἀι"],[8073,"ἁι"],[8074,"ἂι"],[8075,"ἃι"],[8076,"ἄι"],[8077,"ἅι"],[8078,"ἆι"],[8079,"ἇι"],[8080,"ἠι"],[8081,"ἡι"],[8082,"ἢι"],[8083,"ἣι"],[8084,"ἤι"],[8085,"ἥι"],[8086,"ἦι"],[8087,"ἧι"],[8088,"ἠι"],[8089,"ἡι"],[8090,"ἢι"],[8091,"ἣι"],[8092,"ἤι"],[8093,"ἥι"],[8094,"ἦι"],[8095,"ἧι"],[8096,"ὠι"],[8097,"ὡι"],[8098,"ὢι"],[8099,"ὣι"],[8100,"ὤι"],[8101,"ὥι"],[8102,"ὦι"],[8103,"ὧι"],[8104,"ὠι"],[8105,"ὡι"],[8106,"ὢι"],[8107,"ὣι"],[8108,"ὤι"],[8109,"ὥι"],[8110,"ὦι"],[8111,"ὧι"],[8114,"ὰι"],[8115,"αι"],[8116,"άι"],[8119,"ᾶι"],[8124,"αι"],[8130,"ὴι"],[8131,"ηι"],[8132,"ήι"],[8135,"ῆι"],[8140,"ηι"],[8178,"ὼι"],[8179,"ωι"],[8180,"ώι"],[8183,"ῶι"],[8188,"ωι"],[8203,""],[8288,""],[8292,""],[12290,"."],[43888,"Ꭰ"],[43889,"Ꭱ"],[43890,"Ꭲ"],[43891,"Ꭳ"],[43892,"Ꭴ"],[43893,"Ꭵ"],[43894,"Ꭶ"],[43895,"Ꭷ"],[43896,"Ꭸ"],[43897,"Ꭹ"],[43898,"Ꭺ"],[43899,"Ꭻ"],[43900,"Ꭼ"],[43901,"Ꭽ"],[43902,"Ꭾ"],[43903,"Ꭿ"],[43904,"Ꮀ"],[43905,"Ꮁ"],[43906,"Ꮂ"],[43907,"Ꮃ"],[43908,"Ꮄ"],[43909,"Ꮅ"],[43910,"Ꮆ"],[43911,"Ꮇ"],[43912,"Ꮈ"],[43913,"Ꮉ"],[43914,"Ꮊ"],[43915,"Ꮋ"],[43916,"Ꮌ"],[43917,"Ꮍ"],[43918,"Ꮎ"],[43919,"Ꮏ"],[43920,"Ꮐ"],[43921,"Ꮑ"],[43922,"Ꮒ"],[43923,"Ꮓ"],[43924,"Ꮔ"],[43925,"Ꮕ"],[43926,"Ꮖ"],[43927,"Ꮗ"],[43928,"Ꮘ"],[43929,"Ꮙ"],[43930,"Ꮚ"],[43931,"Ꮛ"],[43932,"Ꮜ"],[43933,"Ꮝ"],[43934,"Ꮞ"],[43935,"Ꮟ"],[43936,"Ꮠ"],[43937,"Ꮡ"],[43938,"Ꮢ"],[43939,"Ꮣ"],[43940,"Ꮤ"],[43941,"Ꮥ"],[43942,"Ꮦ"],[43943,"Ꮧ"],[43944,"Ꮨ"],[43945,"Ꮩ"],[43946,"Ꮪ"],[43947,"Ꮫ"],[43948,"Ꮬ"],[43949,"Ꮭ"],[43950,"Ꮮ"],[43951,"Ꮯ"],[43952,"Ꮰ"],[43953,"Ꮱ"],[43954,"Ꮲ"],[43955,"Ꮳ"],[43956,"Ꮴ"],[43957,"Ꮵ"],[43958,"Ꮶ"],[43959,"Ꮷ"],[43960,"Ꮸ"],[43961,"Ꮹ"],[43962,"Ꮺ"],[43963,"Ꮻ"],[43964,"Ꮼ"],[43965,"Ꮽ"],[43966,"Ꮾ"],[43967,"Ꮿ"],[65024,""],[65025,""],[65026,""],[65027,""],[65028,""],[65029,""],[65030,""],[65031,""],[65032,""],[65033,""],[65034,""],[65035,""],[65036,""],[65037,""],[65038,""],[65039,""],[65279,""],[65294,"."],[65377,"."],[113824,""],[113825,""],[113826,""],[113827,""],[120531,"σ"],[120589,"σ"],[120647,"σ"],[120705,"σ"],[120763,"σ"],[917760,""],[917761,""],[917762,""],[917763,""],[917764,""],[917765,""],[917766,""],[917767,""],[917768,""],[917769,""],[917770,""],[917771,""],[917772,""],[917773,""],[917774,""],[917775,""],[917776,""],[917777,""],[917778,""],[917779,""],[917780,""],[917781,""],[917782,""],[917783,""],[917784,""],[917785,""],[917786,""],[917787,""],[917788,""],[917789,""],[917790,""],[917791,""],[917792,""],[917793,""],[917794,""],[917795,""],[917796,""],[917797,""],[917798,""],[917799,""],[917800,""],[917801,""],[917802,""],[917803,""],[917804,""],[917805,""],[917806,""],[917807,""],[917808,""],[917809,""],[917810,""],[917811,""],[917812,""],[917813,""],[917814,""],[917815,""],[917816,""],[917817,""],[917818,""],[917819,""],[917820,""],[917821,""],[917822,""],[917823,""],[917824,""],[917825,""],[917826,""],[917827,""],[917828,""],[917829,""],[917830,""],[917831,""],[917832,""],[917833,""],[917834,""],[917835,""],[917836,""],[917837,""],[917838,""],[917839,""],[917840,""],[917841,""],[917842,""],[917843,""],[917844,""],[917845,""],[917846,""],[917847,""],[917848,""],[917849,""],[917850,""],[917851,""],[917852,""],[917853,""],[917854,""],[917855,""],[917856,""],[917857,""],[917858,""],[917859,""],[917860,""],[917861,""],[917862,""],[917863,""],[917864,""],[917865,""],[917866,""],[917867,""],[917868,""],[917869,""],[917870,""],[917871,""],[917872,""],[917873,""],[917874,""],[917875,""],[917876,""],[917877,""],[917878,""],[917879,""],[917880,""],[917881,""],[917882,""],[917883,""],[917884,""],[917885,""],[917886,""],[917887,""],[917888,""],[917889,""],[917890,""],[917891,""],[917892,""],[917893,""],[917894,""],[917895,""],[917896,""],[917897,""],[917898,""],[917899,""],[917900,""],[917901,""],[917902,""],[917903,""],[917904,""],[917905,""],[917906,""],[917907,""],[917908,""],[917909,""],[917910,""],[917911,""],[917912,""],[917913,""],[917914,""],[917915,""],[917916,""],[917917,""],[917918,""],[917919,""],[917920,""],[917921,""],[917922,""],[917923,""],[917924,""],[917925,""],[917926,""],[917927,""],[917928,""],[917929,""],[917930,""],[917931,""],[917932,""],[917933,""],[917934,""],[917935,""],[917936,""],[917937,""],[917938,""],[917939,""],[917940,""],[917941,""],[917942,""],[917943,""],[917944,""],[917945,""],[917946,""],[917947,""],[917948,""],[917949,""],[917950,""],[917951,""],[917952,""],[917953,""],[917954,""],[917955,""],[917956,""],[917957,""],[917958,""],[917959,""],[917960,""],[917961,""],[917962,""],[917963,""],[917964,""],[917965,""],[917966,""],[917967,""],[917968,""],[917969,""],[917970,""],[917971,""],[917972,""],[917973,""],[917974,""],[917975,""],[917976,""],[917977,""],[917978,""],[917979,""],[917980,""],[917981,""],[917982,""],[917983,""],[917984,""],[917985,""],[917986,""],[917987,""],[917988,""],[917989,""],[917990,""],[917991,""],[917992,""],[917993,""],[917994,""],[917995,""],[917996,""],[917997,""],[917998,""],[917999,""]] diff --git a/test/domain/idna.test.ts b/test/domain/idna.test.ts new file mode 100644 index 00000000..d8cf9a04 Binary files /dev/null and b/test/domain/idna.test.ts differ diff --git a/yarn.lock b/yarn.lock index a490e3ba..88d4e05f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -887,11 +887,6 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.3.tgz#6209321eb2c1712a7e7466422b8cb1fc0d9dd5d8" integrity sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw== -"@types/tr46@^3.0.3": - version "3.0.3" - resolved "https://registry.yarnpkg.com/@types/tr46/-/tr46-3.0.3.tgz#3d0345f5e2534993f493a0dac87aa476caa994b9" - integrity sha512-GHa+gvc6Ci9lVloIZJlO+rufRVMA5PXrruGc5TmmtO3HoCj1N7mG7EOM9wzXsOmT/LFOXWEF+GyjNmj1wUc6eA== - "@types/yargs-parser@*": version "21.0.3" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" @@ -3791,7 +3786,7 @@ proxy-from-env@^2.1.0: resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba" integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== -punycode@^2.1.0, punycode@^2.3.0: +punycode@^2.1.0: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== @@ -4297,13 +4292,6 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -tr46@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-4.1.1.tgz#281a758dcc82aeb4fe38c7dfe4d11a395aac8469" - integrity sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw== - dependencies: - punycode "^2.3.0" - treeify@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/treeify/-/treeify-1.1.0.tgz#4e31c6a463accd0943879f30667c4fdaff411bb8"