diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml new file mode 100644 index 0000000..0f9e9bf --- /dev/null +++ b/.github/workflows/python-test.yml @@ -0,0 +1,31 @@ +name: python-test + +# Minimal PR/push gate for pure-Python regressions (no native engine). +# Currently runs the focused fingerprint canonicalization contract test. + +on: + push: + branches: [main, master] + pull_request: + +permissions: + contents: read + +jobs: + fingerprint-canonical: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Fingerprint canonical regression + run: | + python3 -m py_compile docker/run_json.py tests/test_fingerprint_canonical.py + python3 tests/test_fingerprint_canonical.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43ae0e2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.py[cod] diff --git a/VERSION b/VERSION index 04c5555..f8bc4c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.17 +0.1.18 diff --git a/docker/run_json.py b/docker/run_json.py index 5f7d8eb..ad92d2b 100755 --- a/docker/run_json.py +++ b/docker/run_json.py @@ -41,7 +41,9 @@ "max_drawdown": float, "commission": float, # ABI v2 "entry_bar_index": int, # ABI v2: script-bar index of entry fill - "exit_bar_index": int # ABI v2: script-bar index of exit fill + "exit_bar_index": int, # ABI v2: script-bar index of exit fill + "entry_incarnation": int # run-scoped physical-entry provenance; + # 0 when the strategy lacks the accessor }, ... ], @@ -56,10 +58,11 @@ ... ], "fingerprint": { # decode-able backtest provenance - "token": "", # b64decode -> JSON - "digest": "sha256:", # stable run id over canonical JSON + "token": "", # b64decode -> authoritative UTF-8 bytes + "digest": "sha256:", # hash token bytes; do not assume json.loads round-trip "provenance": { "engine": { version_string, major, minor, patch, commit_sha }, + "feed": { canonicalization, source_values_sha256 }, "codegen": { version, generated_cpp_sha256, transpiled_from_pine }, "strategy": { ...all strategy() params, effective... }, "inputs": { "": { type, default, value }, ... }, @@ -83,6 +86,7 @@ import json import math import re +import struct import sys import time from datetime import datetime, timezone @@ -135,6 +139,23 @@ _INPUT_RE = re.compile( r'get_input_(\w+)\(\s*"((?:[^"\\]|\\.)*)"\s*,\s*((?:[^();]|\([^()]*\))*?)\s*\)') +# Canonical primary-feed identity. Hash the numeric BarC values in source-row +# order, before any validation-only start/end slicing. The domain prefix makes +# the byte contract versioned and prevents cross-domain hash reuse. +SOURCE_FEED_CANONICALIZATION = "pf-ohlcv-barc-le-v1" +_SOURCE_FEED_HASH_PREFIX = b"pineforge:ohlcv:barc-le:v1\0" +_SOURCE_FEED_RECORD = struct.Struct("<5dq") + + +def _new_source_feed_hasher(): + h = hashlib.sha256() + h.update(_SOURCE_FEED_HASH_PREFIX) + return h + + +def _update_source_feed_hash(h, row) -> None: + h.update(_SOURCE_FEED_RECORD.pack(*row)) + def _ctor_body(cpp_text: str) -> str: """Return the GeneratedStrategy constructor body, or '' if not found. @@ -264,7 +285,10 @@ def _codegen_version() -> str: def build_provenance(engine: dict, cpp_path, transpiled: bool, inputs_applied: dict, overrides_applied: dict, - runtime: dict | None) -> dict: + runtime: dict | None, *, source_feed_sha256: str) -> dict: + if not isinstance(source_feed_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", source_feed_sha256): + raise ValueError("source_feed_sha256 must be a lowercase SHA-256 hex digest") cpp_text = "" cpp_sha = None if cpp_path: @@ -276,6 +300,10 @@ def build_provenance(engine: dict, cpp_path, transpiled: bool, cpp_text = "" return { "engine": engine, + "feed": { + "canonicalization": SOURCE_FEED_CANONICALIZATION, + "source_values_sha256": source_feed_sha256, + }, "codegen": { "version": _codegen_version(), "generated_cpp_sha256": cpp_sha, @@ -291,8 +319,225 @@ def build_provenance(engine: dict, cpp_path, transpiled: bool, } +# Product accepted-type domain for exact Python integers: JavaScript +# Number.MAX/MIN_SAFE_INTEGER (= ±(2**53 - 1)). Rejecting larger exact ints +# guarantees unique lossless integer identity across generic ECMAScript +# consumers. This is not a claim that every such magnitude must round as +# binary64 — e.g. float(2**53) is exactly representable and remains legal +# on the float path, while exact int(2**53) is deliberately outside the +# product integer domain. Out-of-domain provenance yields no fingerprint +# under existing callers (exception → fingerprint None / skipped). +# Booleans are handled separately (bool subclasses int). +_JS_MAX_SAFE_INTEGER = 9007199254740991 +_JS_MIN_SAFE_INTEGER = -9007199254740991 + + +def _canonical_json_number(num: float) -> str: + """Serialize a finite IEEE-754 float via ECMAScript NumberToString. + + Matches ECMAScript NumberToString (RFC 8785 / JCS numeric form): + integral values have no trailing ``.0``, ``±0`` is ``0``, and + scientific notation uses ES exponent thresholds (``e`` when the + exponent is < -6 or >= 21). Non-finite values raise ValueError. + Float subclasses are normalized via base ``float.__float__`` first so + hooks such as ``__float__``/``__abs__``/comparisons/``__repr__`` cannot + alter the underlying binary64 value used for math and emission. + """ + # Plain built-in float: ignore subclass __float__/__abs__/__lt__/... + num = float.__float__(num) + if not math.isfinite(num): + raise ValueError( + "non-finite numbers are not permitted in fingerprint JSON") + if num == 0.0: + return "0" + negative = num < 0 + r = repr(abs(num)) + if "e" in r or "E" in r: + mant, exp_s = r.lower().split("e") + exp = int(exp_s) + if "." in mant: + whole, frac = mant.split(".") + digits_raw = whole + frac + n = exp + len(whole) + else: + digits_raw = mant + n = exp + len(digits_raw) + else: + if "." in r: + whole, frac = r.split(".") + digits_raw = whole + frac + n = len(whole) + else: + digits_raw = r + n = len(digits_raw) + # Leading-zero strip adjusts n so value = int(digits)*10^(n-k) holds. + lead = len(digits_raw) - len(digits_raw.lstrip("0")) + digits = digits_raw.lstrip("0") or "0" + if digits != "0": + n -= lead + while len(digits) > 1 and digits[-1] == "0": + digits = digits[:-1] + k = len(digits) + sign = "-" if negative else "" + if 0 < n <= 21: + if k <= n: + return sign + digits + ("0" * (n - k)) + return sign + digits[:n] + "." + digits[n:] + if -6 < n <= 0: + return sign + "0." + ("0" * (-n)) + digits + exp = n - 1 + exp_s = f"+{exp}" if exp >= 0 else str(exp) + if k == 1: + return sign + digits + "e" + exp_s + return sign + digits[0] + "." + digits[1:] + "e" + exp_s + + +def _reject_unpaired_surrogates(s: str, *, what: str) -> None: + """Fail closed on unpaired UTF-16 surrogates (invalid I-JSON / UTF-8).""" + for ch in s: + cp = ord(ch) + if 0xD800 <= cp <= 0xDFFF: + raise ValueError( + f"unpaired UTF-16 surrogate in fingerprint JSON {what}") + + +def _canonical_json_string(s: str) -> str: + """Serialize a string in RFC 8785 / JCS string form. + + JSON control characters, quotes, and backslashes are escaped; valid + Unicode (including non-ASCII, emoji, and U+2028/U+2029) is emitted as + raw code points (not ``ensure_ascii`` ``\\uXXXX`` escapes). Unpaired + surrogates raise ValueError rather than producing invalid I-JSON. + Str subclasses are normalized via base ``str.__str__`` first so + ``__str__``/``__iter__`` hooks cannot redirect iteration or emission. + """ + # Plain built-in str: ignore subclass __str__/__iter__/... + s = str.__str__(s) + _reject_unpaired_surrogates(s, what="string") + return json.dumps(s, ensure_ascii=False) + + +def _utf16_code_unit_key(s: str) -> bytes: + """Object-key sort key matching JCS / ECMAScript UTF-16 code unit order. + + Str subclasses are normalized via base ``str.__str__`` first so + ``__str__``/``__iter__``/``encode`` hooks cannot corrupt key order or + hide unpaired surrogates. + """ + # Plain built-in str: ignore subclass __str__/__iter__/encode hooks. + s = str.__str__(s) + _reject_unpaired_surrogates(s, what="object key") + return s.encode("utf-16-be") + + +def _canonical_fingerprint_json(value) -> str: + """Canonical JSON text for fingerprint token/digest bytes. + + Direct RFC 8785 / JCS-style canonical writer over the accepted Python + value tree. The resulting UTF-8 token bytes are authoritative for + ``digest``; verifiers should hash those bytes rather than assuming plain + ``JSON.stringify`` is itself JCS (it does not sort keys or implement + full JCS). Over values parsed under a JavaScript / IEEE-754 binary64 + number model, this matches a JCS direct encoder for the accepted types: + - objects: each key is normalized once via base ``str.__str__`` to a + plain built-in str; keys sorted by UTF-16 code unit order on those + normalized names; no whitespace. Duplicate normalized names raise + ValueError (fail closed — distinct str-subclass keys can override + ``__eq__``/``__hash__`` so both coexist in a Python dict while + ``str.__str__`` yields the same text; emitting both would produce + duplicate JSON names that JS silently drops). Non-str keys raise + TypeError. The retained original key is used only for value lookup. + - arrays: element order preserved; no whitespace + - numbers (float): ECMAScript NumberToString for every finite IEEE-754 + value after base ``float.__float__`` normalization (subclass hooks + ignored); non-finite numbers raise ValueError + - numbers (int): decimal digits only for exact integers inside the + product safe-integer domain [-(2**53-1), 2**53-1]. This is a strict + accepted-type policy guaranteeing unique lossless integer identity + across generic ECMAScript consumers — not a claim that every larger + individual value is unrepresentable as binary64. Exact integers + outside the domain raise ValueError (e.g. int 2**53); isolated + binary64 floats such as float(2**53) remain legal via the float path. + Int subclasses (e.g. IntEnum) are normalized via base ``int.__index__`` + to a plain built-in int before domain checks and digit emission, so + ``__index__``/``__int__``/comparison/``__str__``/``__repr__``/ + ``__format__`` hooks cannot change the value or bypass rejection. + Booleans are not integers. + - strings: JCS form — control/quote/backslash escapes, raw valid + Unicode; unpaired surrogates raise ValueError. Str subclasses are + normalized via base ``str.__str__`` so ``__str__``/``__iter__``/ + ``encode`` hooks cannot change emission, key order, or bypass + surrogate rejection. + - bools/null: ``true`` / ``false`` / ``null`` + """ + if value is None: + return "null" + if value is True: + return "true" + if value is False: + return "false" + if isinstance(value, str): + return _canonical_json_string(value) + if isinstance(value, int) and not isinstance(value, bool): + # Plain built-in int via base slot: subclass __index__/__int__/ + # comparison hooks must not bypass domain checks or alter digits + # (Python 3.9 IntEnum str was "Enum.NAME"). + value = int.__index__(value) + if value < _JS_MIN_SAFE_INTEGER or value > _JS_MAX_SAFE_INTEGER: + raise ValueError( + "integers outside the JavaScript safe-integer range " + f"[{_JS_MIN_SAFE_INTEGER}, {_JS_MAX_SAFE_INTEGER}] " + "are not permitted in fingerprint JSON") + return int.__repr__(value) + if isinstance(value, float): + return _canonical_json_number(value) + if isinstance(value, list): + return "[" + ",".join( + _canonical_fingerprint_json(v) for v in value) + "]" + if isinstance(value, dict): + # Normalize each original key exactly once to a plain built-in str. + # Distinct str subclasses can override __eq__/__hash__ so two keys + # coexist while str.__str__ yields the same text; emit the + # normalized name, look up values via the retained original key, + # and fail closed on duplicate normalized names. + items = [] # (normalized_key, original_key) + seen_normalized = set() + for k in value.keys(): + if not isinstance(k, str): + raise TypeError( + "fingerprint JSON object keys must be str, " + f"got {type(k).__name__}") + nk = str.__str__(k) + if nk in seen_normalized: + raise ValueError( + "duplicate fingerprint JSON object key after " + f"str normalization: {nk!r}") + seen_normalized.add(nk) + items.append((nk, k)) + parts = [] + for nk, orig in sorted(items, key=lambda ik: _utf16_code_unit_key(ik[0])): + parts.append( + _canonical_json_string(nk) + ":" + + _canonical_fingerprint_json(value[orig])) + return "{" + ",".join(parts) + "}" + raise TypeError( + f"fingerprint JSON cannot encode {type(value).__name__}") + + def build_fingerprint(provenance: dict) -> dict: - canonical = json.dumps(provenance, sort_keys=True, separators=(",", ":")) + """Build ``{token, digest, provenance}`` for a provenance dict. + + ``token`` is base64 of the canonical UTF-8 JSON bytes; ``digest`` is + ``sha256:`` + hex of those same bytes. Verifiers should treat the decoded + token bytes as authoritative and hash them directly. Re-canonicalizing a + decoded value tree requires an RFC 8785/JCS direct encoder with an + IEEE-754 binary64 number model — default Python ``json.loads`` may yield + ints outside the product integer domain for tokens such as ``1e20`` + (``100000000000000000000``). Project tests that rebuild from token text + use ``json.loads(text, parse_int=float)``. Out-of-domain inputs raise + here; existing callers yield no fingerprint (``None`` / skip). + """ + canonical = _canonical_fingerprint_json(provenance) raw = canonical.encode("utf-8") return { "token": base64.b64encode(raw).decode("ascii"), @@ -475,19 +720,23 @@ def check_abi(lib: ctypes.CDLL) -> None: # --- helpers ---------------------------------------------------------- -def load_bars(csv_path: Path) -> tuple[ctypes.Array, int]: +def load_bars(csv_path: Path) -> tuple[ctypes.Array, int, str]: + """Load the source tape once and return bars, count, and canonical hash.""" rows: list[tuple[float, float, float, float, float, int]] = [] + feed_hasher = _new_source_feed_hasher() with csv_path.open(newline="", encoding="utf-8") as f: reader = csv.DictReader(f) for row in reader: - rows.append(( + parsed = ( float(row["open"]), float(row["high"]), float(row["low"]), float(row["close"]), float(row["volume"]), int(row["timestamp"]), - )) + ) + _update_source_feed_hash(feed_hasher, parsed) + rows.append(parsed) n = len(rows) bars = (BarC * n)() for i, (o, h, l, c, v, ts) in enumerate(rows): @@ -497,7 +746,7 @@ def load_bars(csv_path: Path) -> tuple[ctypes.Array, int]: bars[i].close = c bars[i].volume = v bars[i].timestamp = ts - return bars, n + return bars, n, feed_hasher.hexdigest() def load_strategy(so_path: Path) -> ctypes.CDLL: @@ -522,6 +771,10 @@ def load_strategy(so_path: Path) -> ctypes.CDLL: if hasattr(lib, "strategy_get_last_error"): lib.strategy_get_last_error.argtypes = [ctypes.c_void_p] lib.strategy_get_last_error.restype = ctypes.c_char_p + if hasattr(lib, "strategy_closed_trade_entry_incarnation"): + lib.strategy_closed_trade_entry_incarnation.argtypes = [ + ctypes.c_void_p, ctypes.c_int] + lib.strategy_closed_trade_entry_incarnation.restype = ctypes.c_uint64 # syminfo setters — declare argtypes so ctypes does not default the float # args to c_int (which would truncate mintick=0.5 to 0). Guarded with @@ -594,7 +847,8 @@ def build_report_dict(report: ReportC, ohlcv_path: Path, elapsed: float, applied_inputs: dict[str, str], applied_overrides: dict[str, str], - applied_runtime: dict[str, object] | None = None) -> dict: + applied_runtime: dict[str, object] | None = None, + trade_entry_incarnations: list[int] | None = None) -> dict: trades = [] pnls: list[float] = [] for i in range(report.trades_len): @@ -615,6 +869,11 @@ def build_report_dict(report: ReportC, ohlcv_path: Path, "commission": float(t.commission), "entry_bar_index": int(t.entry_bar_index), "exit_bar_index": int(t.exit_bar_index), + "entry_incarnation": ( + int(trade_entry_incarnations[i]) + if trade_entry_incarnations is not None + and i < len(trade_entry_incarnations) else 0 + ), }) n = len(pnls) @@ -823,7 +1082,7 @@ def main() -> int: magnifier_samples = max(2, int(args.magnifier_samples)) magnifier_dist = parse_magnifier_dist(args.magnifier_dist) - bars, n = load_bars(args.ohlcv) + bars, n, source_feed_sha256 = load_bars(args.ohlcv) first_ts, last_ts = bars[0].timestamp, bars[n - 1].timestamp lib = load_strategy(args.so) @@ -916,8 +1175,17 @@ def _run(st, rep): "trade_start_ms": args.trade_start_ms, "chart_tz": args.chart_tz or "", } - out = build_report_dict(report, args.ohlcv, n, first_ts, last_ts, - elapsed, inputs, overrides, applied_runtime) + incarnation_accessor = getattr( + lib, "strategy_closed_trade_entry_incarnation", None) + trade_entry_incarnations = ( + [int(incarnation_accessor(state, i)) + for i in range(report.trades_len)] + if incarnation_accessor is not None else None + ) + out = build_report_dict( + report, args.ohlcv, n, first_ts, last_ts, + elapsed, inputs, overrides, applied_runtime, + trade_entry_incarnations) if timing is not None: out["diagnostics"]["timing"] = timing out["diagnostics"]["throughput"] = _throughput_block( @@ -931,6 +1199,7 @@ def _run(st, rep): inputs, overrides, applied_runtime, + source_feed_sha256=source_feed_sha256, )) except Exception: out["fingerprint"] = None diff --git a/tests/test_fingerprint_canonical.py b/tests/test_fingerprint_canonical.py new file mode 100644 index 0000000..c7b1496 --- /dev/null +++ b/tests/test_fingerprint_canonical.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Focused RFC 8785 / JCS fingerprint regression for docker/run_json.py. + +Loads only the fingerprint-helper block (between markers) so this runs +without a native engine / strategy.so. Exit 0 iff every check passes. +""" +from __future__ import annotations + +import base64 +import ctypes +import hashlib +import importlib.util +import json +import math +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import types +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +RUN_JSON = REPO / "docker" / "run_json.py" + + +class _TaggedStr(str): + """Str subclass: tag-based eq/hash lets duplicate normalized keys coexist.""" + + def __new__(cls, text: str, tag: object): + obj = str.__new__(cls, text) + obj.tag = tag + return obj + + def __eq__(self, other): + if isinstance(other, _TaggedStr): + return (self.tag == other.tag + and str.__str__(self) == str.__str__(other)) + return NotImplemented + + def __hash__(self): + return hash((_TaggedStr, str.__str__(self), self.tag)) + + def __str__(self) -> str: + return "hostile" + + +def _load_helpers(): + """Exec only the marked helper block — no ctypes / native engine load.""" + text = RUN_JSON.read_text(encoding="utf-8") + start = text.index("# >>> fingerprint helpers") + end = text.index("# <<< fingerprint helpers") + len("# <<< fingerprint helpers") + block = text[start:end] + mod = types.ModuleType("pf_fingerprint_helpers") + # Imports the helper block expects from the surrounding module. + mod.__dict__.update({ + "base64": base64, + "hashlib": hashlib, + "json": json, + "math": math, + "re": re, + "struct": struct, + }) + try: + from importlib import metadata as _ilmd + mod._ilmd = _ilmd # type: ignore[attr-defined] + except ImportError: # pragma: no cover + mod._ilmd = None # type: ignore[attr-defined] + exec(compile(block, str(RUN_JSON), "exec"), mod.__dict__) + return mod + + +def _load_runtime(): + """Import the full harness without invoking its CLI main().""" + spec = importlib.util.spec_from_file_location( + "pf_release_run_json", RUN_JSON) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot import {RUN_JSON}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def main() -> int: + m = _load_helpers() + runtime = _load_runtime() + passed = failed = 0 + + def check(name: str, cond: bool) -> None: + nonlocal passed, failed + if cond: + passed += 1 + print(f" OK {name}") + else: + failed += 1 + print(f" FAIL {name}") + + # --- integral floats: no trailing .0 (live MCP bug) ----------------- + for value, expected in ((0.0, "0"), (1.0, "1"), (12345.0, "12345"), + (-0.0, "0"), (-12345.0, "-12345")): + got = m._canonical_fingerprint_json(value) + check(f"integral float {value!r} -> {expected!r}", got == expected) + + live = {"strategy": {"initial_capital": 12345.0, + "commission_value": 0.0, + "default_qty_value": 1.0}} + live_canon = m._canonical_fingerprint_json(live) + check("live nested integral floats have no trailing .0", + live_canon == ('{"strategy":{"commission_value":0,' + '"default_qty_value":1,' + '"initial_capital":12345}}') + and "12345.0" not in live_canon + and "0.0" not in live_canon + and "1.0" not in live_canon) + + # --- token digest hashes exact decoded bytes ----------------------- + fp = m.build_fingerprint(live) + token_bytes = base64.b64decode(fp["token"]) + check("token decodes to canonical UTF-8 bytes", + token_bytes == live_canon.encode("utf-8")) + check("digest is sha256 of exact decoded token bytes", + fp["digest"] + == "sha256:" + hashlib.sha256(token_bytes).hexdigest()) + check("digest prefix", fp["digest"].startswith("sha256:")) + check("deterministic fingerprint", + m.build_fingerprint(live)["token"] == fp["token"] + and m.build_fingerprint(live)["digest"] == fp["digest"]) + + # --- source tape identity is part of authoritative provenance ------- + source_sha = "a" * 64 + provenance = m.build_provenance( + {"version_string": "0.12.2"}, None, True, {}, {}, {}, + source_feed_sha256=source_sha) + check("source feed identity is included", + provenance["feed"] == { + "canonicalization": "pf-ohlcv-barc-le-v1", + "source_values_sha256": source_sha, + }) + bad_source_sha_rejected = False + try: + m.build_provenance( + {}, None, False, {}, {}, {}, source_feed_sha256="not-a-sha") + except ValueError: + bad_source_sha_rejected = True + check("invalid source feed identity fails closed", bad_source_sha_rejected) + + with tempfile.TemporaryDirectory() as td: + tape_a = Path(td) / "a.csv" + tape_b = Path(td) / "b.csv" + header = "open,high,low,close,volume,timestamp\n" + tape_a.write_text( + header + + "1,2,0.5,1.5,10,1000\n" + + "1.5,3,1,2.5,20,2000\n", + encoding="utf-8") + tape_b.write_text( + header + + "1,2,0.5,1.5,10,1000\n" + + "1.5,3,1,2.6,20,2000\n", + encoding="utf-8") + bars_a, count_a, feed_a = runtime.load_bars(tape_a) + _, count_b, feed_b = runtime.load_bars(tape_b) + expected_feed = hashlib.sha256() + expected_feed.update(b"pineforge:ohlcv:barc-le:v1\0") + expected_feed.update(struct.pack( + "<5dq", 1.0, 2.0, 0.5, 1.5, 10.0, 1000)) + expected_feed.update(struct.pack( + "<5dq", 1.5, 3.0, 1.0, 2.5, 20.0, 2000)) + check("CSV loader emits canonical source feed identity", + count_a == count_b == 2 + and bars_a[1].close == 2.5 + and feed_a == expected_feed.hexdigest()) + check("changed source value changes feed identity", feed_a != feed_b) + provenance_a = runtime.build_provenance( + {}, None, False, {}, {}, {}, source_feed_sha256=feed_a) + provenance_b = runtime.build_provenance( + {}, None, False, {}, {}, {}, source_feed_sha256=feed_b) + check("changed source tape changes run fingerprint", + runtime.build_fingerprint(provenance_a)["digest"] + != runtime.build_fingerprint(provenance_b)["digest"]) + + trades = (runtime.TradeC * 2)() + trades[0].is_long = 1 + trades[1].is_long = 0 + report = runtime.ReportC() + report.trades = ctypes.cast( + trades, ctypes.POINTER(runtime.TradeC)) + report.trades_len = 2 + rendered = runtime.build_report_dict( + report, Path("fixture.csv"), 2, 1000, 2000, 0.0, {}, {}, + trade_entry_incarnations=[41]) + check("entry incarnation aligns by trade with guarded fallback", + rendered["trades"][0]["entry_incarnation"] == 41 + and rendered["trades"][1]["entry_incarnation"] == 0) + + # --- UTF-16 object-key order + raw Unicode -------------------------- + chinese = m._canonical_fingerprint_json({"中文键": "中文值"}) + check("Chinese object is raw Unicode JSON", + chinese == '{"中文键":"中文值"}' and "\\u" not in chinese) + + emoji = m._canonical_fingerprint_json("😀") + check("emoji is raw Unicode", emoji == '"😀"' and "\\u" not in emoji) + + seps = m._canonical_fingerprint_json("\u2028\u2029") + check("U+2028/U+2029 raw (not ensure_ascii)", + seps == '"\u2028\u2029"' and "\\u2028" not in seps) + + # U+10000 sorts *before* U+E000 under UTF-16 code units (JCS/JS), + # but *after* under Unicode code-point order (Python sorted()). + key_order = m._canonical_fingerprint_json({"\U00010000": 1, "\uE000": 2}) + check("UTF-16 key order U+10000 before U+E000", + key_order == '{"\U00010000":1,"\uE000":2}') + check("UTF-16 order differs from code-point order", + sorted(["\U00010000", "\uE000"]) == ["\uE000", "\U00010000"]) + + special = m._canonical_fingerprint_json( + {"2": 2, "10": 10, "__proto__": "own"}) + check("special keys 10/2/__proto__ lexical UTF-16 order", + special == '{"10":10,"2":2,"__proto__":"own"}') + + # --- fail closed: nonfinite, surrogates, unsafe ints, dup keys ----- + nonfinite_ok = True + for bad in (float("nan"), float("inf"), float("-inf")): + try: + m._canonical_fingerprint_json({"x": bad}) + nonfinite_ok = False + except ValueError: + pass + check("non-finite numbers rejected", nonfinite_ok) + + surrogate_ok = True + for bad in ("\ud800", "\udfff", "a\ud800b"): + try: + m._canonical_fingerprint_json(bad) + surrogate_ok = False + except ValueError: + pass + try: + m._canonical_fingerprint_json({bad: "x"}) + surrogate_ok = False + except ValueError: + pass + check("unpaired surrogates rejected", surrogate_ok) + + unsafe_ok = True + for bad in (9007199254740992, # exact int 2**53 + 9007199254740993, + -9007199254740992, + 9223372036854775807): + try: + m._canonical_fingerprint_json(bad) + unsafe_ok = False + except ValueError: + pass + try: + m.build_fingerprint({"n": bad}) + unsafe_ok = False + except ValueError: + pass + check("unsafe exact ints rejected (safe-integer domain)", unsafe_ok) + # Isolated binary64 float(2**53) remains legal on the float path. + check("float 2**53 remains legal binary64 token", + m._canonical_fingerprint_json(9007199254740992.0) + == "9007199254740992") + + tagged_a = _TaggedStr("same", "a") + tagged_b = _TaggedStr("same", "b") + tagged_dup = {tagged_a: 1, tagged_b: 2} + check("tagged str keys coexist pre-normalization", + len(tagged_dup) == 2 + and str.__str__(tagged_a) == str.__str__(tagged_b) == "same") + dup_rejected = False + try: + m._canonical_fingerprint_json(tagged_dup) + except ValueError: + dup_rejected = True + check("duplicate normalized str-subclass keys fail closed", dup_rejected) + + # --- direct Node ECMAScript / JCS reconstruction (required) -------- + node = shutil.which("node") + if node is None: + check("Node available for JCS bridge", False) + print(" (install Node.js to run the ECMAScript/JCS bridge)") + else: + bridge_payload = { + "strategy": { + "initial_capital": 12345.0, + "commission_value": 0.0, + "default_qty_value": 1.0, + }, + "nested": {"arr": [0.0, -0.0, 1.0, 12345.0]}, + "中文键": "中文值", + "emoji": "😀", + "sep": "\u2028\u2029", + "\U00010000": "supplementary", + "\uE000": "pua", + "10": 10, + "2": 2, + "__proto__": "own", + } + py_canon = m._canonical_fingerprint_json(bridge_payload) + # Non-canonical Python JSON a Worker would re-canonicalize. + py_legacy = json.dumps(bridge_payload, sort_keys=True, + separators=(",", ":"), ensure_ascii=True) + script = r""" +const fs = require('fs'); +const input = JSON.parse(fs.readFileSync(0, 'utf8')); +// RFC 8785/JCS direct canonical writer over JS-parsed values. +// Do NOT rebuild a plain object (array-index key reorder; __proto__ drop). +function canonical(v) { + if (Array.isArray(v)) { + let s = '['; + for (let i = 0; i < v.length; i++) { + if (i) s += ','; + s += canonical(v[i]); + } + return s + ']'; + } + if (v !== null && typeof v === 'object') { + const keys = Object.keys(v).sort(); + let s = '{'; + for (let i = 0; i < keys.length; i++) { + const k = keys[i]; + if (i) s += ','; + s += JSON.stringify(k) + ':' + canonical(v[k]); + } + return s + '}'; + } + return JSON.stringify(v); +} +const fromLegacy = canonical(JSON.parse(input.legacy)); +const fromCanon = canonical(JSON.parse(input.canonical)); +process.stdout.write(JSON.stringify({fromLegacy, fromCanon})); +""" + proc = subprocess.run( + [node, "-e", script], + input=json.dumps({ + "legacy": py_legacy, + "canonical": py_canon, + }, ensure_ascii=False), + text=True, capture_output=True, check=False) + if proc.returncode != 0: + check("Node JCS bridge executes", False) + print(proc.stderr) + else: + out = json.loads(proc.stdout) + check("Node direct JCS reconstruction is byte-identical", + out["fromLegacy"] == py_canon + and out["fromCanon"] == py_canon) + js_digest = ("sha256:" + + hashlib.sha256( + out["fromLegacy"].encode("utf-8")).hexdigest()) + check("Node-emitted bytes hash to the same digest", + js_digest == m.build_fingerprint(bridge_payload)["digest"]) + check("bridge payload retains special keys + UTF-16 order", + '"10":10' in py_canon + and '"2":2' in py_canon + and '"__proto__":"own"' in py_canon + and py_canon.index('"\U00010000"') + < py_canon.index('"\uE000"') + and "中文键" in py_canon + and "\\u4e2d" not in py_canon) + + print(f"\n{passed} passed, {failed} failed") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main())