diff --git a/CLAUDE.md b/CLAUDE.md index 7bec8e9cd0..ec5ca93260 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1552 +**Current Version:** 0.5.1553 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index c55de3dccc..e82c8fe74a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5690,7 +5690,7 @@ checksum = "060b4682849d20ebcba05d68f9584a1bba20af4b7838c688cfb37af572f562f4" [[package]] name = "perry" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "base64 0.22.1", @@ -5754,7 +5754,7 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-dispatch", "serde", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-audio-miniaudio" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "cc", "libc", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "aho-corasick", "anyhow", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-hir", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-hir", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-dispatch", @@ -5814,7 +5814,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-hir", @@ -5822,7 +5822,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "base64 0.22.1", @@ -5834,7 +5834,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-hir", @@ -5842,7 +5842,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "async-trait", @@ -5870,14 +5870,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "serde", "serde_json", @@ -5885,7 +5885,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1552" +version = "0.5.1553" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "clap", @@ -5911,7 +5911,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "block2", "objc2", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "argon2", "perry-ffi", @@ -5930,7 +5930,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "reqwest", @@ -5939,7 +5939,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "bcrypt", "perry-ffi", @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "rusqlite", @@ -5955,7 +5955,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "scraper", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "perry-runtime", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "chrono", "cron", @@ -5981,7 +5981,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "chrono", "perry-ffi", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "rust_decimal", @@ -5997,7 +5997,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "serde_json", @@ -6005,7 +6005,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "rand 0.10.2", @@ -6013,7 +6013,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "perry-runtime", @@ -6021,14 +6021,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "bytes", "http-body-util", @@ -6046,7 +6046,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "bytes", "lazy_static", @@ -6059,7 +6059,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "bytes", @@ -6091,7 +6091,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "lazy_static", "perry-ffi", @@ -6101,7 +6101,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "jsonwebtoken", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "lru", "perry-ffi", @@ -6121,7 +6121,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "chrono", "perry-ffi", @@ -6129,7 +6129,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "bson", "futures-util", @@ -6141,7 +6141,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "chrono", "perry-ffi", @@ -6153,7 +6153,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "nanoid", "perry-ffi", @@ -6162,7 +6162,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "bytes", "perry-ffi", @@ -6177,7 +6177,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "const-oid 0.10.2", "der 0.8.1", @@ -6196,7 +6196,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "lettre", "perry-ffi", @@ -6206,7 +6206,7 @@ dependencies = [ [[package]] name = "perry-ext-parcel-watcher" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "notify", "perry-ffi", @@ -6218,7 +6218,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "printpdf", @@ -6226,7 +6226,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "sqlx", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "perry-ext-qs" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "perry-runtime", @@ -6244,7 +6244,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "governor", "perry-ffi", @@ -6252,7 +6252,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "fast_image_resize", "image", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "perry-ext-streams" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "lazy_static", "perry-ffi", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "perry-ext-typescript" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-ffi", @@ -6292,7 +6292,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "perry-runtime", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "uuid", @@ -6309,7 +6309,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "perry-validation", @@ -6318,7 +6318,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "futures-util", "lazy_static", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "brotli", "flate2", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "dashmap 6.2.1", "once_cell", @@ -6351,7 +6351,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-api-manifest", @@ -6372,11 +6372,11 @@ dependencies = [ [[package]] name = "perry-native-registration" -version = "0.5.1552" +version = "0.5.1553" [[package]] name = "perry-parser" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-diagnostics", @@ -6390,7 +6390,7 @@ dependencies = [ [[package]] name = "perry-perex" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perex", "regex", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "ahash", "anyhow", @@ -6458,14 +6458,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6560,14 +6560,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "perry-hir", @@ -6576,7 +6576,7 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "perry-ffi", "perry-ui-model", @@ -6584,7 +6584,7 @@ dependencies = [ [[package]] name = "perry-ui-android" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "itoa", @@ -6602,7 +6602,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "rand 0.10.2", "serde", @@ -6612,7 +6612,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "cairo-rs 0.22.9", @@ -6635,7 +6635,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "block2", @@ -6652,7 +6652,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "block2", @@ -6669,7 +6669,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1552" +version = "0.5.1553" [[package]] name = "perry-ui-test" @@ -6680,11 +6680,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1552" +version = "0.5.1553" [[package]] name = "perry-ui-tvos" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "block2", @@ -6701,7 +6701,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "block2", @@ -6718,7 +6718,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "block2", "libc", @@ -6732,7 +6732,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "libc", @@ -6751,7 +6751,7 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "base64 0.22.1", "libc", @@ -6764,7 +6764,7 @@ dependencies = [ [[package]] name = "perry-updater" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "anyhow", "base64 0.22.1", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "perry-validation" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "idna", "regex", @@ -6790,7 +6790,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1552" +version = "0.5.1553" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 81fc773085..2a5886a934 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,7 +338,7 @@ codegen-units = 1 codegen-units = 1 [workspace.package] -version = "0.5.1552" +version = "0.5.1553" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/10177-json-leaf-arena-route.md b/changelog.d/10177-json-leaf-arena-route.md new file mode 100644 index 0000000000..76a031d05a --- /dev/null +++ b/changelog.d/10177-json-leaf-arena-route.md @@ -0,0 +1,3 @@ +### perf(gc): birth a large JSON result in the arena when the young generation already holds a document + +JSON results at or above 512 KB are malloc-tracked so the next minor can reclaim a discarded result cheaply. On a parse-then-stringify loop over a document-sized input that inverted: the non-empty malloc registry forbids the untraced in-place promotion, so every minor traced the whole freshly parsed tree (55 ms for 20 MB, half the roundtrip's wall time) to reclaim one leaf. Such a leaf is now born old in the arena when the young generation holds at least the leaf's bytes and a base nursery's worth of data and was not measured as dying, and the next trigger decision gives the nursery minor one-time priority over old-reclaim while the young generation is unmeasured, so a stringify-only loop still promotes its input once and then keeps the malloc path. Measured on `records_array_20m:roundtrip`: CPU 206.8 → 114.1 ms (0.55×, now ahead of the better of Node 26.5.1 / Bun 1.3.14 at 162.1) and peak RSS 295 → 266 MiB (Node 261); 20 MB stringify rows −40 MB peak RSS at flat CPU; every other JSON row within noise (#10169). diff --git a/changelog.d/10186-fast-emit-budget-x86-64.md b/changelog.d/10186-fast-emit-budget-x86-64.md new file mode 100644 index 0000000000..aadb6d5fde --- /dev/null +++ b/changelog.d/10186-fast-emit-budget-x86-64.md @@ -0,0 +1,18 @@ +Raise the optimized machine-pipeline budget on x86-64 targets from 100,000 to +600,000 post-optimization instructions. The budget's fallback demotes the whole +codegen unit to LLVM's O0 machine pipeline, not just the over-budget function +(LLVM selects the pipeline per module, and `optnone` on one function frees its +siblings without bounding regalloc time or memory), so on the OpenCode build +61 over-budget functions dragged 140 MiB of ordinary sibling code into O0 +emission with them: 42 % of the binary's text. With the corpus's whole giant +population admitted, the three specimen modules lose 25 % / 42 % / 69 % of +their `.text`, runtime instructions and RSS are unchanged, and compile cost +stays bounded (+13–28 % wall, ~2.5 GB peak on the worst specimen). + +aarch64/arm64 and every other unmeasured target keep the old 100,000 ceiling: +the two observations that set it (a 100k-instruction function past ~10 GiB +RSS; a 277k-instruction function >16 min in register allocation) are arm64 +and have not been re-measured. `PERRY_LL_FAST_EMIT_MAX_INSTRS` still overrides +on every target; on x86-64 `=100000` reproduces the previous output byte for +byte. The fallback diagnostic now names every over-budget function, widest +first, and says how many functions in its unit are demoted alongside it. diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs index 7b4affa613..f8fe02b633 100644 --- a/crates/perry-codegen/src/inprocess.rs +++ b/crates/perry-codegen/src/inprocess.rs @@ -352,9 +352,12 @@ pub struct UnitCodegenStats { /// Functions stamped `"disable-tail-calls"` because their alloca-walk /// estimate exceeded [`DEFAULT_TRE_MAX_ALLOCA_WALK`] (#8883). pub tail_call_elim_skipped: Vec, - /// The widest function which made this unit use LLVM's bounded O0 machine - /// pipeline after completing the requested IR optimization pipeline. - pub fast_emit_fallback: Option, + /// Every function over the target's ceiling (see + /// [`default_fast_emit_max_instrs`]), widest first — + /// the ones which made this unit use LLVM's bounded O0 machine pipeline + /// after completing the requested IR optimization pipeline. Empty when + /// the unit kept the optimized machine pipeline. + pub fast_emit_fallbacks: Vec, } fn function_instruction_count(function: inkwell::values::FunctionValue<'_>) -> usize { @@ -400,20 +403,85 @@ fn module_instruction_census( /// live-interval and register-allocation pipeline for a unit containing an /// extreme generated function. /// -/// The threshold is bracketed by real arm64/LLVM 22 measurements. Machine-IR -/// expansion depends on CFG shape, so raw IR size is deliberately only a -/// conservative guard: one 161k-instruction function emitted normally in -/// ~19s, while a different 100,152-instruction Claude Code 2.1.259 function -/// grew past ~10 GiB RSS in the optimized machine pipeline. The same function -/// emitted through an O0 target machine in 6s. Another 277k-instruction async -/// state-machine function remained in LiveIntervals / register allocation for -/// more than 16 minutes at ~10 GiB RSS; its already-Os-optimized IR emitted -/// through an O0 target machine in 3.5s at ~550 MiB RSS. 100k is immediately -/// below the smallest observed pathological case. +/// **The demotion is a whole-unit act, so the budget must not be set where +/// ordinary functions pay for it.** A `TargetMachine`'s optimization level is +/// a per-module property: LLVM has no per-function escape from the optimized +/// machine pipeline (`optnone` reaches instruction selection and the optional +/// machine passes, but *not* LiveIntervals or the greedy register allocator — +/// measured below), so every ordinary function sharing the unit with one +/// extreme function is emitted through the O0 machine pipeline too. /// -/// `PERRY_LL_FAST_EMIT_MAX_INSTRS=` raises or lowers the ceiling; `0` / -/// `off` disables the fallback. -const DEFAULT_FAST_EMIT_MAX_INSTRS: usize = 100_000; +/// Measured on `@babel/parser`'s unit 0, LLVM 22 / x86-64 / `-Os` IR pipeline: +/// one 227,108-instruction closure (163,100 of those are `gc.relocate`) and +/// 282 ordinary siblings, each arm emitting the same post-`default` IR: +/// +/// | machine pipeline | unit `.text` | the closure | its 282 siblings | `llc` | peak RSS | +/// |---|---|---|---|---|---| +/// | optimized (`-O2`) | 1,689,851 B | 241,218 B | 1.382 MiB | 10.0 s | 464 MiB | +/// | O0 (this fallback) | 5,862,077 B | 2,253,658 B | 3.441 MiB | 3.9 s | 499 MiB | +/// | `optnone` on the closure only | 2,070,326 B | 621,693 B | 1.382 MiB | 9.5 s | 518 MiB | +/// | the same unit *without* the closure | 1,448,633 B | — | 1.382 MiB | 6.4 s | 208 MiB | +/// +/// So the siblings are pure loss: the fallback costs them 2.06 MiB of machine +/// code (168 of 282 functions change) to save ~6 s, and their emitted code is +/// byte-for-byte what a unit without the extreme function produces as soon as +/// the unit keeps the optimized pipeline. The `optnone` row is why this is a +/// budget and not a per-function demotion: it frees the siblings but bounds +/// neither time (9.5 s of 10.0 s) nor memory (518 MiB — *above* the -O2 arm), +/// because the greedy allocator still runs on the demoted function. +/// +/// On x86-64 the ceiling is therefore set above the whole measured +/// population of extreme generated functions rather than immediately below +/// the smallest pathological one. On the OpenCode corpus 60 of the 61 +/// functions past the old 100k ceiling are under 600k (median 153,455; +/// largest 982,912), and the +/// largest one measured end-to-end — the 522,756-instruction `mime` +/// `types/other.ts` constructor — emits through the optimized machine pipeline +/// in 704 s at 2.26 GB peak RSS, against 393 s at 2.34 GB demoted, for a +/// module `.text` of 13.72 MB against 36.97 MB. +/// +/// **Every measurement above is x86-64, so only x86-64 gets the raised +/// ceiling.** Machine-IR expansion depends on CFG shape *and* on the target's +/// instruction selection and register allocation, and the two observations +/// that set the 100k ceiling in the first place are both arm64/LLVM 22: a +/// 100,152-instruction Claude Code 2.1.259 function grew past ~10 GiB RSS in +/// the optimized machine pipeline (6 s through an O0 target machine), and a +/// 277k-instruction async state-machine function sat in LiveIntervals / +/// register allocation for more than 16 minutes at ~10 GiB (3.5 s at ~550 MiB +/// demoted). Both postdate #8679's shadow-frame retry, so they are current +/// observations, not stale ones — and both sit *inside* the 600k band. Every +/// CI runner and developer build here is macOS arm64, so raising the ceiling +/// there on x86-64 evidence would trade a measured size win for an unmeasured +/// 10 GiB compile. aarch64/arm64 — and every other target nobody has measured +/// — therefore keep 100k until someone measures them the way `x86_64` was +/// measured above, at which point `default_fast_emit_max_instrs` grows a +/// match arm and this comment grows a row. +/// +/// `PERRY_LL_FAST_EMIT_MAX_INSTRS=` raises or lowers the ceiling on every +/// target; `0` / `off` disables the fallback. On x86-64, +/// `PERRY_LL_FAST_EMIT_MAX_INSTRS=100000` reproduces the old behaviour +/// byte-for-byte. +const DEFAULT_FAST_EMIT_MAX_INSTRS_X86_64: usize = 600_000; + +/// The ceiling for every target whose optimized machine pipeline has not been +/// measured against a corpus of extreme generated functions — including +/// aarch64/arm64, where the two pathological observations quoted in +/// [`DEFAULT_FAST_EMIT_MAX_INSTRS_X86_64`] were made. +const DEFAULT_FAST_EMIT_MAX_INSTRS_UNMEASURED: usize = 100_000; + +/// The ceiling for the target this unit is being emitted *for* — not the host. +/// A cross-compile from an x86-64 box to arm64 runs arm64's instruction +/// selection and register allocator, so it is arm64's ceiling that applies. +fn default_fast_emit_max_instrs(effective_target: &str) -> usize { + let arch = effective_target + .split('-') + .next() + .unwrap_or(effective_target); + match arch { + "x86_64" | "x86_64h" | "amd64" => DEFAULT_FAST_EMIT_MAX_INSTRS_X86_64, + _ => DEFAULT_FAST_EMIT_MAX_INSTRS_UNMEASURED, + } +} #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum FastEmitBudget { @@ -421,19 +489,20 @@ enum FastEmitBudget { Cap(usize), } -fn parse_fast_emit_budget(value: Option<&str>) -> FastEmitBudget { +fn parse_fast_emit_budget(value: Option<&str>, effective_target: &str) -> FastEmitBudget { + let default = || FastEmitBudget::Cap(default_fast_emit_max_instrs(effective_target)); match value.map(str::trim) { - None | Some("") => FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS), + None | Some("") => default(), Some("0") | Some("off") | Some("false") => FastEmitBudget::Off, Some(v) => match v.parse::() { Ok(0) => FastEmitBudget::Off, Ok(n) => FastEmitBudget::Cap(n), - Err(_) => FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS), + Err(_) => default(), }, } } -fn fast_emit_budget() -> FastEmitBudget { +fn fast_emit_budget(effective_target: &str) -> FastEmitBudget { #[cfg(test)] if let Some(budget) = TEST_FAST_EMIT_BUDGET.with(std::cell::Cell::get) { return budget; @@ -442,6 +511,7 @@ fn fast_emit_budget() -> FastEmitBudget { std::env::var("PERRY_LL_FAST_EMIT_MAX_INSTRS") .ok() .as_deref(), + effective_target, ) } @@ -456,23 +526,34 @@ thread_local! { /// other LLVM tests in this binary. #[cfg(test)] fn with_test_fast_emit_budget(cap: usize, run: impl FnOnce() -> T) -> T { + with_test_fast_emit_budget_value(FastEmitBudget::Cap(cap), run) +} + +/// [`with_test_fast_emit_budget`] for a budget that is not a cap — the arm +/// that proves what an undemoted unit emits. +#[cfg(test)] +fn with_test_fast_emit_budget_value(budget: FastEmitBudget, run: impl FnOnce() -> T) -> T { struct Restore(Option); impl Drop for Restore { fn drop(&mut self) { TEST_FAST_EMIT_BUDGET.with(|budget| budget.set(self.0)); } } - let old = TEST_FAST_EMIT_BUDGET.replace(Some(FastEmitBudget::Cap(cap))); + let old = TEST_FAST_EMIT_BUDGET.replace(Some(budget)); let _restore = Restore(old); run() } -/// The extreme function which selected bounded machine-code emission. +/// One extreme function which selected bounded machine-code emission, and how +/// many defined functions in its unit are demoted along with it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FastEmitFallback { pub name: String, pub instructions: usize, pub cap: usize, + /// Defined functions in the unit — the size of the collateral, since the + /// machine pipeline is selected per module and not per function. + pub unit_functions: usize, } impl std::fmt::Display for FastEmitFallback { @@ -480,43 +561,57 @@ impl std::fmt::Display for FastEmitFallback { write!( f, "`{}` has {} instructions after IR optimization, above the optimized machine-pipeline \ - budget {}; keeping the requested IR optimization, then emitting this unit through \ - LLVM's O0 machine pipeline to bound instruction selection, live intervals and \ - register allocation. Override with PERRY_LL_FAST_EMIT_MAX_INSTRS= (raise) or \ - =0 (disable).", - self.name, self.instructions, self.cap + budget {}; keeping the requested IR optimization, then emitting this unit — all {} \ + of its defined functions, not only this one — through LLVM's O0 machine pipeline to \ + bound instruction selection, live intervals and register allocation. LLVM selects \ + that pipeline per module, so the siblings are demoted too and grow: shrinking this \ + function is what lifts the whole unit back. Override with \ + PERRY_LL_FAST_EMIT_MAX_INSTRS= (raise) or =0 (disable).", + self.name, self.instructions, self.cap, self.unit_functions ) } } -fn fast_emit_fallback( +/// Every defined function over `budget`, widest first. +/// +/// The decision is per function; the consequence cannot be (see +/// [`DEFAULT_FAST_EMIT_MAX_INSTRS_X86_64`]), which is why every offender is +/// returned +/// rather than only the widest: the compile log then names each function that +/// has to shrink before the unit can keep the optimized machine pipeline, +/// instead of naming one and re-reporting a new widest on the next build. +fn fast_emit_fallbacks( module: &inkwell::module::Module<'_>, budget: FastEmitBudget, -) -> Option { +) -> Vec { let cap = match budget { - FastEmitBudget::Off => return None, + FastEmitBudget::Off => return Vec::new(), FastEmitBudget::Cap(cap) => cap, }; - let mut widest: Option = None; + let mut defined = 0usize; + let mut over: Vec<(String, usize)> = Vec::new(); let mut function = module.get_first_function(); while let Some(f) = function { if f.count_basic_blocks() > 0 { + defined += 1; let instructions = function_instruction_count(f); - if instructions > cap - && widest - .as_ref() - .is_none_or(|current| instructions > current.instructions) - { - widest = Some(FastEmitFallback { - name: f.get_name().to_string_lossy().into_owned(), - instructions, - cap, - }); + if instructions > cap { + over.push((f.get_name().to_string_lossy().into_owned(), instructions)); } } function = f.get_next_function(); } - widest + // Widest first, ties by name: one deterministic order for the log and the + // per-unit report, whatever order LLVM holds the functions in. + over.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + over.into_iter() + .map(|(name, instructions)| FastEmitFallback { + name, + instructions, + cap, + unit_functions: defined, + }) + .collect() } /// Instruction budget for ONE function after `rewrite-statepoints-for-gc`. diff --git a/crates/perry-codegen/src/inprocess/optimize_emit.rs b/crates/perry-codegen/src/inprocess/optimize_emit.rs index a0f1e240ec..e7b879b809 100644 --- a/crates/perry-codegen/src/inprocess/optimize_emit.rs +++ b/crates/perry-codegen/src/inprocess/optimize_emit.rs @@ -175,18 +175,25 @@ pub(super) fn optimize_and_emit( // register allocation. Use an O0 target machine only for final emission // of that unit; ordinary units keep `tm`, and the optimized IR is not // rebuilt or demoted. + // + // The selection is per function, the emission cannot be — a TargetMachine + // carries one optimization level for the whole module, and `optnone` does + // not reach LiveIntervals or the register allocator. Every ordinary + // function in the unit is demoted with the offender, which is why the + // budget sits above the measured population of extreme functions and why + // the log below names each of them. let fast_emit = if opt == '0' { - None + Vec::new() } else { - fast_emit_fallback(module, fast_emit_budget()) + fast_emit_fallbacks(module, fast_emit_budget(effective_target)) }; - if let Some(fallback) = &fast_emit { + for fallback in &fast_emit { eprintln!("perry: {fallback}"); } if let Some(stats) = stats.as_deref_mut() { - stats.fast_emit_fallback = fast_emit.clone(); + stats.fast_emit_fallbacks = fast_emit.clone(); } - let fast_tm = if fast_emit.is_some() { + let fast_tm = if !fast_emit.is_empty() { Some( target .create_target_machine( @@ -1014,27 +1021,47 @@ entry: ); } + /// Spellings, on both an x86-64 and an arm64 target: only the default + /// differs between them, and an explicit setting wins on either. #[test] fn fast_emit_budget_spellings() { - assert_eq!( - parse_fast_emit_budget(None), - FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS) - ); - assert_eq!( - parse_fast_emit_budget(Some("")), - FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS) - ); - assert_eq!(parse_fast_emit_budget(Some("0")), FastEmitBudget::Off); - assert_eq!(parse_fast_emit_budget(Some("off")), FastEmitBudget::Off); - assert_eq!(parse_fast_emit_budget(Some("false")), FastEmitBudget::Off); - assert_eq!( - parse_fast_emit_budget(Some(" 250000 ")), - FastEmitBudget::Cap(250_000) - ); - assert_eq!( - parse_fast_emit_budget(Some("lots")), - FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS) - ); + for (target, default) in [ + ("x86_64-unknown-linux-gnu", 600_000), + ("arm64-apple-darwin", 100_000), + ] { + assert_eq!( + parse_fast_emit_budget(None, target), + FastEmitBudget::Cap(default), + "{target}" + ); + assert_eq!( + parse_fast_emit_budget(Some(""), target), + FastEmitBudget::Cap(default), + "{target}" + ); + assert_eq!( + parse_fast_emit_budget(Some("0"), target), + FastEmitBudget::Off + ); + assert_eq!( + parse_fast_emit_budget(Some("off"), target), + FastEmitBudget::Off + ); + assert_eq!( + parse_fast_emit_budget(Some("false"), target), + FastEmitBudget::Off + ); + assert_eq!( + parse_fast_emit_budget(Some(" 250000 "), target), + FastEmitBudget::Cap(250_000), + "an explicit ceiling wins on every target" + ); + assert_eq!( + parse_fast_emit_budget(Some("lots"), target), + FastEmitBudget::Cap(default), + "{target}" + ); + } } /// Two functions: `wide` has 4 allocas across 9 instructions (estimate @@ -1132,32 +1159,47 @@ entry: } /// Selection is per function, the boundary is inclusive, declarations do - /// not count, and the diagnostic names the widest violating function. + /// not count, every violator is reported (widest first) and the + /// diagnostic names the function and the size of the collateral. #[test] fn fast_emit_budget_selects_only_above_the_boundary() { let context = Context::create(); let module = parse_ir_text(&context, alloca_walk_fixture(), "fast_emit_fixture") .expect("fixture parses"); - assert!(fast_emit_fallback(&module, FastEmitBudget::Off).is_none()); - assert!(fast_emit_fallback(&module, FastEmitBudget::Cap(9)).is_none()); + assert!(fast_emit_fallbacks(&module, FastEmitBudget::Off).is_empty()); + assert!(fast_emit_fallbacks(&module, FastEmitBudget::Cap(9)).is_empty()); - let fallback = fast_emit_fallback(&module, FastEmitBudget::Cap(8)) - .expect("wide is one instruction over the budget"); + let fallbacks = fast_emit_fallbacks(&module, FastEmitBudget::Cap(8)); assert_eq!( - fallback, - FastEmitFallback { + fallbacks, + vec![FastEmitFallback { name: "wide".to_string(), instructions: 9, cap: 8, - } + // `narrow` is under the cap and `sink` is a declaration; both + // are still emitted by the demoted machine pipeline. + unit_functions: 2, + }], + "only the function over the budget is selected" ); - let message = fallback.to_string(); + + // Every violator, widest first: one build names the whole set that + // has to shrink instead of re-reporting a new widest each time. + let both = fast_emit_fallbacks(&module, FastEmitBudget::Cap(2)); + let named: Vec<(&str, usize)> = both + .iter() + .map(|f| (f.name.as_str(), f.instructions)) + .collect(); + assert_eq!(named, [("wide", 9), ("narrow", 3)]); + + let message = fallbacks[0].to_string(); for needle in [ "`wide`", "9 instructions", "budget 8", "requested IR optimization", "O0 machine pipeline", + "all 2 of its defined functions", "PERRY_LL_FAST_EMIT_MAX_INSTRS", ] { assert!( @@ -1167,6 +1209,60 @@ entry: } } + /// The ceiling is a calibration, not a round number, and it is only as + /// wide as the target it was measured on. x86-64 must sit above the + /// measured population of extreme generated functions (OpenCode corpus: + /// 60 of the 61 functions past the old 100k ceiling are under 600k, and + /// the largest one measured end to end is `mime` `types/other.ts`'s + /// 522,756-instruction constructor, which emits through the optimized + /// machine pipeline in 704 s at 2.26 GB) and below the RS4GC + /// relocation-fan-out budget, which is the backstop that runs first and + /// re-lowers rather than demotes. Every unmeasured target — aarch64 + /// above all, where both pathological observations were made — keeps + /// 100k. Raising one of those is a measurement, not an edit. + #[test] + fn fast_emit_budget_admits_the_measured_giant_population_on_measured_targets_only() { + assert_eq!(DEFAULT_FAST_EMIT_MAX_INSTRS_X86_64, 600_000); + assert!( + DEFAULT_FAST_EMIT_MAX_INSTRS_X86_64 > 522_756, + "the largest measured-affordable function must not be demoted" + ); + assert!( + DEFAULT_FAST_EMIT_MAX_INSTRS_X86_64 < DEFAULT_RS4GC_MAX_INSTRS, + "a function this wide is handled by the RS4GC budget first" + ); + assert_eq!(DEFAULT_FAST_EMIT_MAX_INSTRS_UNMEASURED, 100_000); + + for target in [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64h-apple-darwin", + ] { + assert_eq!( + default_fast_emit_max_instrs(target), + 600_000, + "{target} is measured" + ); + } + // The host is irrelevant: a cross-compile runs the *target's* ISel and + // register allocator, so these keep the unmeasured ceiling even from an + // x86-64 box. + for target in [ + "arm64-apple-darwin", + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "arm64_32-apple-watchos", + "i686-unknown-linux-gnu", + "", + ] { + assert_eq!( + default_fast_emit_max_instrs(target), + 100_000, + "{target:?} has not been measured" + ); + } + } + /// A self-recursive tail call that TailCallElim turns into a loop at /// the pinned LLVM: with no attribute the recursive `call` disappears, /// with `"disable-tail-calls"="true"` (exactly what the budget stamps) @@ -1269,7 +1365,7 @@ entry: /// A tiny test cap proves the shipped path records and successfully uses /// the second, O0 target machine only after running the requested Os IR /// pipeline. The production threshold is pinned by the parser test and - /// the real Claude-Code measurement in its constant's documentation. + /// the real measurements in its constant's documentation. #[test] fn fast_emit_budget_is_applied_by_the_shipped_pipeline() { global_init(&[]); @@ -1290,8 +1386,10 @@ entry: .expect("the already-optimized module emits through the bounded target machine"); assert!(!object.is_empty()); let fallback = stats - .fast_emit_fallback - .expect("the shipped path must report the selected fallback"); + .fast_emit_fallbacks + .first() + .expect("the shipped path must report the selected fallback") + .clone(); assert_eq!(fallback.name, "wide"); assert!(fallback.instructions > fallback.cap); assert_eq!(fallback.cap, 1); @@ -1311,6 +1409,133 @@ entry: ) }) .expect("-O0 emits"); - assert!(stats.fast_emit_fallback.is_none()); + assert!(stats.fast_emit_fallbacks.is_empty()); + } + + /// One function over the budget, one ordinary function that is nowhere + /// near it, and one external callee so nothing folds away. `narrow` + /// holds three values across three calls, which is what makes its + /// machine code differ between the optimized and the O0 register + /// allocators. + fn sibling_cost_fixture(with_wide: bool) -> String { + let wide = r#" +define i64 @wide(i64 %n) { +entry: + %a = call i64 @src(i64 %n) + %b = call i64 @src(i64 %a) + %c = call i64 @src(i64 %b) + %d = call i64 @src(i64 %c) + %s = add i64 %a, %b + %t = add i64 %s, %c + %u = add i64 %t, %d + ret i64 %u +} +"#; + format!( + r#" +declare i64 @src(i64) + +define i64 @narrow(i64 %x, i64 %y) {{ +entry: + %a = call i64 @src(i64 %x) + %b = call i64 @src(i64 %y) + %c = call i64 @src(i64 %a) + %s = add i64 %a, %b + %t = add i64 %s, %c + ret i64 %t +}} +{}"#, + if with_wide { wide } else { "" } + ) + } + + /// The assembly of one function, from its label to the end of its body. + /// Tolerates ELF (`narrow:` / `.size`) and Mach-O (`_narrow:`) spelling. + fn function_assembly(asm: &str, name: &str) -> String { + let label_elf = format!("{name}:"); + let label_macho = format!("_{name}:"); + let mut body: Vec<&str> = Vec::new(); + let mut inside = false; + for line in asm.lines() { + let trimmed = line.trim(); + if !inside { + inside = trimmed == label_elf || trimmed == label_macho; + continue; + } + let next_symbol = trimmed.ends_with(':') + && !trimmed.starts_with('.') + && !trimmed.starts_with('L') + && !trimmed.contains(' '); + if trimmed.starts_with(".size") || trimmed == ".cfi_endproc" || next_symbol { + break; + } + body.push(line); + } + assert!( + body.len() > 3, + "no body extracted for `{name}` — the assertion below would be vacuous:\n{asm}" + ); + body.join("\n") + } + + fn emit_assembly(ir: &str, module_name: &str, budget: FastEmitBudget) -> (String, Vec) { + global_init(&[]); + let target = crate::codegen::default_target_triple(); + let context = Context::create(); + let module = parse_ir_text(&context, ir, module_name).expect("fixture parses"); + let mut stats = UnitCodegenStats::default(); + let asm = with_test_fast_emit_budget_value(budget, || { + optimize_and_emit_module_with_stats( + &module, + &target, + &["-Os".into(), "-S".into()], + false, + Some(&mut stats), + ) + }) + .expect("the fixture emits"); + ( + String::from_utf8(asm).expect("LLVM emits UTF-8 assembly"), + stats + .fast_emit_fallbacks + .iter() + .map(|f| f.name.clone()) + .collect(), + ) + } + + /// What the budget actually costs, and why it is calibrated above the + /// measured population instead of below the smallest pathological case: + /// the demotion is a whole-unit act. An ordinary function emits the same + /// machine code whether or not an extreme function shares its unit — but + /// only while the unit keeps the optimized machine pipeline. Cross the + /// budget and that ordinary function's code changes too, without ever + /// having been over any budget itself. + #[test] + fn the_budget_is_what_makes_ordinary_siblings_pay() { + let with_wide = sibling_cost_fixture(true); + let alone = sibling_cost_fixture(false); + + let (undemoted, none) = emit_assembly(&with_wide, "sibling_cost_ok", FastEmitBudget::Off); + assert!(none.is_empty(), "this arm must not demote: {none:?}"); + let (solo, _) = emit_assembly(&alone, "sibling_cost_alone", FastEmitBudget::Off); + assert_eq!( + function_assembly(&undemoted, "narrow"), + function_assembly(&solo, "narrow"), + "an undemoted unit emits an ordinary function exactly as a unit without the \ + extreme function does" + ); + + // Same module, same IR pipeline, budget crossed: `narrow` is not over + // it and is compiled differently anyway. + let (demoted, over) = + emit_assembly(&with_wide, "sibling_cost_demoted", FastEmitBudget::Cap(7)); + assert_eq!(over, ["wide"], "only `wide` is over the budget"); + assert_ne!( + function_assembly(&demoted, "narrow"), + function_assembly(&undemoted, "narrow"), + "if the demotion did not reach `narrow`, LLVM grew a per-function escape from the \ + optimized machine pipeline and this budget can move back down" + ); } } diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 9223e1c058..2098869ce1 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -39,9 +39,11 @@ mod json_defer; mod policy; pub(crate) use json_defer::JsonParseAllocation; pub(crate) use policy::gc_runtime_safepoint; +pub(crate) use policy::note_young_leaf_born_old; /// The one writer of `GC_SAFEPOINT_PENDING` — it also keeps the poll's global /// arming shadow in step. See `gc/poll_arm.rs`. pub(crate) use policy::set_safepoint_pending; +pub(crate) use policy::young_generation_holds_a_nursery; pub use policy::*; mod progress; pub use progress::*; @@ -231,10 +233,17 @@ mod native_stack_scan; /// mechanism is `arena/promote.rs`; this decides when to use it. mod promote_in_place; use promote_in_place::*; +#[cfg(test)] +pub(crate) use promote_in_place::{ + clear_young_survival_for_tests, last_young_survival_permille, seed_young_survival_for_tests, +}; pub use promote_in_place::{ first_cycle_promotion_attempts, first_cycle_promotion_rollbacks, in_place_promoted_objects, in_place_promotion_cycles, untraced_promoted_objects, untraced_promotion_cycles, }; +pub(crate) use promote_in_place::{ + young_generation_measured_dying, young_generation_measured_retained, +}; /// Instrument-liveness counters (#7604): copying minors completed, objects /// relocated, loop back-edge polls reached. Mode-independent — they count what /// the COLLECTOR did, not what forced it, so they outlive any one stress knob. diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index b2a25b101c..6a772c0b14 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -120,6 +120,19 @@ pub(super) fn young_scavenge_cap_due() -> bool { from_space_in_use >= scavenge_nursery_cap_dueness_bytes() } +/// #10169: does the young generation hold at least one BASE nursery cap of +/// bytes — a minor's worth of data, whatever the adaptive cap currently says? +/// Read by `string::json_leaf_prefers_arena` from inside a suppressed +/// construction window. The base cap rather than [`young_scavenge_cap_due`]'s +/// adaptive one on purpose: after a fully-live minor the adaptive cap scales +/// past the very tree that made it scale, and a gate keyed on it would flip +/// the next leaf back to the malloc registry, where one tracked leaf is enough +/// to veto the untraced promotion the route exists to enable. +pub(crate) fn young_generation_holds_a_nursery() -> bool { + nursery_cap_active() + && crate::arena::copying_from_space_in_use_bytes() >= gc_scavenge_nursery_cap_bytes() +} + /// The cap value [`young_scavenge_cap_due`] compares against. /// /// Split out only so a test can make the cap due without allocating the real @@ -1085,6 +1098,9 @@ crate::perry_thread_local! { pub(super) static GC_DEFERRED_REQUEST: Cell = const { Cell::new(DeferredGcRequest::None) }; pub(super) static GC_OLD_RECLAIM_PENDING: Cell = const { Cell::new(false) }; + /// #10169: a document-sized JSON leaf was born old under young pressure + /// since the last trigger decision (`note_young_leaf_born_old`). + pub(super) static GC_YOUNG_LEAF_BORN_OLD: Cell = const { Cell::new(false) }; pub(super) static GC_LAST_OLD_RECLAIM_IN_USE_BYTES: Cell = const { Cell::new(0) }; /// Live allocated arena bytes measured right after the last FULL /// mark-sweep — the baseline for major-GC pacing @@ -3117,7 +3133,7 @@ struct BudgetedGcCycle { } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum BudgetedGcTrigger { +pub(super) enum BudgetedGcTrigger { OldReclaim, ArenaBytes, /// The young-generation scavenge cap ([`young_scavenge_cap_due`]). @@ -3230,7 +3246,31 @@ pub(crate) fn trigger_path_hot_slot_indices() -> Vec<(&'static str, u32)> { ] } -fn gc_budgeted_due_trigger() -> Option { +/// #10169: record that a document-sized JSON leaf was just born old in the +/// arena because the young generation is at least that large and has not been +/// measured as dying (`string::json_leaf_prefers_arena`). Read once by the +/// next trigger decision. +pub(crate) fn note_young_leaf_born_old() { + GC_YOUNG_LEAF_BORN_OLD.with(|flag| flag.set(true)); +} + +pub(super) fn gc_budgeted_due_trigger() -> Option { + // #10169: a leaf born old under young pressure gives the nursery minor + // ONE-TIME priority over old-reclaim, and only while the young generation + // is still unmeasured. A young generation that a minor has already + // measured as retained wholesale is either promoted (so the next leaf + // finds it small) or, when it dies at every loop edge, best left to the + // old-reclaim full that sweeps it in Eden together with the leaf. The one + // case that must not fall through is an unmeasured young generation that + // stays live: old-reclaim would re-mark it in place at every full, so it + // is promoted by a minor first. The flag is consumed here whatever the + // decision, so it can never starve old-reclaim. + if GC_YOUNG_LEAF_BORN_OLD.with(Cell::get) { + GC_YOUNG_LEAF_BORN_OLD.with(|flag| flag.set(false)); + if !super::young_generation_measured_retained() && young_scavenge_cap_due() { + return Some(BudgetedGcTrigger::YoungScavengeCap); + } + } let old_pending = GC_OLD_RECLAIM_PENDING.with(Cell::get); // #6010: external Map/Set side-buffer bytes escalate to OldReclaim too. let old_in_use = diff --git a/crates/perry-runtime/src/gc/promote_in_place.rs b/crates/perry-runtime/src/gc/promote_in_place.rs index 9e36e15ad0..5f2fe7d5b0 100644 --- a/crates/perry-runtime/src/gc/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/promote_in_place.rs @@ -577,6 +577,27 @@ pub(crate) fn last_young_survival_permille() -> Option { LAST_YOUNG_SURVIVAL_PERMILLE.with(Cell::get) } +/// #10169: did the previous copying minor measure the young generation as +/// retained wholesale? This is the in-place promotion signal read directly, +/// without the test-build opt-in that gates the promotion itself, so a +/// scheduling decision elsewhere can key on the measurement alone. `None` — +/// no copying minor has run on this thread — is deliberately `false`. +pub(crate) fn young_generation_measured_retained() -> bool { + LAST_YOUNG_SURVIVAL_PERMILLE + .with(Cell::get) + .is_some_and(|permille| permille >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE) +} + +/// #10169: the complement that is NOT `!measured_retained`: the previous +/// copying minor measured the young generation as mostly garbage. `None` — no +/// measurement yet — is `false` here too, so an unmeasured young generation +/// is neither retained nor dying. +pub(crate) fn young_generation_measured_dying() -> bool { + LAST_YOUNG_SURVIVAL_PERMILLE + .with(Cell::get) + .is_some_and(|permille| permille < PROMOTE_SURVIVAL_THRESHOLD_PERMILLE) +} + #[cfg(test)] pub(crate) fn promoted_dead_bytes_since_full() -> usize { PROMOTED_DEAD_BYTES.with(Cell::get) @@ -644,12 +665,12 @@ impl Drop for InPlacePromotionTestGuard { /// MEASUREMENT of "almost nothing survived", and only this exercises the /// `None` arm of the decision. #[cfg(test)] -pub(super) fn clear_young_survival_for_tests() { +pub(crate) fn clear_young_survival_for_tests() { LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(None)); } #[cfg(test)] -pub(super) fn seed_young_survival_for_tests(permille: u64) { +pub(crate) fn seed_young_survival_for_tests(permille: u64) { LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(permille))); } diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 85f22246c4..635f3af432 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -74,4 +74,5 @@ mod triggers; mod typed_layout_intact_residual; mod u8_inline_cache; mod weak_read_barrier; +mod young_leaf_route; mod young_log_tests; diff --git a/crates/perry-runtime/src/gc/tests/young_leaf_route.rs b/crates/perry-runtime/src/gc/tests/young_leaf_route.rs new file mode 100644 index 0000000000..2865f520b5 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/young_leaf_route.rs @@ -0,0 +1,67 @@ +//! #10169: a document-sized JSON leaf born old under young pressure gives the +//! nursery minor one-time priority over old-reclaim, and only while the young +//! generation is unmeasured. Both halves are asserted: the priority fires +//! exactly once per leaf, and a measured young generation buys none. + +use super::super::policy::{ + gc_budgeted_due_trigger, note_young_leaf_born_old, BudgetedGcTrigger, + ScavengeNurseryCapTestGuard, GC_OLD_RECLAIM_PENDING, +}; +use super::super::*; +use super::support::*; + +#[test] +fn young_leaf_born_old_prioritises_the_nursery_minor_until_measured() { + let _isolation = GcTestIsolationGuard::new(); + let _pacing = crate::gc::policy::force_moving_gc_pacing(); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _cap_due = ScavengeNurseryCapTestGuard::due_at_bytes(1); + // The isolated arena starts empty; one young allocation makes "due at one + // byte" actually due. + let filler = [b'y'; 64]; + crate::string::js_string_from_bytes(filler.as_ptr(), filler.len() as u32); + assert!( + crate::arena::copying_from_space_in_use_bytes() >= 1, + "fixture: the young generation must hold something for the cap to be due" + ); + let previous_survival = last_young_survival_permille(); + GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(true)); + assert_eq!( + gc_budgeted_due_trigger(), + Some(BudgetedGcTrigger::OldReclaim), + "fixture: old-reclaim must be due before the leaf can outrank it" + ); + + // Unmeasured young generation: the leaf buys the nursery minor exactly one + // decision, then old-reclaim is back. + clear_young_survival_for_tests(); + note_young_leaf_born_old(); + assert_eq!( + gc_budgeted_due_trigger(), + Some(BudgetedGcTrigger::YoungScavengeCap) + ); + assert_eq!( + gc_budgeted_due_trigger(), + Some(BudgetedGcTrigger::OldReclaim) + ); + + // Measured as retained: no priority, and the flag is still consumed. + seed_young_survival_for_tests(999); + note_young_leaf_born_old(); + assert_eq!( + gc_budgeted_due_trigger(), + Some(BudgetedGcTrigger::OldReclaim) + ); + clear_young_survival_for_tests(); + assert_eq!( + gc_budgeted_due_trigger(), + Some(BudgetedGcTrigger::OldReclaim), + "a consumed flag must not be honoured later" + ); + + GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false)); + match previous_survival { + Some(permille) => seed_young_survival_for_tests(permille), + None => clear_young_survival_for_tests(), + } +} diff --git a/crates/perry-runtime/src/json/mod.rs b/crates/perry-runtime/src/json/mod.rs index a050ed3dfd..82f3cd309e 100644 --- a/crates/perry-runtime/src/json/mod.rs +++ b/crates/perry-runtime/src/json/mod.rs @@ -604,13 +604,15 @@ pub(crate) fn json_string_from_native_output_bytes(bytes: &[u8]) -> *mut StringH crate::string::compute_utf16_len(bytes.as_ptr(), len) }; stringify_flat::service_json_output_sweep_boundary(); - let (ptr, data) = crate::string::json_output_storage_alloc(len); + let (ptr, data, malloc_tracked) = crate::string::json_output_storage_alloc(len); unsafe { crate::string::init_string_header(ptr, utf16_len, len, len, 0, 0); // GC_STORE_AUDIT(POINTER_FREE): completed JSON payload bytes. std::ptr::copy_nonoverlapping(bytes.as_ptr(), data, len as usize); } - stringify_flat::note_completed_malloc_json_output(len); + if malloc_tracked { + stringify_flat::note_completed_malloc_json_output(len); + } ptr } diff --git a/crates/perry-runtime/src/json/stringify_flat.rs b/crates/perry-runtime/src/json/stringify_flat.rs index 08b688fbee..84dbf69b85 100644 --- a/crates/perry-runtime/src/json/stringify_flat.rs +++ b/crates/perry-runtime/src/json/stringify_flat.rs @@ -416,7 +416,7 @@ unsafe fn emit_two_field_parsed_string_object( let large_output = bytes >= JSON_MALLOC_OUTPUT_THRESHOLD; let construction = large_output.then(crate::gc::GcSuppressScope::new); - let (result, output) = json_output_storage_alloc(bytes); + let (result, output, malloc_tracked) = json_output_storage_alloc(bytes); let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| { let keys = crate::object::object_keys_array(obj); init_string_header(result, units, bytes, bytes, 0, 0); @@ -449,7 +449,7 @@ unsafe fn emit_two_field_parsed_string_object( Some(JSValue::string_ptr(result)) }); drop(construction); - if large_output { + if malloc_tracked { note_completed_malloc_json_output(bytes); } value diff --git a/crates/perry-runtime/src/json/stringify_record_output.rs b/crates/perry-runtime/src/json/stringify_record_output.rs index 582fb35cc2..889b9429f1 100644 --- a/crates/perry-runtime/src/json/stringify_record_output.rs +++ b/crates/perry-runtime/src/json/stringify_record_output.rs @@ -460,7 +460,7 @@ unsafe fn emit_cached_record_uncached( return None; } let construction = large_output.then(crate::gc::GcSuppressScope::new); - let (result, output) = json_output_storage_alloc(bytes); + let (result, output, malloc_tracked) = json_output_storage_alloc(bytes); init_string_header(result, units, bytes, bytes, 0, 0); let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| { let mut at = 0usize; @@ -505,7 +505,7 @@ unsafe fn emit_cached_record_uncached( Some(JSValue::string_ptr(result)) }); drop(construction); - if large_output { + if malloc_tracked { super::stringify_flat::note_completed_malloc_json_output(bytes); } value @@ -562,7 +562,7 @@ unsafe fn emit_cached_record_memo( } let repeated_candidate = bytes as usize <= MAX_REPEATED_OUTPUT_BYTES; let construction = large_output.then(crate::gc::GcSuppressScope::new); - let (result, output) = json_output_storage_alloc(bytes); + let (result, output, malloc_tracked) = json_output_storage_alloc(bytes); init_string_header(result, units, bytes, bytes, 0, 0); let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| { let mut at = 0usize; @@ -665,7 +665,7 @@ unsafe fn emit_cached_record_memo( Some(JSValue::string_ptr(result)) }); drop(construction); - if large_output { + if malloc_tracked { super::stringify_flat::note_completed_malloc_json_output(bytes); } value @@ -745,7 +745,7 @@ unsafe fn emit_record(obj: *const crate::ObjectHeader, fields: usize) -> Option< return None; } let construction = large_output.then(crate::gc::GcSuppressScope::new); - let (result, output) = json_output_storage_alloc(bytes); + let (result, output, malloc_tracked) = json_output_storage_alloc(bytes); init_string_header(result, units, bytes, bytes, 0, 0); let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| { let keys = crate::object::object_keys_array(obj); @@ -797,7 +797,7 @@ unsafe fn emit_record(obj: *const crate::ObjectHeader, fields: usize) -> Option< Some(JSValue::string_ptr(result)) }); drop(construction); - if large_output { + if malloc_tracked { super::stringify_flat::note_completed_malloc_json_output(bytes); } value diff --git a/crates/perry-runtime/src/json/stringify_string_tests.rs b/crates/perry-runtime/src/json/stringify_string_tests.rs index 7806aa07da..bc3a14b0b5 100644 --- a/crates/perry-runtime/src/json/stringify_string_tests.rs +++ b/crates/perry-runtime/src/json/stringify_string_tests.rs @@ -98,3 +98,63 @@ fn direct_quoted_raw_strings_preserve_fallback_length_semantics() { } } } + +/// #10169: a large leaf stays malloc-tracked while the young generation is +/// smaller than the leaf or was measured as mostly garbage; it is born old in +/// the arena when the young generation holds at least the leaf's own bytes +/// and is unmeasured or measured as retained — the shape of a freshly parsed +/// document whose result the caller is about to stringify. +#[test] +fn large_json_leaf_routes_by_young_generation_occupancy() { + if !crate::gc::gen_gc_enabled() { + return; + } + let previous_survival = crate::gc::last_young_survival_permille(); + let _suppress = crate::gc::GcSuppressScope::new(); + let young_before = crate::arena::copying_from_space_in_use_bytes(); + let leaf = (young_before as u32 + (1 << 20)).max(crate::string::JSON_MALLOC_OUTPUT_THRESHOLD); + crate::gc::seed_young_survival_for_tests(999); + let (_, _, tracked_below) = crate::string::json_output_storage_alloc(leaf); + assert!( + tracked_below, + "a young generation smaller than the leaf keeps malloc tracking" + ); + + let filler = vec![b'y'; 1024]; + while crate::arena::copying_from_space_in_use_bytes() < leaf as usize { + crate::string::js_string_from_bytes(filler.as_ptr(), filler.len() as u32); + } + if !crate::gc::young_generation_holds_a_nursery() { + let (_, _, tracked_below_nursery) = crate::string::json_output_storage_alloc(leaf); + assert!( + tracked_below_nursery, + "a young generation below one nursery keeps malloc tracking" + ); + while !crate::gc::young_generation_holds_a_nursery() { + crate::string::js_string_from_bytes(filler.as_ptr(), filler.len() as u32); + } + } + crate::gc::clear_young_survival_for_tests(); + let (_, _, tracked_unmeasured) = crate::string::json_output_storage_alloc(leaf); + assert!( + !tracked_unmeasured, + "an unmeasured young generation at or above the leaf size births the leaf in the arena" + ); + crate::gc::seed_young_survival_for_tests(100); + let (_, _, tracked_dying) = crate::string::json_output_storage_alloc(leaf); + assert!( + tracked_dying, + "a young generation measured as mostly garbage keeps malloc tracking" + ); + crate::gc::seed_young_survival_for_tests(999); + let (_, _, tracked_retained) = crate::string::json_output_storage_alloc(leaf); + assert!( + !tracked_retained, + "a retained young generation at or above the leaf size births the leaf in the arena" + ); + + match previous_survival { + Some(permille) => crate::gc::seed_young_survival_for_tests(permille), + None => crate::gc::clear_young_survival_for_tests(), + } +} diff --git a/crates/perry-runtime/src/string/json_construction.rs b/crates/perry-runtime/src/string/json_construction.rs index f8770e8f8c..d3a1ec861b 100644 --- a/crates/perry-runtime/src/string/json_construction.rs +++ b/crates/perry-runtime/src/string/json_construction.rs @@ -23,7 +23,8 @@ pub(crate) unsafe fn string_from_json_bytes( }; let (header, data) = if raw.is_null() { if large_json_leaf { - json_output_storage_alloc(len) + let (header, data, _malloc_tracked) = json_output_storage_alloc(len); + (header, data) } else { string_storage_alloc(len) } diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index fceb76cd36..9e2ebac193 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -682,23 +682,62 @@ pub(crate) fn string_storage_alloc(capacity: u32) -> (*mut StringHeader, *mut u8 /// JSON results at or above this size use individually tracked storage. pub(crate) const JSON_MALLOC_OUTPUT_THRESHOLD: u32 = 512 * 1024; -/// Allocate a large, pointer-free JSON result outside old-generation arenas. +/// Allocate a large, pointer-free JSON result. The flag reports whether it is +/// malloc-tracked (`true`) or born old in the arena (`false`), so callers +/// charge malloc-output debt only for tracked leaves. /// /// Ordinary large strings are born old because copying them through survivor /// space is wasteful. Repeated `JSON.stringify` is different: each result is a /// leaf commonly discarded at the next loop edge. Tracking that leaf as an /// individual malloc object lets the next minor sweep reclaim it without a /// whole-old-heap trace. Smaller results retain the arena fast path. +/// +/// #10169: that trade inverts when the young generation already holds at +/// least as many bytes as the leaf — typically a freshly parsed document the +/// caller is still using — unless the previous minor measured the young +/// generation as mostly garbage. A non-empty malloc registry forbids the +/// untraced in-place promotion (`skip_remembering` needs it empty), so the +/// next minor would trace that whole tree — 55 ms for a 20 MB document — to +/// reclaim one leaf; and a single stale tracked leaf keeps vetoing it until +/// swept. Such a leaf is born old in the arena instead, and the collector is +/// told (`gc::note_young_leaf_born_old`) so that an UNMEASURED young +/// generation gets a minor before old-reclaim can re-mark it in place: a +/// stringify-only loop over one parsed input thus has its input promoted +/// once, after which the young generation is too small for this route, while +/// a parse/stringify loop's transient tree dies in Eden under the old-reclaim +/// full that also reclaims the leaf. #[inline] -pub(crate) fn json_output_storage_alloc(capacity: u32) -> (*mut StringHeader, *mut u8) { +pub(crate) fn json_output_storage_alloc(capacity: u32) -> (*mut StringHeader, *mut u8, bool) { if capacity < JSON_MALLOC_OUTPUT_THRESHOLD { - return string_storage_alloc(capacity); + let (ptr, data) = string_storage_alloc(capacity); + return (ptr, data, false); + } + if json_leaf_prefers_arena(capacity) { + let (ptr, data) = string_storage_alloc(capacity); + crate::gc::note_young_leaf_born_old(); + return (ptr, data, false); } let payload_size = std::mem::size_of::() + capacity as usize; let raw = crate::gc::gc_malloc(payload_size, crate::gc::GC_TYPE_STRING); let ptr = raw as *mut StringHeader; let data = unsafe { raw.add(std::mem::size_of::()) }; - (ptr, data) + (ptr, data, true) +} + +/// The young generation holds at least the leaf's own bytes and at least one +/// nursery's worth of data (so a minor is what comes next, which is the only +/// time tracking the leaf costs a whole-young trace), and was not measured as +/// dying. Below a nursery the malloc path stays: a loop that stringifies a +/// small parsed input into document-sized results never collects at all, and +/// born-old results would only trade malloc's recycled pages for fresh arena +/// pages (measured: +17 % CPU on a 1 MB stringify loop). See +/// [`json_output_storage_alloc`]. +#[inline] +pub(crate) fn json_leaf_prefers_arena(capacity: u32) -> bool { + crate::gc::gen_gc_enabled() + && crate::arena::copying_from_space_in_use_bytes() >= capacity as usize + && crate::gc::young_generation_holds_a_nursery() + && !crate::gc::young_generation_measured_dying() } /// Maximum number of UTF-16 code units in one Perry string. Mirrors V8's diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index c7498b1d77..6bc9927516 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -307,7 +307,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -324,8 +324,8 @@ "sources": { "crates/perry-runtime/src/gc/census.rs": "3f9e6be47b4454022b70ff2357bbdf3a80ef6a84c4986e58763acbdcce9142c1", "crates/perry-runtime/src/gc/cycle.rs": "77eadaf7c4157308c3b14be5e1ff11d7198b84b503a50ff2d244851248d3e800", - "crates/perry-runtime/src/gc/mod.rs": "496d42cf8f41e4c71da6d50470ec1e796b065a919549b8239f579a5bc1a5b175", - "crates/perry-runtime/src/gc/policy.rs": "a701257f2e2310adabe16e33c0afcd935c7cd28e1ddc157b4974b48e2688cc4f", + "crates/perry-runtime/src/gc/mod.rs": "5b0c6dcf8ad8b919e86458f69ef4ad24679e6c7c08eeacf6629439b4e9ff0177", + "crates/perry-runtime/src/gc/policy.rs": "aea89274a017156efea3516b4f49124f48a66527a08195b662cfbf61672ac042", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } @@ -402,6 +402,12 @@ "verdict": "not_a_gc_pointer", "why": "Tiny-JSON completion countdown in a Cell. It stores only the number of bounded parse completions before the next arena-pressure poll (0..63), never an address or a NaN-boxed value." }, + { + "file": "crates/perry-runtime/src/gc/policy.rs", + "name": "GC_YOUNG_LEAF_BORN_OLD", + "verdict": "not_a_gc_pointer", + "why": "#10169: a `Cell` scheduling flag — set when a document-sized JSON result was born old under young pressure, consumed by the next `gc_budgeted_due_trigger` decision. Holds no address of any kind." + }, { "file": "crates/perry-runtime/src/gc/policy.rs", "name": "GC_TINY_PARSE_PRESSURE_BASE_BYTES",