diff --git a/.gitignore b/.gitignore index 9de71d3..9ab031a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,9 @@ tests/esp32p4/embed/ tests/esp32p4/fw.elf tests/esp32p4/*.log contrib/esp32p4/smoke_test.elf +contrib/esp32p4/idf-benchmark/main/embed/ +contrib/esp32p4/idf-benchmark/main/embed_list.inc +contrib/esp32p4/idf-benchmark/main/embed_includes.inc +contrib/esp32p4/idf-benchmark/build/ +contrib/esp32p4/idf-benchmark/sdkconfig +contrib/esp32p4/idf-benchmark/sdkconfig.old diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c3cb36..9ea3c90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,12 @@ if (WITH_SYSTEM_ZLIB) else() if(NOT TARGET miniz) add_subdirectory(deps/miniz-3.1.2 EXCLUDE_FROM_ALL) + # ESP ROMs export an older miniz that would otherwise capture this one + # at link time; applied here rather than inside deps/ so a miniz bump + # can't drop it. No-op off ESP-IDF. + include(${CMAKE_CURRENT_LIST_DIR}/cmake/EspRomMinizWorkaround.cmake) + libchdr_apply_esp_rom_miniz_workaround(miniz) + set(CHDR_NEEDS_MINIZ_RENAME TRUE) endif() list(APPEND CHDR_LIBS miniz) endif() @@ -88,6 +94,17 @@ else() endif() if(CHDR_VERIFY_BLOCK_CRC) + # The per-hunk CRC chdman stores covers the fully reconstituted hunk, ECC and + # sync header included, so a build that skips ecc_generate() cannot match it. + # The failure is content-dependent - hunks holding only audio frames have no + # ECC to regenerate and still verify - which makes it look like sporadic file + # corruption rather than a build misconfiguration. Refuse the combination. + if(NOT CHDR_WANT_RAW_DATA_SECTOR) + message(FATAL_ERROR + "CHDR_VERIFY_BLOCK_CRC=ON requires CHDR_WANT_RAW_DATA_SECTOR=ON: without the " + "regenerated ECC and sync header the decoded hunk cannot match the CRC stored " + "in the file. Turn CHDR_VERIFY_BLOCK_CRC off as well to build without raw sectors.") + endif() list(APPEND CHDR_DEFINES VERIFY_BLOCK_CRC=1) else() list(APPEND CHDR_DEFINES VERIFY_BLOCK_CRC=0) @@ -124,6 +141,12 @@ set(CHDR_SOURCES add_library(chdr-static STATIC ${CHDR_SOURCES}) target_include_directories(chdr-static INTERFACE include) target_link_libraries(chdr-static PRIVATE ${CHDR_LIBS} ${PLATFORM_LIBS}) +# libchdr_codec_zlib.c calls tinfl_decompress() directly, so it must be renamed +# alongside miniz's definition - otherwise on ESP-IDF this call binds to the +# ROM's older copy and we are back to the split decoder this branch fixed. +if(CHDR_NEEDS_MINIZ_RENAME) + libchdr_apply_esp_rom_miniz_workaround(chdr-static) +endif() target_compile_definitions(chdr-static PRIVATE ${CHDR_DEFINES}) if(MSVC) @@ -140,6 +163,9 @@ if (BUILD_SHARED_LIBS) add_library(chdr SHARED ${CHDR_SOURCES}) target_include_directories(chdr INTERFACE include) target_link_libraries(chdr PRIVATE ${CHDR_LIBS} ${PLATFORM_LIBS}) + if(CHDR_NEEDS_MINIZ_RENAME) + libchdr_apply_esp_rom_miniz_workaround(chdr) + endif() target_compile_definitions(chdr PRIVATE ${CHDR_DEFINES}) if(MSVC) diff --git a/cmake/EspRomMinizWorkaround.cmake b/cmake/EspRomMinizWorkaround.cmake new file mode 100644 index 0000000..713c96e --- /dev/null +++ b/cmake/EspRomMinizWorkaround.cmake @@ -0,0 +1,70 @@ +# Work around Espressif ROMs exporting their own, older miniz. +# +# ESP32 ROMs (S3/C3/C6/P4/...) bake in an older miniz and export its tinfl +# entry points from the target's ROM linker script as *absolute* symbols - +# see the "Group miniz" block in +# $IDF_PATH/components/esp_rom//ld/.rom.ld, e.g. +# +# tinfl_decompress = 0x4fc000f8; +# +# A linker-script assignment outranks an ordinary object definition, so an +# ESP-IDF link silently binds those names to ROM and drops the copies +# compiled from deps/miniz-3.1.2/miniz.c - even though both are present in +# the archive. The result is a *split decoder*: mz_inflateInit2()/mz_inflate() +# from miniz 3.1.2 build and interpret a 3.1.2-layout tinfl_decompressor, then +# hand it to a ROM tinfl_decompress() that lays that struct out differently +# (miniz 3.0 reworked the Huffman tables from tinfl_huff_table m_tables[3] to +# the flattened m_look_up/m_tree_N form, changing field offsets and total +# size). The ROM decoder writes past the end of the smaller m_decomp and +# corrupts the enclosing inflate_state. +# +# Observed on an ESP32-P4 (rev v3.1) against a 128-file CHD corpus: the first +# inflate of a stream mostly survives, then every later one fails, because the +# overrun lands on inflate_state::m_window_bits (which sits just before +# m_dict[32768]). mz_inflate() then sees m_window_bits > 0, sets +# TINFL_FLAG_PARSE_ZLIB_HEADER on a raw-deflate stream opened with +# inflateInit2(..., -MAX_WBITS), consumes exactly 2 bytes on the CMF/FLG check +# and returns MZ_DATA_ERROR. It presents as CHDERR_DECOMPRESSION_ERROR and +# looks exactly like corrupt input or a silicon/codegen bug. +# +# Renaming the colliding symbols keeps miniz.c's own definitions reachable. +# Only miniz.c references these names, so applying the defines to whatever +# target compiles miniz.c is sufficient. mz_free matters independently of the +# decoder mismatch: bound to ROM it would hand ESP-IDF-heap pointers to the +# ROM allocator. mz_adler32 is benign but renamed for consistency. +# +# Deliberately NOT patched into deps/miniz-3.1.2/miniz.h - that tree is +# vendored verbatim so it can be re-synced from upstream, and a local edit +# there would be silently dropped by the next version bump. Keep this file as +# the single definition; both build paths below include it. +# +# Regression check (cheap, no flashing) - this must print nothing: +# +# grep -hoE '^[A-Za-z_][A-Za-z0-9_]* = 0x' \ +# "$IDF_PATH"/components/esp_rom//ld/.rom*.ld \ +# | sed 's/ = 0x//' | sort -u > /tmp/rom_syms.txt +# -nm \ +# | awk '$2 ~ /^[TDBR]$/ {print $3}' | sort -u > /tmp/chdr_syms.txt +# comm -12 /tmp/rom_syms.txt /tmp/chdr_syms.txt +# +# Re-run it after any miniz bump, and after adding any dep the ROM also +# ships - the rom.ld files list them by group. + +set(LIBCHDR_ESP_ROM_MINIZ_COLLISIONS + tinfl_decompress + tinfl_decompress_mem_to_heap + tinfl_decompress_mem_to_mem + tinfl_decompress_mem_to_callback + mz_adler32 + mz_free +) + +# Apply the renames to a target that compiles miniz.c. No-op off ESP-IDF. +function(libchdr_apply_esp_rom_miniz_workaround target) + if(NOT ESP_PLATFORM) + return() + endif() + foreach(sym IN LISTS LIBCHDR_ESP_ROM_MINIZ_COLLISIONS) + target_compile_definitions(${target} PRIVATE "${sym}=libchdr_${sym}") + endforeach() +endfunction() diff --git a/contrib/esp32p4/idf-benchmark/CMakeLists.txt b/contrib/esp32p4/idf-benchmark/CMakeLists.txt new file mode 100644 index 0000000..54979cb --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/CMakeLists.txt @@ -0,0 +1,3 @@ +cmake_minimum_required(VERSION 3.16) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(libchdr_esp32p4_benchmark) diff --git a/contrib/esp32p4/idf-benchmark/README.md b/contrib/esp32p4/idf-benchmark/README.md new file mode 100644 index 0000000..068bd33 --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/README.md @@ -0,0 +1,497 @@ +# ESP32-P4 real-hardware throughput benchmark + +A real ESP-IDF application (not the compile+link smoke test in +`contrib/esp32p4/smoke_test.c`) that actually runs on a physical ESP32-P4 +board, opens and fully decodes a CHD corpus embedded in flash, and reports +per-file and aggregate decode throughput over the USB serial console. + +Built and run against a real **Waveshare ESP32-P4-NANO** (chip revision +v3.1, 16MB flash, 32MB PSRAM - PSRAM unused here, everything runs from +internal SRAM). + +## Corpus + +`gen_embed.sh` generates `main/embed/*.h` (gitignored, regenerated on +demand) from: + +- `tests/corpus/seeds/*.chd` - the full 17-file synthetic corpus (one file + per codec/order combination) also used by `lowram-correctness.yml`. +- `tests/avhuff_corpus/{regtest,synth}/*.chd` - the 4-file AVHuff corpus. +- Up to 3 small real MAME-derived CHDs pulled from an actual game library + (`REAL_CHD_DIR`, default `/userdata/roms/pcenginecd`) - optional, skipped + if not present, kept small (<2MB each) so they fit in flash without a + custom partition layout beyond `partitions.csv`'s 6MB app slot. + +24 files, ~2.4MB of embedded CHD data total. + +## Building and running + +```sh +git clone --recursive -b v5.5.5 https://github.com/espressif/esp-idf.git ~/esp/esp-idf +~/esp/esp-idf/install.sh esp32p4 +source ~/esp/esp-idf/export.sh + +cd contrib/esp32p4/idf-benchmark +./gen_embed.sh +idf.py set-target esp32p4 +idf.py -p /dev/ttyACM0 build flash monitor +``` + +**Must be ESP-IDF v5.5.3 or later** (or v6.0+) if your board's ESP32-P4 is +chip revision v3.x - earlier ESP-IDF versions' bootloader rejects it +outright (`bootloader/bootloader.bin requires chip revision in range +[v0.1 - v1.99]`). v3.x silicon changed over 50 registers vs earlier +revisions; see +[Espressif's chip revision v3.x guide](https://documentation.espressif.com/esp32-p4-chip-revision-v3.x_user_guide_en.html). + +## Results (2026-09-02, real hardware) + +``` +=== libchdr ESP32-P4 real-hardware throughput benchmark === +free heap: 587264 bytes (largest block: 524288) +``` + +Two corpora are run back to back: the flash-embedded synthetic one (below) +and a real SD card holding 128 CHDs from a 43GB retro ROM set, read through +the real FATFS/SDMMC stack - the actual ESP32-P4-NANO deployment path. + +This build sets **`LOWRAM_TARGET=1`**, which turns out to matter enormously +on this class of target - see "LOWRAM_TARGET is not optional here" below. + +**Flash corpus: 24/24 files** decode and CRC-verify correctly +(`VERIFY_BLOCK_CRC=1`), AVHuff included. + +**SD corpus: 112/128 files** - i.e. **every valid file in the set**: + +| outcome | files | what it is | +|---|---|---| +| decoded + CRC-verified | 112 | all 112 non-broken CHDs on the card | +| `OPEN FAILED: invalid file` | 13 | **0-byte files** in the source ROM set | +| `OPEN FAILED: read error` | 3 | **truncated files** - header's `mapoffset` points past EOF | + +Zero decompression errors, zero OOM, one reset in the whole run (the +initial power-on). The 16 failures are broken files, not a target +limitation: they fail identically on desktop x86-64, and libchdr rejects +them with the correct error. Verified directly rather than assumed - +`stat` shows 13 at size 0, and the other 3 have a `mapoffset` beyond their +own file length. + +Representative throughput (400MHz): + +| file | codec | hunkbytes | out MB/s | +|---|---|---|---| +| cd_none | uncompressed | 19584 | 16.4 | +| cd_cdzl / cd_cdzs | CD zlib/zstd | 19584 | ~9.2 | +| cd_cdlz | CD LZMA | 19584 | 7.8 | +| cd_cdfl / cd_default | CD FLAC(+subcode) | 19584 | ~3.4 | +| hd_lzma | LZMA | 4096 | 9.9 | +| hd_zstd | zstd | 4096 | 18.3 | +| hd_huff | huffman | 4096 | 3.9 | +| real_Hawiian_Island_Girls (147 real CD hunks) | mixed | 19584 | 2.3 | + +Same board, same card, across the two changes that mattered: + +| | flash corpus | SD corpus | decompression errors | OOM | +|---|---|---|---|---| +| baseline | 17/24, 4.78 MB out | 18/128 | 16 | 81 | +| + ROM-collision fix | 20/24, 10.63 MB out | 31/128 | **0** | 81 | +| + `LOWRAM_TARGET=1` | **24/24**, 14.16 MB out | **112/128** | **0** | **0** | + +Read the SD rows carefully: the baseline run was uncapped, the later two +cap each file at `SD_MAX_HUNKS_PER_FILE` (600). The cap was added *because* +of the fix - files that used to bail out at hunk 0 now decode in full, and +an uncapped 128-file sweep of a 43GB set runs for hours. A cap can only +ever hide failures **past** hunk 600, and all 16 pre-fix failures occurred +at hunks 0-272, so it cannot be manufacturing the "0 decompression errors" +result. The flash corpus is uncapped in all three rows and is the +like-for-like comparison. + +## `LOWRAM_TARGET` is not optional here - and it had a correctness bug + +With `LOWRAM_TARGET=0` this board looks far more limited than it is: 76 of +128 SD files failed at `chd_open()` with `CHDERR_OUT_OF_MEMORY`, plus 3 +CD-FLAC and 2 AVHuff failures further in. The boundary was sharp - the +largest file that opened had a 164KB rawmap, the smallest that failed +needed 176KB, against 582KB free / 524KB largest block. Sizes ran to a +3.1MB rawmap, which cannot fit in this part's SRAM at all under an eager +map. + +`LOWRAM_TARGET=1` takes all of that to zero: lazy CHDv5 map decode, +`compressed` grown on demand, and codec `init()` deferred until a hunk +needs that slot. Measured heap cost, once the instrumentation was fixed: + +| content | open cost | libchdr resident | +|---|---|---| +| CD (19584B hunks) | **5 KB** | ~6 KB | +| HD (4096B hunks) | **5 KB** | ~5 KB | +| AVHuff (223668B hunks) | 5 KB | ~7 KB | + +**Opening a 271328-hunk CHD costs 5KB**, and libchdr's resident footprint +is 5-7KB regardless of file size. Hunk count no longer drives RAM at all; +the real constraint is the caller's own hunk buffer. + +It is also close to free on throughput: across the 20 flash-corpus files +that pass in both configurations - uncapped, identical content - total +decode time is 3826ms versus 3822ms, or 1.00x. + +**But it selects a second-level huffman lookup table that the default build +never uses, and that path had a bug** (fixed; see the commit resetting the +huffman subtable arena). `huffman_build_lookup_table()` rebuilt the lookup +from scratch on every call without resetting `subtable_count`, so the count +grew monotonically across hunks until it tripped a 2048 guard, after which +every subsequent huffman hunk failed. It also leaked up to 256KB - in the +configuration whose entire purpose is saving memory. + +Two things about how that was nearly missed are worth recording: + +- It is only reachable with `LOWRAM_TARGET=1` **and** on CHDs that use + `CHD_CODEC_HUFFMAN`, which is rare - 0.2% of hunks in the file that + exposed it. +- The sweep that reported "112/128 files, zero decompression errors" was + **capped at 600 hunks per file**. These files fail ~20000 hunks in. The + cap introduced to make the sweep tractable is exactly what hid it. + +Treat `LOWRAM_TARGET=1` as the default for any target in this memory class - +with that fix applied. + +## FLAC: the cost is the decode, not the per-hunk rebuild + +`flac_decoder_reset()` calls `drflac_open_with_metadata()` once per hunk - +a full decoder teardown and rebuild, ~40KB allocation, STREAMINFO reparse - +where `cdlz` simply reuses its LZMA state. cdfl costs about 3.6x cdlz per +hunk at identical geometry, so the obvious conclusion was that the rebuild +explained it. + +**It does not.** `CHDR_PROFILE_CDFL` splits the three stages, measured on +hardware: + +| stage | share | per hunk | +|---|---|---| +| `flac_reset` (the rebuild) | 0.7 - 7.9% | ~0.03 ms | +| `flac_decode` | **58.6 - 95.6%** | 0.245 - 3.622 ms | +| `subcode` (zlib) | 3.7 - 33.5% | ~0.14 ms | + +The rebuild is about 1% of cdfl's time. The cost is genuine FLAC decoding. +Reusing the drflac instance across hunks is therefore a **memory** +optimisation, not a throughput one - worth doing, because that 40KB +per-hunk allocation is what produced `CHDERR_OUT_OF_MEMORY` on the three +largest pcenginecd titles under the eager map, but it will not make cdfl +meaningfully faster. + +Recorded because the wrong version of this conclusion is very easy to +reach: the rebuild is real, it is obviously wasteful, and it sits directly +in the hot path. Only measurement separates "wasteful" from "expensive". + +For reference, per-codec cost at identical geometry (10 hunks of 19584 +bytes, RAM-sourced, so no I/O in the number): + +| codec | ms/hunk | minus `cd_none` baseline | +|---|---|---| +| `cd_none` | 1.218 | - | +| `cd_cdzs` | 1.901 | 0.683 | +| `cd_cdzl` | 1.945 | 0.727 | +| `cd_cdlz` | 2.303 | 1.085 | +| `cd_cdfl` | 5.087 | **3.869** | + +## The storage stack costs 8.4x, and most of it is avoidable + +Layering the same card at the same 40MHz clock three ways: + +| path | 4 KB | 32 KB | 64 KB | 256 KB | +|---|---|---|---|---| +| `sdmmc_read_sectors()` (driver) | 11.28 | 16.49 | 17.80 | **18.96 MB/s** | +| `f_read()` (FatFs direct) | 9.71 | 10.59 | 10.58 | - | +| `fread()` (VFS + newlib stdio) | 2.30 | 2.30 | 2.30 | 2.30 MB/s | + +The card sustains ~19 MB/s and scales with transfer size. FatFs costs 1.8x. +**The VFS and newlib stdio wrapper above it costs another 4.6x**, and delivers +a flat 2.30 MB/s no matter how large the request is. Halving the SD clock to +20MHz costs only 13%, so the bus is not the constraint either. + +libchdr never has to care. `core_file_callbacks` already lets the embedder +supply any reader, so a FatFs-backed backend is a drop-in replacement for the +stdio one and **needs no change to the library**. + +### Two things the backend must do + +Going straight to FatFs skips ESP-IDF's VFS - and the VFS is what normally +allocates `cltbl` and runs `f_lseek(CREATE_LINKMAP)` (`vfs_fat.c`). Without +that, **the backend silently has no fast seek**, and since every +`COMPRESSION_SELF` reference is a backward seek, FatFs restarts its +cluster-chain walk from the first cluster on each one. On a 271328-hunk file +that is 29.2% self-references, an uncapped sweep was still on that single file +after 47 minutes. Build the cluster link map at open; see `fffile_open()`. + +That failure is invisible to a short run: near the start of a file the chain +walks are cheap, so a 3000-hunk comparison shows the backend as a clean win +while an uncapped one hangs. + +### Measured, full 14-file sample, uncapped, 7.32 GB each + +| configuration | time | throughput | vs stdio | +|---|---|---|---| +| stdio backend | 3641 s | 2.01 MB/s | - | +| FatFs + cluster map | 2549 s | 2.87 MB/s | **1.43x** | +| FatFs + cluster map + 64KB read-ahead | **2307 s** | **3.17 MB/s** | **1.58x** | + +14/14 files decode and CRC-verify in every configuration. + +Per file, and the gain tracks how I/O-bound each one was: + +| file | stdio | FatFs | +read-ahead | +|---|---|---|---| +| Castlevania X | 1.32 | 2.71 | **2.96** (2.24x) | +| kinst2 | 3.46 | 5.24 | **6.24** (1.80x) | +| Shadowrun | 1.67 | 2.83 | 2.90 (1.74x) | +| Insanity | 2.69 | 4.22 | 4.43 (1.65x) | +| Bonk III | 1.65 | 1.95 | 2.81 (1.70x) | +| Ikaruga | 5.84 | 6.06 | 6.46 (1.11x) | + +Ikaruga moves least because it was already ~90% CPU; Castlevania X moves most +because it was 88% I/O. + +**Anyone running libchdr on ESP-IDF over FATFS should implement +`core_file_callbacks` over `f_read` with a cluster link map, rather than +`fopen`/`fread`.** Select it in this benchmark with `-DBENCH_FATFS_BACKEND=1`. + +### Read-ahead pays here - but only on this backend + +`chd_set_cache_budget()` (0 by default) gives libchdr a byte budget for a +compressed read-ahead window. Its value depends entirely on what is underneath: + +| backend | read-ahead gain | p99 effect | +|---|---|---| +| stdio | **1.01x** | up to **22x worse** | +| FatFs + cluster map | **1.10x** | mostly *better* than baseline | + +On stdio the per-byte cost inside the VFS dominates, so cutting the number of +calls by 6-16x changes almost nothing while a window-sized refill wrecks tail +latency. Once that per-byte cost is gone, per-*call* cost is a real fraction +and the same change earns 10%. Bonk III gains most (1.95 -> 2.81) - smallest +hunks, so the most calls. + +This is the clearest example on this branch of an optimisation being right or +wrong depending on a layer below it, rather than on its own merits. + +## Decode versus storage, against an x86-64 desktop + +Same files, same build flags, versus a Ryzen 7 PRO 8840HS: + +| | ESP32-P4 | x86-64 | ratio | +|---|---|---|---| +| whole sample, wall clock | 2345.9 s | 145.3 s | 16.1x | +| decode only (wall x (1 - io%)) | | | **6.0-8.6x**, mean ~7.2x | + +The CPU-only ratio is tight across four codec families, five hunk geometries +and a 500x file-size range, which says libchdr's decode is not doing anything +pathological on RISC-V - roughly what clock and microarchitecture predict on +their own. Everything beyond that ~7x was storage, and most of it turned out +to be the software above the driver rather than the hardware. + +Measured before the FatFs backend existed, so the wall-clock column reflects +the stdio path; the decode-only column is unaffected by it. + +### Read amplification + +libchdr reads whole hunks only - no sub-hunk API, no decoded-hunk cache for +ordinary reads - so a sub-hunk request costs a full hunk decode. Measured +at 8 units per hunk: + +| units | aligned hunks/req | amp | unaligned hunks/req | amp | +|---|---|---|---|---| +| 1 | 1.00 | **8.00x** | 1.00 | 8.00x | +| 2 | 1.00 | 4.00x | 1.10 | 4.40x | +| 4 | 1.00 | 2.00x | 1.43 | 2.85x | +| 8 | 1.00 | **1.00x** | 2.00 | **2.00x** | +| 16 | 2.00 | 1.00x | 3.00 | 1.50x | + +Latency is flat from 1 to 8 units - a one-sector read takes the same wall +time as a whole hunk. **Integrators should read whole, hunk-aligned +hunks**; an unaligned 8-unit read touches two hunks and doubles the work +for nothing. + +Random access costs **1.8-2.6x** sequential, which is the first +measurement of what `v5_resume_cache`'s sequential fast path is worth. + +### FATFS fast seek is essential on this storage + +FatFs walks the FAT cluster chain on every `f_lseek`, and restarts from the +first cluster on any **backward** seek. Every `COMPRESSION_SELF` reference +is backward (measured: 100%, all 14 sample files), so a self-reference-heavy +CHD pays a full chain walk per hunk. On a 271328-hunk, 29.2%-self-ref file +this took reads from 2.7 to 85 ms/hunk and made a run look hung. + +`CONFIG_FATFS_USE_FASTSEEK=y` makes `lseek` O(1) via a cluster link map +table. In the same 14 minutes the run then reached hunk 245000 instead of +25000 - about **10x**. + +Caveat worth knowing: ESP-IDF allocates a fixed +`CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE` (64 words, ~31 fragments) per open +file and, if a file needs more, **silently frees it and reverts to the slow +path** - no error, no log line. Every file on this card is a single +fragment (32KB clusters), so it engages here; a fragmented card would +quietly get nothing. FatFs reports the required size in `cltbl[0]` even +when it fails, so a two-phase allocation would size it exactly. + +## Resolved: ESP32 ROM `miniz` symbol collision + +**Symptom** (what this section used to describe as an unsolved +silicon/toolchain mystery): `hd_zlib` plus 15 real CHDs failed with +`CHDERR_DECOMPRESSION_ERROR` from `zlib_codec_decompress()`, only on real +hardware, bit-identically across power cycles. + +**Root cause.** Espressif's ROMs bake in an *older* miniz and export its +entry points from the target's ROM linker script as **absolute** symbols +(`esp_rom/esp32p4/ld/esp32p4.rom.ld`, "Group miniz": +`tinfl_decompress = 0x4fc000f8;`). A linker-script assignment outranks an +ordinary object definition, so an ESP-IDF build silently binds that name +to ROM and drops the copy compiled from `deps/miniz-3.1.2/miniz.c` - even +though both are present in the archive. The result was a *split decoder*: +`mz_inflateInit2()`/`mz_inflate()` from miniz 3.1.2 set up and interpret a +3.1.2-layout `tinfl_decompressor`, then handed it to a ROM +`tinfl_decompress()` that lays that struct out differently (miniz 3.0 +reworked the Huffman tables from `tinfl_huff_table m_tables[3]` to the +flattened `m_look_up`/`m_tree_N` form, changing both field offsets and +total size). + +Confirmed directly, not inferred: + +``` +$ riscv32-esp-elf-nm build/...elf | grep -w tinfl_decompress +4fc000f8 A tinfl_decompress <-- absolute, = the ROM address +$ riscv32-esp-elf-objdump -d ... <-- inside our own mz_inflate: +40026f14: jalr 484(ra) # 4fc000f8 +``` + +while `nm build/esp-idf/libchdr/liblibchdr.a` shows libchdr's own +`00000000 T tinfl_decompress` compiled but never linked. + +**Why the failure looked like bad data.** Every one of the 16 failures +consumed *exactly 2 bytes* with `total_out=0` and `zerr=-3`: + +``` +zlib_codec_decompress: FAILED complen=20 destlen=4096 zerr=-3 total_out=0 avail_in=18 ... +zlib_codec_decompress: FAILED complen=14653 destlen=18816 zerr=-3 total_out=0 avail_in=14651 ... +``` + +That is not a decode failure - it is `mz_inflate()` taking the +`TINFL_FLAG_PARSE_ZLIB_HEADER` branch (2-byte CMF/FLG check) on a +*raw-deflate* stream opened with `inflateInit2(..., -MAX_WBITS)`. The +flag is set from `pState->m_window_bits > 0`, and `m_window_bits` lives +*inside* the `inflate_state` allocation, just before `m_dict[32768]`: the +larger ROM decoder writing its tables past the end of the smaller +3.1.2-layout `m_decomp` corrupts it. Hence the signature "first hunk of a +stream decodes, every later one fails". + +`CONFIG_HEAP_POISONING_COMPREHENSIVE=y` was structurally blind to this - +the corruption is *intra-block*, so it never reaches a canary, and +`zlib_fast_alloc()`'s 1KB size rounding leaves enough slack to absorb the +overrun. + +**Fix.** Namespace the colliding symbols on ESP-IDF builds. Six names +collide: `tinfl_decompress`, `tinfl_decompress_mem_to_{heap,mem,callback}`, +`mz_adler32` and `mz_free` (the last one matters independently - binding it +to ROM hands ESP-IDF-heap pointers to the ROM allocator). + +The renames live in **`cmake/EspRomMinizWorkaround.cmake`** at the repo +root, applied as compile definitions to whichever target compiles +`miniz.c` - the `miniz` target from libchdr's own CMake, and +`${COMPONENT_LIB}` from this benchmark's `components/libchdr`. It is a +no-op off ESP-IDF. Deliberately *not* patched into +`deps/miniz-3.1.2/miniz.h`: that tree is vendored verbatim so it can be +re-synced from upstream, and an edit there would be silently dropped by +the next miniz bump - which would resurrect this bug with no diff to +point at. Only `miniz.c` references these names, so scoping the defines +to that target is sufficient. + +Cheap regression check, no flashing required - this must print nothing: + +```sh +grep -hoE '^[A-Za-z_][A-Za-z0-9_]* = 0x' \ + "$IDF_PATH"/components/esp_rom/esp32p4/ld/esp32p4.rom*.ld \ + | sed 's/ = 0x//' | sort -u > /tmp/rom_syms.txt +riscv32-esp-elf-nm build/esp-idf/libchdr/liblibchdr.a \ + | awk '$2 ~ /^[TDBR]$/ {print $3}' | sort -u > /tmp/chdr_syms.txt +comm -12 /tmp/rom_syms.txt /tmp/chdr_syms.txt +``` + +**Why this took so long to find.** Every desktop and QEMU null result +below was *correct* - and none of them could have caught this, because +none was linked against ESP-IDF's ROM linker scripts, so all of them used +real miniz 3.1.2. They are now confirming evidence rather than confusing +evidence. Kept for the record: + +- `heap_caps_check_integrity_all()` (with `CONFIG_HEAP_POISONING_COMPREHENSIVE=y`) + never flags corruption, and free/largest-block heap numbers are flat + through the run - **not** heap corruption or fragmentation. +- Per-hunk tracing shows `inflate()` itself returns `Z_DATA_ERROR` (-3) a + couple of bytes into a short/degenerate compressed stream (e.g. + `complen=20` for a 4096-byte all-zero hunk) - the destination buffer is + never written (still shows the pre-fill poison pattern). +- The exact same 20 compressed bytes, run through the exact same + `miniz.c`/`mz_inflate` with the exact same `MINIZ_NO_*` defines, decode + **correctly** on: x86-64 desktop, x86-32 desktop (`gcc -m32`, so ILP32 + vs LP64 isn't it either), and vanilla `riscv64-unknown-elf-gcc` 13.2.0 + targeting `rv32imafc/ilp32f` under `qemu-system-riscv32` (so it isn't a + general RV32/ILP32 portability bug in miniz). +- Only Espressif's own GCC 14.2.0 `riscv32-esp-elf` toolchain, compiling + for the real chip, reproduces it. Ruled out as the specific cause: + their `xesppie` ISA extension (`-march=rv32imafc_zicsr_zifencei`, + dropping it, made no difference) and optimization level (`-Og`, `-O2`, + and `-O0` all fail identically). +- zstd/LZMA/huffman/FLAC-coded hunks are all unaffected - this is specific + to the zlib/miniz `inflate()` path. + +A *faithful* repro - real `zlib_codec_init()`/`zlib_codec_decompress()`, +called twice on the same codec instance with the real hunk 0 (42 bytes, +succeeds) then hunk 1 (20 bytes, the one that failed on hardware) +compressed bytes pulled straight from a `CHDR_DEBUG_ZLIB` dump - also +passed on x86-64, x86-32, and (freestanding, no ESP-IDF, no libc) +Espressif's own GCC 14.2.0 under `qemu-system-riscv32`. Freestanding is +exactly the point: no ROM linker script, so no collision. + +`src/libchdr_codec_zlib.c` keeps a `CHDR_DEBUG_ZLIB`-gated, +failure-path-only diagnostic printf (`zerr`/`avail_in`/`avail_out`/full +compressed-input hex), off by default. `libchdr_cdrom.c` and +`libchdr_codec_cdfl.c` add stage tags (`stage=base`/`subcode`, +`stage=flac_reset`/`flac_decode`/`subcode`) under the same flag, which is +what separated the two independent bugs here. Note the earlier +unconditional per-call version of this dump flooded UART badly enough to +destabilize a 128-file run on its own - keep it on the failure path. + +**Generalizable lesson.** Espressif ROMs export more than miniz. Any +third-party dep vendored into an ESP-IDF build should be checked against +the target's `rom.ld` symbol list before its behaviour is blamed on +silicon. The check is three shell lines (above) and would have saved this +investigation several days. + +## AVHuff RAM headroom + +**All 4 AVHuff files pass under `LOWRAM_TARGET=1`.** The working-set +figures below still hold and still argue for the streaming redesign - what +changed is that deferring codec `init()` stops AVHuff's state from being +allocated alongside the three other codecs a CHDv5 header may list, which +is enough headroom on this part. Read this section as "why AVHuff is tight +even when it works", not as a blocker. + +With `LOWRAM_TARGET=0`, all 4 AVHuff corpus files failed with +`malloc(219660)`/`malloc(223668) failed` despite ~587KB heap reported free +moments earlier. +`heap_caps_print_heap_info()` at the failure point shows the real number: +only ~78-82KB actually free, because `chd_open()` on an AVHuff CHD +allocates ~500-530KB of internal codec working state (previous-frame +reference buffer + working buffer + Huffman tables - AVHuff's current, +non-streaming decode design keeps at least one full video frame +resident). That's on top of the ~220KB destination buffer this benchmark +also allocates. This matches, almost exactly, the ~633KB whole-hunk +working-set figure already predicted against BL616's RAM budget earlier in +this project (see `project_avhuff_wip` memory) - this board's usable +internal SRAM (~560-590KB total across `RETENT_RAM`/`RAM`/two smaller +pools) hits the same wall. The board has 32MB PSRAM, unused by this build +(`CONFIG_SPIRAM` not enabled) - wiring PSRAM into the heap would likely +fix this specific failure, at the cost of PSRAM's slower access time +skewing AVHuff's throughput numbers relative to internal-SRAM-only +figures above. Confirms the streaming-decode redesign scoped earlier in +this project (avhuff-only additional entry point, ~257KB projected working +set) is worth doing for real memory-constrained targets, not just a +theoretical concern. diff --git a/contrib/esp32p4/idf-benchmark/components/libchdr/CMakeLists.txt b/contrib/esp32p4/idf-benchmark/components/libchdr/CMakeLists.txt new file mode 100644 index 0000000..d426170 --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/components/libchdr/CMakeLists.txt @@ -0,0 +1,91 @@ +# Wraps libchdr as a plain ESP-IDF component, pointing straight at this +# repo's real src/include/deps layout instead of vendoring a copy - the +# codec headers under src/ resolve their dep headers via file-relative +# quote-includes (e.g. src/codec_lzma.h does +# #include "../deps/lzma-26.02/include/LzmaDec.h"), so as long as the real +# directory structure stays intact, no extra include path is needed for +# them - only include/ has to be exposed, for et al. +# +# Only the decoder half of each dep is built (LzmaDec.c, miniz.c with the +# archive/deflate/stdio/time APIs compiled out, zstd's single-file decoder +# amalgamation) - same source list CMakeLists.txt's deps/*/CMakeLists.txt +# targets use, just registered directly as this component's sources instead +# of as nested static-lib targets, since ESP-IDF's build graph doesn't need +# libchdr's own multi-target dependency setup (WITH_SYSTEM_ZLIB/ZSTD, shared +# lib, install rules, etc.) - none of that applies to a firmware image. + +if(NOT DEFINED LOWRAM_TARGET_VAL) + set(LOWRAM_TARGET_VAL 1) +endif() + +set(LIBCHDR_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../../..") + +idf_component_register( + SRCS + "${LIBCHDR_ROOT}/src/libchdr_bitstream.c" + "${LIBCHDR_ROOT}/src/libchdr_cdrom.c" + "${LIBCHDR_ROOT}/src/libchdr_chd.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_avhuff.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_cdfl.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_cdlz.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_cdzl.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_cdzs.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_flac.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_huff.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_lzma.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_zlib.c" + "${LIBCHDR_ROOT}/src/libchdr_codec_zstd.c" + "${LIBCHDR_ROOT}/src/libchdr_flac.c" + "${LIBCHDR_ROOT}/src/libchdr_huffman.c" + "${LIBCHDR_ROOT}/deps/lzma-26.02/src/LzmaDec.c" + "${LIBCHDR_ROOT}/deps/miniz-3.1.2/miniz.c" + "${LIBCHDR_ROOT}/deps/zstd-1.5.7/zstddeclib.c" + INCLUDE_DIRS + "${LIBCHDR_ROOT}/include" +) + +target_compile_definitions(${COMPONENT_LIB} PRIVATE + WANT_RAW_DATA_SECTOR=1 + WANT_SUBCODE=1 + VERIFY_BLOCK_CRC=1 + LOWRAM_TARGET=${LOWRAM_TARGET_VAL} + MINIZ_NO_ARCHIVE_APIS + MINIZ_NO_DEFLATE_APIS + MINIZ_NO_STDIO + MINIZ_NO_TIME + # CHDR_DEBUG_ZLIB: failure-path-only diagnostics (zlib_codec_decompress's + # hex dump, cd_codec_decompress's base/subcode stage tag, cdfl's + # flac/subcode stage tag) - an earlier version of this printed + # unconditionally on every decompress() call and, over a 100+-file real + # corpus, flooded UART with blocking printf()s badly enough to + # destabilize the run on its own. Safe across a full corpus now. + CHDR_DEBUG_ZLIB +) + +# ESP-IDF's default -Werror=all catches a real GCC 14 false positive in +# zstd's single-file decoder amalgamation (HUF_decompress4X{1,2}... +# _usingDTable_internal_fast(): a local struct is fully assigned through a +# helper before the flagged read, GCC just can't see it at -Og) - scoped to +# this one file/component only, not a blanket warning suppression. +set_source_files_properties( + "${LIBCHDR_ROOT}/deps/zstd-1.5.7/zstddeclib.c" + PROPERTIES COMPILE_OPTIONS "-Wno-error=maybe-uninitialized" +) + +# RESOLVED (was: "miniz inflate() returns Z_DATA_ERROR on real P4 silicon"). +# Root cause was a link-time symbol collision, not silicon, codegen or the +# allocator: Espressif's ROMs export an older ROM-baked miniz's +# tinfl_decompress() from their linker scripts as an *absolute* symbol, which +# outranks this component's own definition. Full mechanism, and the symbol +# list, live in cmake/EspRomMinizWorkaround.cmake at the repo root (kept out +# of deps/ so a miniz upgrade can't silently drop it); measured before/after +# is in ../../README.md. +# +# This component compiles miniz.c itself rather than going through libchdr's +# CMake, so it applies the same renames directly. +include("${LIBCHDR_ROOT}/cmake/EspRomMinizWorkaround.cmake") +libchdr_apply_esp_rom_miniz_workaround(${COMPONENT_LIB}) + +if(CHDR_PROFILE_CDFL) + target_compile_definitions(${COMPONENT_LIB} PRIVATE CHDR_PROFILE_CDFL=1) +endif() diff --git a/contrib/esp32p4/idf-benchmark/gen_embed.sh b/contrib/esp32p4/idf-benchmark/gen_embed.sh new file mode 100755 index 0000000..96098c3 --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/gen_embed.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Generates C byte-array headers (main/embed/*.h) from the FULL CHD corpus +# (tests/corpus/seeds/*.chd + tests/avhuff_corpus/{regtest,synth}), for +# embedding into the real-hardware ESP32-P4 throughput benchmark +# (main/benchmark_main.c). Not committed - regenerated on demand, same +# convention as tests/esp32p4/gen_embed.sh (which only embeds a curated +# 7-file subset for the RAM-budget probe; this one embeds everything, since +# the point here is measuring real decode throughput across the whole +# codec/order corpus, not just peak heap per codec). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +OUT_DIR="$SCRIPT_DIR/main/embed" + +mkdir -p "$OUT_DIR" +rm -f "$OUT_DIR"/*.h + +emit() { + local src="$1" name="$2" + xxd -i "$src" | sed \ + -e "s/unsigned char [A-Za-z0-9_]*\[\]/const unsigned char ${name}_chd[]/" \ + -e "s/unsigned int [A-Za-z0-9_]*_len/const unsigned int ${name}_chd_len/" \ + > "$OUT_DIR/${name}.h" +} + +count=0 +list_file="$OUT_DIR/../embed_list.inc" +: > "$list_file" + +for src in "$REPO_ROOT"/tests/corpus/seeds/*.chd; do + name="$(basename "$src" .chd)" + emit "$src" "$name" + echo "{ \"$name\", ${name}_chd, ${name}_chd_len }," >> "$list_file" + count=$((count + 1)) +done + +for src in "$REPO_ROOT"/tests/avhuff_corpus/regtest/*/out.chd; do + name="avhu_$(basename "$(dirname "$src")")" + emit "$src" "$name" + echo "{ \"$name\", ${name}_chd, ${name}_chd_len }," >> "$list_file" + count=$((count + 1)) +done + +for src in "$REPO_ROOT"/tests/avhuff_corpus/synth/*.chd; do + name="avhu_$(basename "$src" .chd)" + emit "$src" "$name" + echo "{ \"$name\", ${name}_chd, ${name}_chd_len }," >> "$list_file" + count=$((count + 1)) +done + +# A handful of real, small MAME-CD-derived CHDs from an actual game library +# (/userdata/roms), not just the synthetic test corpus - flash-sized (<2MB +# each) so they fit without a custom partition table. Optional: skipped if +# REAL_CHD_DIR isn't set or the files aren't there, so this script still +# works standalone for anyone without that ROM directory. +REAL_CHD_DIR="${REAL_CHD_DIR:-/userdata/roms/pcenginecd}" +REAL_CHDS=( + "Pyramid Plunder (USA) (Unl).cue.chd" + "Hawiian Island Girls (USA) (Unl).cue.chd" + "Local Girls of Hawaii, The (USA) (Unl).cue.chd" +) +for fname in "${REAL_CHDS[@]}"; do + src="$REAL_CHD_DIR/$fname" + if [ ! -f "$src" ]; then + echo "gen_embed.sh: skipping missing real CHD: $src" >&2 + continue + fi + name="real_$(basename "$fname" .chd)" + name="$(echo "$name" | tr -c 'A-Za-z0-9_' '_')" + emit "$src" "$name" + echo "{ \"$name\", ${name}_chd, ${name}_chd_len }," >> "$list_file" + count=$((count + 1)) +done + +# one #include line per header, generated alongside the entry list so +# benchmark_main.c doesn't need to be edited when the corpus changes +inc_file="$OUT_DIR/../embed_includes.inc" +: > "$inc_file" +for h in "$OUT_DIR"/*.h; do + echo "#include \"embed/$(basename "$h")\"" >> "$inc_file" +done + +echo "Generated $count embedded CHD headers in $OUT_DIR" diff --git a/contrib/esp32p4/idf-benchmark/main/CMakeLists.txt b/contrib/esp32p4/idf-benchmark/main/CMakeLists.txt new file mode 100644 index 0000000..4e02313 --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/main/CMakeLists.txt @@ -0,0 +1,57 @@ +idf_component_register( + SRCS "benchmark_main.c" + INCLUDE_DIRS "." + REQUIRES libchdr + PRIV_REQUIRES esp_timer fatfs sdmmc esp_driver_sdmmc +) + +# Pass C lever sweep selector: idf.py -DBENCH_MODE_LEVER=1 builds the small +# 4-file capped subset used for config comparisons instead of the full +# characterized Pass A sample. +if(BENCH_MODE_LEVER) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_MODE_LEVER=1) +endif() + +# Pass D: idf.py -DCHDR_PROFILE_CDFL=1 turns on libchdr's cdfl stage profiler +# (needs the same define on the libchdr component, see its CMakeLists). +if(CHDR_PROFILE_CDFL) + target_compile_definitions(${COMPONENT_LIB} PRIVATE CHDR_PROFILE_CDFL=1) +endif() +if(SEEK_ELISION) + target_compile_definitions(${COMPONENT_LIB} PRIVATE SEEK_ELISION=1) +endif() + +# single-file bisect mode: idf.py -DBENCH_ONE_FILE='"/sdcard/path.chd"' +if(BENCH_ONE_FILE) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_ONE_FILE=${BENCH_ONE_FILE}) +endif() +if(BENCH_PROGRESS_EVERY) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_PROGRESS_EVERY=${BENCH_PROGRESS_EVERY}) +endif() +if(BENCH_FSINFO) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_FSINFO=${BENCH_FSINFO}) +endif() +if(BENCH_HUNK_CAP) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_HUNK_CAP=${BENCH_HUNK_CAP}) +endif() +if(BENCH_CACHE_BUDGET) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_CACHE_BUDGET=${BENCH_CACHE_BUDGET}) +endif() +if(BENCH_QUICK) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_QUICK=1) +endif() +if(BENCH_RAWIO) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_RAWIO=1) +endif() +if(BENCH_FATFS_BACKEND) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_FATFS_BACKEND=1) +endif() +# Integer multiply latency probe: idf.py -DBENCH_MULPROBE=1 runs only the +# dependent-chain timing for add/xor/mul/mulh and returns, skipping the corpus. +if(BENCH_MULPROBE) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_MULPROBE=1) +endif() +# ECC inner-loop variant comparison: idf.py -DBENCH_MULPROBE=1 -DBENCH_ECCPROBE=1 +if(BENCH_ECCPROBE) + target_compile_definitions(${COMPONENT_LIB} PRIVATE BENCH_ECCPROBE=1) +endif() diff --git a/contrib/esp32p4/idf-benchmark/main/benchmark_main.c b/contrib/esp32p4/idf-benchmark/main/benchmark_main.c new file mode 100644 index 0000000..daa20b9 --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/main/benchmark_main.c @@ -0,0 +1,1581 @@ +/* + * Real-hardware libchdr throughput benchmark for ESP32-P4 (Waveshare + * ESP32-P4-NANO, dual-core RISC-V @ up to ~400MHz, 768KB HP L2MEM). + * + * Opens every CHD embedded into flash by gen_embed.sh (the synthetic + * tests/corpus/seeds + tests/avhuff_corpus set, plus a few small real + * MAME-derived CHDs pulled from an actual game library), reads every hunk + * of each with block-CRC verification on (VERIFY_BLOCK_CRC=1, see + * components/libchdr/CMakeLists.txt), times the decode with + * esp_timer_get_time(), and prints per-file + aggregate MB/s (both + * compressed-in and decompressed-out) over the USB-C serial console. + * + * Storage is an in-memory core_file backend over the embedded flash data + * (same pattern as tests/esp32p4/fw.c's RAM-budget probe) - no SD card + * needed, so this runs with zero GPIO/pin configuration. Real large CHDs + * (segacd/psp/saturn/etc, tens to hundreds of MB) don't fit in flash this + * way and need the SD card path instead - see contrib/esp32p4/README.md. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "esp_timer.h" +#include "esp_heap_caps.h" +#include "esp_heap_caps_init.h" +#include "esp_vfs_fat.h" +#include "sdmmc_cmd.h" +#include "ff.h" +#include "driver/sdmmc_host.h" +#if SOC_SDMMC_IO_POWER_EXTERNAL +#include "sd_pwr_ctrl_by_on_chip_ldo.h" +#endif + +#include +#include + +#include "embed_includes.inc" + +/* Waveshare ESP32-P4-NANO onboard TF/SD slot, native SDMMC slot 0, 4-bit, + * per Waveshare's own 06_sdmmc example (confirmed against an independent + * community bring-up suite: github.com/Tangerino/micropython-p4-test-suite). + * SD card IO is powered from the P4's on-chip LDO channel 4 - without + * enabling that, the slot times out with ESP_ERR_TIMEOUT. */ +#define SD_PIN_CLK 43 +#define SD_PIN_CMD 44 +#define SD_PIN_D0 39 +#define SD_PIN_D1 40 +#define SD_PIN_D2 41 +#define SD_PIN_D3 42 +#define SD_LDO_CHAN 4 +#define SD_MOUNT_POINT "/sdcard" + +/* SD bus clock. ESP-IDF defaults to SDMMC_FREQ_DEFAULT (20MHz) when + * max_freq_khz is left unset, which is what every measurement before the + * bottleneck sweep used. SDMMC_FREQ_HIGHSPEED is 40MHz. */ +#ifndef SD_FREQ_KHZ +#define SD_FREQ_KHZ SDMMC_FREQ_HIGHSPEED +#endif + +/* ---- in-memory core_file backend (chd_open_core_file_callbacks) ---- */ + +typedef struct { + const unsigned char *data; + size_t len; + size_t pos; +} membuf; + +static uint64_t mem_fsize(void *argp) +{ + membuf *m = (membuf *)argp; + return (uint64_t)m->len; +} + +static size_t mem_fread(void *ptr, size_t size, size_t nmemb, void *argp) +{ + membuf *m = (membuf *)argp; + size_t want = size * nmemb; + size_t avail = (m->pos < m->len) ? (m->len - m->pos) : 0; + size_t take = (want < avail) ? want : avail; + memcpy(ptr, m->data + m->pos, take); + m->pos += take; + return take / size; +} + +static int mem_fclose(void *argp) +{ + (void)argp; + return 0; +} + +static int mem_fseek(void *argp, int64_t offset, int whence) +{ + membuf *m = (membuf *)argp; + int64_t base = (whence == SEEK_SET) ? 0 : (whence == SEEK_CUR) ? (int64_t)m->pos : (int64_t)m->len; + int64_t newpos = base + offset; + if (newpos < 0 || (uint64_t)newpos > m->len) return -1; + m->pos = (size_t)newpos; + return 0; +} + +static const core_file_callbacks mem_callbacks = { + .fsize = mem_fsize, + .fread = mem_fread, + .fclose = mem_fclose, + .fseek = mem_fseek, +}; + +/* ---- real file core_file backend (chd_open_core_file_callbacks), for the + * SD/mass-storage path - reads through the FATFS VFS mounted on the + * onboard SDMMC slot, same interface libchdr sees on any other host. ---- */ + +static uint64_t sdfile_fsize(void *argp) +{ + FILE *f = (FILE *)argp; + long cur = ftell(f); + fseek(f, 0, SEEK_END); + long sz = ftell(f); + fseek(f, cur, SEEK_SET); + return (uint64_t)sz; +} + +/* Bottleneck attribution without touching libchdr: these callbacks are the + * only path from the decoder to the card, so accounting here splits a hunk's + * wall time into "waiting on SD" and "everything else" (decode + CRC + map) + * exactly, with no guesswork and no instrumentation inside the library. */ +static struct { + uint64_t read_us, seek_us; + uint64_t read_bytes; + uint64_t reads, seeks, seeks_elided; + int64_t pos; /* our idea of the current file offset */ + int pos_valid; +} g_io; + +static void io_reset(void) { memset(&g_io, 0, sizeof(g_io)); } + +/* SEEK_ELISION: libchdr issues core_fseek + core_fread for every hunk + * unconditionally. During a sequential sweep the file is very often already + * at the requested offset, and on FATFS an f_lseek is not free - it can walk + * the cluster chain. Skipping the call when the position already matches is + * three lines and costs nothing. Measured because the Pass A attribution + * showed seek time equalling or exceeding read time on every large file. */ +#ifndef SEEK_ELISION +#define SEEK_ELISION 0 +#endif + +static size_t sdfile_fread(void *ptr, size_t size, size_t nmemb, void *argp) +{ + int64_t t0 = esp_timer_get_time(); + size_t n = fread(ptr, size, nmemb, (FILE *)argp); + g_io.read_us += (uint64_t)(esp_timer_get_time() - t0); + g_io.read_bytes += (uint64_t)n * size; + g_io.reads++; + if (g_io.pos_valid) g_io.pos += (int64_t)(n * size); + return n; +} + +static int sdfile_fclose(void *argp) +{ + return fclose((FILE *)argp); +} + +static int sdfile_fseek(void *argp, int64_t offset, int whence) +{ + int64_t t0, r; +#if SEEK_ELISION + if (whence == SEEK_SET && g_io.pos_valid && g_io.pos == offset) { + g_io.seeks_elided++; + return 0; + } +#endif + t0 = esp_timer_get_time(); + r = fseek((FILE *)argp, (long)offset, whence); + g_io.seek_us += (uint64_t)(esp_timer_get_time() - t0); + g_io.seeks++; + if (r == 0 && whence == SEEK_SET) { g_io.pos = offset; g_io.pos_valid = 1; } + else g_io.pos_valid = 0; + return (int)r; +} + +/* ---- FatFs-backed core_file, bypassing VFS and newlib stdio ---- + * + * Measured on this board, same card and clock: sdmmc_read_sectors() sustains + * 18.96 MB/s, FatFs f_read() 10.58, and fread() through the VFS and newlib + * stdio only 2.30. The wrapper above FatFs costs 4.6x. + * + * libchdr never needs to know: core_file_callbacks already lets the embedder + * supply whatever reader it likes, so this is a drop-in replacement for the + * stdio backend and requires no change to the library. */ +static uint64_t fffile_fsize(void *argp) +{ + return (uint64_t)f_size((FIL *)argp); +} + +static size_t fffile_fread(void *ptr, size_t size, size_t nmemb, void *argp) +{ + UINT got = 0; + int64_t t0 = esp_timer_get_time(); + FRESULT r = f_read((FIL *)argp, ptr, (UINT)(size * nmemb), &got); + g_io.read_us += (uint64_t)(esp_timer_get_time() - t0); + g_io.read_bytes += got; + g_io.reads++; + if (r != FR_OK) return 0; + return size ? (got / size) : 0; +} + +static int fffile_fclose(void *argp) +{ + FIL *fp = (FIL *)argp; + DWORD *clmt = fp->cltbl; + FRESULT r; + fp->cltbl = NULL; + r = f_close(fp); + free(clmt); + free(fp); + return (r == FR_OK) ? 0 : -1; +} + +static int fffile_fseek(void *argp, int64_t offset, int whence) +{ + FIL *fp = (FIL *)argp; + FSIZE_t target; + int64_t t0; + FRESULT r; + + switch (whence) { + case SEEK_SET: target = (FSIZE_t)offset; break; + case SEEK_CUR: target = f_tell(fp) + (FSIZE_t)offset; break; + case SEEK_END: target = f_size(fp) + (FSIZE_t)offset; break; + default: return -1; + } + t0 = esp_timer_get_time(); + r = f_lseek(fp, target); + g_io.seek_us += (uint64_t)(esp_timer_get_time() - t0); + g_io.seeks++; + return (r == FR_OK) ? 0 : -1; +} + +static const core_file_callbacks fffile_callbacks = { + .fsize = fffile_fsize, + .fread = fffile_fread, + .fclose = fffile_fclose, + .fseek = fffile_fseek, +}; + +/* opens a FatFs path (the VFS path minus the mount prefix) */ +/* CLMT size for the FatFs backend. Every file on this card is a single + * fragment, so 3 words suffice; sized well above that so fragmented cards + * still get fast seek. 512 words is 2KB per open file. */ +#define FFFILE_CLMT_WORDS 512 + +static FIL *fffile_open(const char *vfs_path) +{ + const char *ff = vfs_path; + FIL *fp; + DWORD *clmt; + + if (strncmp(ff, SD_MOUNT_POINT, strlen(SD_MOUNT_POINT)) == 0) + ff += strlen(SD_MOUNT_POINT); + fp = (FIL *)malloc(sizeof(FIL)); + if (fp == NULL) return NULL; + if (f_open(fp, ff, FA_READ) != FR_OK) { free(fp); return NULL; } + + /* Build the cluster link map ourselves. Going straight to FatFs skips + * ESP-IDF's VFS, and the VFS is what normally allocates cltbl and runs + * f_lseek(CREATE_LINKMAP) - see vfs_fat.c. Without it every *backward* + * seek restarts FatFs's cluster-chain walk from the first cluster, and + * every COMPRESSION_SELF reference is a backward seek, so a self-ref-heavy + * CHD slows to a crawl. That is the same pathology CONFIG_FATFS_USE_FASTSEEK + * was enabled to cure, and it is easy to miss: a short capped run never + * walks far enough to notice. */ + clmt = (DWORD *)malloc(sizeof(DWORD) * FFFILE_CLMT_WORDS); + if (clmt != NULL) { + fp->cltbl = clmt; + clmt[0] = FFFILE_CLMT_WORDS; + if (f_lseek(fp, CREATE_LINKMAP) != FR_OK) { + /* too fragmented to map - run without fast seek rather than fail */ + printf(" (fast seek unavailable, needs %lu words)\n", (unsigned long)clmt[0]); + fp->cltbl = NULL; + free(clmt); + } + f_lseek(fp, 0); + } + return fp; +} + +static const core_file_callbacks sdfile_callbacks = { + .fsize = sdfile_fsize, + .fread = sdfile_fread, + .fclose = sdfile_fclose, + .fseek = sdfile_fseek, +}; + +/* mounts the onboard TF/SD slot; returns the card handle on success, NULL + * (with a logged reason) if no card is present or the mount fails - this + * benchmark still runs the flash-embedded corpus either way. */ +static sdmmc_card_t *mount_sdcard(void) +{ + esp_err_t ret; + sdmmc_card_t *card = NULL; + + esp_vfs_fat_sdmmc_mount_config_t mount_config = { + .format_if_mount_failed = false, + .max_files = 8, + .allocation_unit_size = 16 * 1024, + }; + + sdmmc_host_t host = SDMMC_HOST_DEFAULT(); + host.slot = SDMMC_HOST_SLOT_0; + host.max_freq_khz = SD_FREQ_KHZ; + +#if SOC_SDMMC_IO_POWER_EXTERNAL + sd_pwr_ctrl_ldo_config_t ldo_config = { .ldo_chan_id = SD_LDO_CHAN }; + sd_pwr_ctrl_handle_t pwr_ctrl_handle = NULL; + ret = sd_pwr_ctrl_new_on_chip_ldo(&ldo_config, &pwr_ctrl_handle); + if (ret != ESP_OK) { + printf("SD: sd_pwr_ctrl_new_on_chip_ldo failed: %s\n", esp_err_to_name(ret)); + return NULL; + } + host.pwr_ctrl_handle = pwr_ctrl_handle; +#endif + + sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT(); + slot_config.width = 4; + slot_config.clk = SD_PIN_CLK; + slot_config.cmd = SD_PIN_CMD; + slot_config.d0 = SD_PIN_D0; + slot_config.d1 = SD_PIN_D1; + slot_config.d2 = SD_PIN_D2; + slot_config.d3 = SD_PIN_D3; + slot_config.flags |= SDMMC_SLOT_FLAG_INTERNAL_PULLUP; + + ret = esp_vfs_fat_sdmmc_mount(SD_MOUNT_POINT, &host, &slot_config, &mount_config, &card); + if (ret != ESP_OK) { + printf("SD: mount failed (%s) - is a card inserted?\n", esp_err_to_name(ret)); + return NULL; + } + + sdmmc_card_print_info(stdout, card); + return card; +} + +/* recursively finds every *.chd under root, up to max_files entries; each + * found path is stored as a heap-allocated string in out[]. Returns the + * count found. */ +#define SD_MAX_FILES 128 +#define SD_MAX_DEPTH 6 + +/* BENCH_MODE_LEVER: the bottleneck/headroom sweep. Same harness, but a + * 4-file codec-representative subset with a hunk cap, so one build+flash+run + * cycle is ~2 minutes and several toolchain/driver configurations can be + * compared in one sitting. Relative numbers are what matter here, so coverage + * is deliberately traded for turnaround. */ +#ifndef BENCH_MODE_LEVER +#define BENCH_MODE_LEVER 0 +#endif + +/* Per-file hunk cap for the SD sweep (0 = read every hunk). A full uncapped + * sweep of all 292 valid CHDs on the card is ~151GB decompressed, ~24h. Pass A + * instead runs uncapped over the characterized sample below, ~6.8GB. */ +#ifdef BENCH_HUNK_CAP +#define SD_MAX_HUNKS_PER_FILE BENCH_HUNK_CAP +#elif BENCH_MODE_LEVER +#define SD_MAX_HUNKS_PER_FILE 3000 +#else +#define SD_MAX_HUNKS_PER_FILE 0 +#endif + +static uint32_t g_max_hunks = 0; + +#define ARRAY_LEN(a) (sizeof(a) / sizeof((a)[0])) + +/* BENCH_PROGRESS_EVERY: emit a progress line every N hunks. Off by default. + * Without it a stalled run is indistinguishable from a merely slow one - the + * board just stops printing, and "hung" vs "still going, 40x slower than it + * started" are completely different diagnoses. With it, the slope of ms/hunk + * over the file separates a driver stall (flat, then nothing) from map-window + * thrashing (progressively degrading). */ +#ifndef BENCH_PROGRESS_EVERY +#define BENCH_PROGRESS_EVERY 0 +#endif + +/* Read-ahead budget handed to libchdr per file. 0 keeps the historical + * one-read-per-hunk behaviour, which is the control this run needs. */ +#ifndef BENCH_CACHE_BUDGET +#define BENCH_CACHE_BUDGET 0 +#endif + +/* Quick validation subset: the five files that between them are the only + * ones exercising each thing under test - cdzs context sharing (Ikaruga), + * the huffman fix and the LOWRAM map-read floor (kinst2), cdzl base streams + * (Shadowrun), the most I/O-bound file (Castlevania X) and the smallest + * hunks / most transactions (Bonk III). */ +#ifndef BENCH_QUICK +#define BENCH_QUICK 0 +#endif + +#ifndef BENCH_RAWIO +#define BENCH_RAWIO 0 +#endif + +/* Use the FatFs-backed core_file instead of the stdio one. */ +#ifndef BENCH_FATFS_BACKEND +#define BENCH_FATFS_BACKEND 0 +#endif + +/* ---- filesystem geometry / fragmentation ---- + * + * Whether FATFS_USE_FASTSEEK does anything at all depends on how many + * contiguous cluster runs a file occupies: the CLMT buffer is fixed size + * (CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE, 64 DWORDs by default = ~31 runs) and + * ESP-IDF silently falls back to the slow path when a file needs more, with + * no error and no log line. So measure it rather than assume. + * + * f_lseek(CREATE_LINKMAP) writes the *required* table size into cltbl[0] even + * when it fails with FR_NOT_ENOUGH_CORE, which is exactly the number wanted. + * Table layout is [size][n0][c0][n1][c1]... so runs = (items - 1) / 2. */ +#ifndef BENCH_FSINFO +#define BENCH_FSINFO 0 +#endif + +#if BENCH_FSINFO +#define FSINFO_CLMT_WORDS 8192 +static void fs_report(const char *const *paths, size_t n) +{ + DWORD nclust = 0; + FATFS *fs = NULL; + if (f_getfree("", &nclust, &fs) == FR_OK && fs) { + printf("FS: cluster = %u sectors = %u bytes | free %lu clusters (%.2f GB)\n", + (unsigned)fs->csize, (unsigned)fs->csize * 512, + (unsigned long)nclust, (double)nclust * fs->csize * 512 / 1e9); + } else { + printf("FS: f_getfree failed\n"); + } + + DWORD *tbl = malloc(sizeof(DWORD) * FSINFO_CLMT_WORDS); + if (!tbl) { printf("FS: no memory for CLMT probe\n"); return; } + + for (size_t i = 0; i < n; i++) { + const char *vfs = paths[i]; + const char *ffpath = vfs; + if (strncmp(ffpath, SD_MOUNT_POINT, strlen(SD_MOUNT_POINT)) == 0) + ffpath += strlen(SD_MOUNT_POINT); /* FatFs sees the path without the VFS prefix */ + FIL fp; + if (f_open(&fp, ffpath, FA_READ) != FR_OK) { + printf(" %-52s f_open failed\n", ffpath); + continue; + } + fp.cltbl = tbl; + tbl[0] = FSINFO_CLMT_WORDS; + FRESULT r = f_lseek(&fp, CREATE_LINKMAP); + DWORD items = tbl[0]; + DWORD runs = items > 1 ? (items - 1) / 2 : 0; + printf(" %-52s %8llu KB %5lu fragments%s\n", + ffpath, (unsigned long long)(f_size(&fp) / 1024), (unsigned long)runs, + (r == FR_NOT_ENOUGH_CORE) ? " (EXCEEDS default 64-word CLMT -> fastseek silently disabled)" + : (r != FR_OK ? " (linkmap failed)" : "")); + fp.cltbl = NULL; + f_close(&fp); + } + free(tbl); +} +#endif + +/* ---- Pass A sample ---- + * + * Picked by characterizing all 292 valid CHDs on the card (see + * tools/characterize.c) and then covering every axis that actually varies, + * rather than taking "a few files per system" and hoping. Between them these + * 14 cover: + * geometry all 5 hunkbytes/units-per-hunk combinations in the corpus + * (19584/8, 4096/8, 9792/4, 2448/1, 4096/2) + * codecs all 11 selectors seen anywhere: cdlz cdzl cdfl cdzs zlib lzma + * huff flac zstd, plus uncompressed and self-referencing hunks + * codec mix files using 1, 2, 3 and 4 distinct codecs + * tracks 0, 1, 2, 3, 13, 17 and 96 tracks + * size 2MB to 1.25GB logical + * systems all 7 present on the card + * + * Two entries are deliberately non-obvious: Bonk III is the only 1-unit-per- + * hunk file in the corpus, so it is the control for the read-amplification + * sweep (no amplification is possible), and Shadowrun is the most cdzl-heavy + * file at 58.7% - cdzl being the codec the ESP ROM miniz collision broke, so + * it is worth keeping permanently in the regression path. */ +#define SD_USE_SAMPLE 1 + +/* Pass B is cheap (a few minutes) but only meaningful on the sample */ +#define PASS_B_ENABLE (!BENCH_MODE_LEVER) + +#if BENCH_QUICK +static const char *const g_sd_sample[] = { + "/sdcard/roms/psp/Castlevania X.chd", + "/sdcard/roms/dreamcast/Ikaruga (Japan).chd", + "/sdcard/roms/mame/kinst2/kinst2.chd", + "/sdcard/roms/segacd/Shadowrun (J).chd", + "/sdcard/roms/pcenginecd/Bonk III - Bonk's Big Adventure (USA).chd", +}; +#elif defined(BENCH_ONE_FILE) +/* single-file bisect mode, for chasing a hang down to one configuration */ +static const char *const g_sd_sample[] = { BENCH_ONE_FILE }; +#elif BENCH_MODE_LEVER +static const char *const g_sd_sample[] = { + "/sdcard/roms/segacd/Shadowrun (J).chd", /* cdzl 58.7% */ + "/sdcard/roms/saturn/SS-parodius-sexy.chd", /* cdlz 93.6% */ + "/sdcard/roms/pcenginecd/Insanity (USA) (Unl).cue.chd", /* cdfl 79.0% */ + "/sdcard/roms/mame/kinst2/kinst2.chd", /* lzma+zlib+huff, 4096B hunks */ +}; +#else +static const char *const g_sd_sample[] = { + /* system geometry ncodec tracks dominant mix */ + "/sdcard/roms/dreamcast/Ikaruga (Japan).chd", /* 19584/8 2 3 cdzs 92.9% */ + "/sdcard/roms/mame/kinst2/kinst2.chd", /* 4096/8 4 0 lzma+zlib+huff+flac */ + "/sdcard/roms/mame/simpbowl/simpbowl.chd", /* 9792/4 3 1 cdlz 55.6% cdzl 36.8% */ + "/sdcard/roms/naomi/vathlete/gds-0019.chd", /* 19584/8 3 3 cdlz 88.6%, largest */ + "/sdcard/roms/pcenginecd/Bonk III - Bonk's Big Adventure (USA).chd", /* 2448/1 2 17 cdfl 93.6%, no amplification */ + "/sdcard/roms/pcenginecd/Hawiian Island Girls (USA) (Unl).cue.chd", /* 19584/8 1 1 cdlz 100%, tiny */ + "/sdcard/roms/pcenginecd/Insanity (USA) (Unl).cue.chd", /* 19584/8 3 13 cdfl 79.0% */ + "/sdcard/roms/pcenginecd/Pyramid Plunder (USA) (Unl).cue.chd", /* 19584/8 2 1 cdlz 97.6%, smallest */ + "/sdcard/roms/psp/Castlevania X.chd", /* 4096/2 2 0 zstd 35.6% uncomp 34.8% */ + "/sdcard/roms/saturn/SS-parodius-sexy.chd", /* 19584/8 3 2 cdlz 93.6% */ + "/sdcard/roms/segacd/Cadillacs & Dinosaurs - The Second Cataclysm (U).chd", /* 19584/8 1 2 cdlz 100% */ + "/sdcard/roms/segacd/Sensible Soccer (E) (Demo).chd", /* 19584/8 3 96 cdfl 64.6%, most tracks */ + "/sdcard/roms/segacd/Shadowrun (J).chd", /* 19584/8 3 3 cdzl 58.7% */ + "/sdcard/roms/segacd/Surgical Strike (Brazil) (32X CD).chd", /* 19584/8 3 3 cdlz 90.6% */ +}; +#endif + +__attribute__((unused)) +static int sd_scan_dir(const char *dir, char **out, int count, int depth) +{ + if (depth > SD_MAX_DEPTH || count >= SD_MAX_FILES) + return count; + + DIR *d = opendir(dir); + if (!d) return count; + + struct dirent *ent; + while ((ent = readdir(d)) != NULL && count < SD_MAX_FILES) { + if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) + continue; + + char path[300]; + snprintf(path, sizeof(path), "%s/%s", dir, ent->d_name); + + struct stat st; + if (stat(path, &st) != 0) + continue; + + if (S_ISDIR(st.st_mode)) { + count = sd_scan_dir(path, out, count, depth + 1); + } else { + size_t plen = strlen(path); + if (plen > 4 && !strcasecmp(path + plen - 4, ".chd")) + out[count++] = strdup(path); + } + } + closedir(d); + return count; +} + +/* ---- corpus table ---- */ + +typedef struct { + const char *name; + const unsigned char *data; + unsigned int len; +} corpus_entry; + +static const corpus_entry g_corpus[] = { +#include "embed_list.inc" +}; + +#ifdef CHDR_PROFILE_CDFL +/* clock for libchdr's cdfl stage profiler (see src/libchdr_codec_cdfl.c) */ +int64_t chdr_prof_now_us(void) { return esp_timer_get_time(); } +extern uint64_t chdr_prof_flac_reset_us, chdr_prof_flac_decode_us, + chdr_prof_subcode_us, chdr_prof_cdfl_hunks; +static void prof_reset(void) +{ + chdr_prof_flac_reset_us = chdr_prof_flac_decode_us = 0; + chdr_prof_subcode_us = chdr_prof_cdfl_hunks = 0; +} +static void prof_print(void) +{ + uint64_t n = chdr_prof_cdfl_hunks; + uint64_t tot = chdr_prof_flac_reset_us + chdr_prof_flac_decode_us + chdr_prof_subcode_us; + if (!n || !tot) return; + printf(" cdfl stages over %" PRIu64 " hunks: reset %5.1f%% (%.3f ms/hunk) " + "decode %5.1f%% (%.3f ms/hunk) subcode %5.1f%% (%.3f ms/hunk)\n", + n, + 100.0 * chdr_prof_flac_reset_us / tot, chdr_prof_flac_reset_us / 1000.0 / n, + 100.0 * chdr_prof_flac_decode_us / tot, chdr_prof_flac_decode_us / 1000.0 / n, + 100.0 * chdr_prof_subcode_us / tot, chdr_prof_subcode_us / 1000.0 / n); +} +#else +static void prof_reset(void) {} +static void prof_print(void) {} +#endif + +/* ---- latency histogram ---- + * + * A mean hunk-decode time is close to useless for judging whether a CD read + * will glitch audio - that is a tail property. Keeping every sample is not an + * option either (271328 hunks in one corpus file), so accumulate into a + * fixed-size log-scale histogram and read percentiles back off it. + * + * Bucket index packs 4 sub-buckets per octave: (log2(us) << 2) | top 2 + * mantissa bits. That is ~19% worst-case bucket width, which is plenty to + * separate a cache-ish hit from a full LZMA hunk decode. */ +#define LAT_SUBBITS 2 +#define LAT_SUB (1u << LAT_SUBBITS) +#define LAT_BUCKETS (32 * LAT_SUB) + +typedef struct { + uint32_t bucket[LAT_BUCKETS]; + uint64_t count, sum_us, min_us, max_us; +} lat_hist; + +static void lat_init(lat_hist *h) +{ + memset(h, 0, sizeof(*h)); + h->min_us = UINT64_MAX; +} + +static void lat_add(lat_hist *h, uint64_t us) +{ + unsigned idx, e; + if (us == 0) us = 1; + e = 31u - (unsigned)__builtin_clz((uint32_t)(us > 0xFFFFFFFFu ? 0xFFFFFFFFu : us)); + if (e >= LAT_SUBBITS) + idx = (e << LAT_SUBBITS) | (unsigned)((us >> (e - LAT_SUBBITS)) & (LAT_SUB - 1)); + else + idx = (unsigned)us; + if (idx >= LAT_BUCKETS) idx = LAT_BUCKETS - 1; + h->bucket[idx]++; + h->count++; + h->sum_us += us; + if (us < h->min_us) h->min_us = us; + if (us > h->max_us) h->max_us = us; +} + +/* lower edge of a bucket, in us - report the conservative (low) end */ +static uint64_t lat_bucket_us(unsigned idx) +{ + unsigned e = idx >> LAT_SUBBITS; + if (e < LAT_SUBBITS) return idx; + return ((uint64_t)(LAT_SUB | (idx & (LAT_SUB - 1)))) << (e - LAT_SUBBITS); +} + +static uint64_t lat_pct(const lat_hist *h, double p) +{ + uint64_t want, seen = 0; + unsigned i; + if (h->count == 0) return 0; + want = (uint64_t)(p * (double)h->count); + for (i = 0; i < LAT_BUCKETS; i++) { + seen += h->bucket[i]; + if (seen >= want) return lat_bucket_us(i); + } + return h->max_us; +} + +static void lat_print(const char *label, const lat_hist *h) +{ + if (h->count == 0) { printf(" %-22s (no samples)\n", label); return; } + printf(" %-22s n=%-7" PRIu64 " min=%-7" PRIu64 " p50=%-7" PRIu64 " p95=%-8" PRIu64 + " p99=%-8" PRIu64 " max=%-8" PRIu64 " mean=%" PRIu64 " (us)\n", + label, h->count, h->min_us, lat_pct(h, 0.50), lat_pct(h, 0.95), + lat_pct(h, 0.99), h->max_us, h->sum_us / h->count); +} + +/* ---- Pass B: request-granularity / latency characterization ---- + * + * libchdr's only read entry point is chd_read(chd, hunknum, buf) - there is no + * sub-hunk API and no decoded-hunk cache (file_cache precaches the whole + * *compressed* file, hopeless for a 500MB CHD here; v5_resume_cache is a + * sequential map/codec fast path, not a data cache). So a caller asking for N + * units must decode every hunk those units land in and slice. + * + * That means "latency vs request size" is really *read amplification*: at 8 + * units per hunk a 1-unit request costs a full hunk, i.e. 8x the bytes it + * wanted. This measures what that actually costs in wall time, and how much + * worse it gets when the request is not hunk-aligned (16 units at unit offset + * 1 spans 3 hunks, not 2). + * + * Random vs sequential is measured alongside because v5_resume_cache only + * helps sequential access, and nothing has ever quantified what it is worth. */ + +#define PASSB_SAMPLES 120 + +static uint32_t rnd_state = 0x1234567u; +static uint32_t rnd_next(void) +{ + rnd_state ^= rnd_state << 13; + rnd_state ^= rnd_state >> 17; + rnd_state ^= rnd_state << 5; + return rnd_state; +} + +__attribute__((unused)) +static void pass_b_file(const char *name) +{ + static const uint32_t sizes[] = { 1, 2, 4, 8, 16 }; + FILE *f = fopen(name, "rb"); + chd_file *chd = NULL; + if (!f) { printf(" %-40s fopen FAILED\n", name); return; } + if (chd_open_core_file_callbacks(&sdfile_callbacks, f, CHD_OPEN_READ, NULL, &chd) != CHDERR_NONE) { + printf(" %-40s OPEN FAILED\n", name); + return; + } + const chd_header *h = chd_get_header(chd); + uint32_t uph = h->unitbytes ? h->hunkbytes / h->unitbytes : 1; + uint8_t *buf = malloc(h->hunkbytes); + if (!buf) { printf(" %-40s malloc failed\n", name); chd_close(chd); return; } + + printf("\n %s\n hunkbytes=%" PRIu32 " unitbytes=%" PRIu32 " units/hunk=%" PRIu32 + " totalhunks=%" PRIu32 "\n", name, h->hunkbytes, h->unitbytes, uph, h->totalhunks); + + /* sequential vs random, whole hunks - isolates v5_resume_cache's value */ + { + lat_hist seq, rnd; + lat_init(&seq); lat_init(&rnd); + uint32_t n = h->totalhunks < PASSB_SAMPLES ? h->totalhunks : PASSB_SAMPLES; + uint32_t base = h->totalhunks > n ? (rnd_next() % (h->totalhunks - n)) : 0; + for (uint32_t i = 0; i < n; i++) { + int64_t a = esp_timer_get_time(); + if (chd_read(chd, base + i, buf) != CHDERR_NONE) break; + lat_add(&seq, (uint64_t)(esp_timer_get_time() - a)); + } + for (uint32_t i = 0; i < n; i++) { + uint32_t hn = rnd_next() % h->totalhunks; + int64_t a = esp_timer_get_time(); + if (chd_read(chd, hn, buf) != CHDERR_NONE) break; + lat_add(&rnd, (uint64_t)(esp_timer_get_time() - a)); + } + lat_print("1 hunk sequential", &seq); + lat_print("1 hunk random", &rnd); + if (seq.count && rnd.count) + printf(" -> random/sequential mean = %.2fx\n", + (double)(rnd.sum_us / rnd.count) / (double)(seq.sum_us / seq.count)); + } + + /* request-size sweep, aligned and unaligned, random placement */ + printf(" %-9s %-9s %8s %8s %9s %9s %7s\n", + "units", "align", "hunks/req", "p50(us)", "p95(us)", "max(us)", "amp"); + for (size_t s = 0; s < ARRAY_LEN(sizes); s++) { + for (int unaligned = 0; unaligned < 2; unaligned++) { + lat_hist lh; + uint64_t touched = 0, delivered = 0, decoded = 0; + uint32_t n = PASSB_SAMPLES; + lat_init(&lh); + for (uint32_t i = 0; i < n; i++) { + /* pick a unit offset; unaligned deliberately straddles */ + uint64_t totalunits = (uint64_t)h->totalhunks * uph; + if (totalunits <= sizes[s]) break; + uint64_t uoff = rnd_next() % (totalunits - sizes[s]); + if (!unaligned) uoff -= uoff % uph; + else if (uph > 1 && (uoff % uph) == 0) uoff += 1; + uint32_t first = (uint32_t)(uoff / uph); + uint32_t last = (uint32_t)((uoff + sizes[s] - 1) / uph); + if (last >= h->totalhunks) continue; + int64_t a = esp_timer_get_time(); + int ok = 1; + for (uint32_t hn = first; hn <= last; hn++) + if (chd_read(chd, hn, buf) != CHDERR_NONE) { ok = 0; break; } + if (!ok) break; + lat_add(&lh, (uint64_t)(esp_timer_get_time() - a)); + touched += (last - first + 1); + decoded += (uint64_t)(last - first + 1) * h->hunkbytes; + delivered += (uint64_t)sizes[s] * h->unitbytes; + } + if (!lh.count) continue; + printf(" %-9" PRIu32 " %-9s %8.2f %8" PRIu64 " %9" PRIu64 " %9" PRIu64 " %6.2fx\n", + sizes[s], unaligned ? "unaligned" : "aligned", + (double)touched / (double)lh.count, + lat_pct(&lh, 0.50), lat_pct(&lh, 0.95), lh.max_us, + delivered ? (double)decoded / (double)delivered : 0); + } + } + + free(buf); + chd_close(chd); +} + +/* ---- benchmark driver ---- */ + +typedef struct { + uint64_t compressed_bytes; + uint64_t decompressed_bytes; + uint64_t elapsed_us; + int ok; +} run_result; + +static run_result run_one(const char *name, const core_file_callbacks *cb, void *argp, uint64_t reported_len) +{ + run_result r = {0}; + chd_file *chd = NULL; + chd_error err; + int64_t t0, t1; + + io_reset(); + prof_reset(); + /* Heap accounting. An earlier version of this used + * heap_caps_get_minimum_free_size(), which is the low-water mark *since + * boot* - it never rises again, so after the first corpus file every + * delta read as zero and the whole measurement was silently useless. + * Use free-size deltas instead and track the minimum by sampling during + * the read loop: free_size is a sum of per-region counters, cheap enough + * to poll. What matters for "does this CHD fit on a smaller part" is the + * resident cost of an open file plus its decode working set. */ + size_t heap_at_entry = heap_caps_get_free_size(MALLOC_CAP_DEFAULT); + size_t heap_after_open = heap_at_entry; + size_t heap_min_free = heap_at_entry; + size_t largest_min = heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT); + t0 = esp_timer_get_time(); + + err = chd_open_core_file_callbacks(cb, argp, CHD_OPEN_READ, NULL, &chd); + if (err != CHDERR_NONE) { + printf("%-48s OPEN FAILED: %s\n", name, chd_error_string(err)); + /* chd_open_core_file_callbacks()'s cleanup: path calls chd_close() + * (hence core_fclose(argp)) on every failure past its first, + * near-unfailable malloc(sizeof(chd_file)) - i.e. argp is already + * closed here in every failure mode this benchmark actually hits. + * An extra cb->fclose(argp) here is a double-close: harmless on the + * flash path (mem_fclose() is a no-op) but a real double-free of + * the FATFS file object on the SD path, which corrupts a FreeRTOS + * queue used by the VFS/SDMMC layer and hard-resets the board (seen + * live: an OOM opening a large real CHD from SD, immediately + * followed by "assert failed: xQueueSemaphoreTake queue.c:1713"). */ + return r; + } + +#if BENCH_CACHE_BUDGET + { + chd_error be = chd_set_cache_budget(chd, BENCH_CACHE_BUDGET); + if (be != CHDERR_NONE) + printf(" (cache budget %d refused: %s)\n", (int)BENCH_CACHE_BUDGET, chd_error_string(be)); + } +#endif + heap_after_open = heap_caps_get_free_size(MALLOC_CAP_DEFAULT); + if (heap_after_open < heap_min_free) heap_min_free = heap_after_open; + + const chd_header *header = chd_get_header(chd); + unsigned char *buf = malloc(header->hunkbytes); + if (!buf) { + printf("%-48s malloc(%" PRIu32 ") failed\n", name, header->hunkbytes); + void *retry = heap_caps_malloc(header->hunkbytes, MALLOC_CAP_DEFAULT); + printf(" retry heap_caps_malloc(same size, DEFAULT) = %p\n", retry); + if (retry) heap_caps_free(retry); + heap_caps_print_heap_info(MALLOC_CAP_DEFAULT); + chd_close(chd); + return r; + } + + /* Once the ROM-miniz collision was fixed, files that used to bail out at + * hunk 0-few now decode in full, and a 128-file uncapped SD sweep of a + * real 43GB ROM set runs for hours. g_max_hunks caps the per-file read so + * every file in the corpus still gets exercised in one sitting; 0 = read + * everything (what the flash corpus and the throughput table use). */ + uint32_t nhunks = header->totalhunks; + int capped = 0; + if (g_max_hunks != 0 && nhunks > g_max_hunks) { nhunks = g_max_hunks; capped = 1; } + + /* Per-hunk latency comes free here - we are already in the read loop, and + * this is the sequential access pattern, i.e. the one v5_resume_cache's + * fast path is built for. Pass B measures random access against it. */ + lat_hist seqlat; + lat_init(&seqlat); + + int bad = 0; + uint32_t bad_hunk = 0; +#if BENCH_PROGRESS_EVERY + int64_t prog_t = esp_timer_get_time(); + uint64_t prog_io = 0; +#endif + for (uint32_t i = 0; i < nhunks; i++) { + int64_t h0, h1; +#if BENCH_PROGRESS_EVERY + if (i && (i % BENCH_PROGRESS_EVERY) == 0) { + int64_t now = esp_timer_get_time(); + uint64_t io_now = g_io.read_us + g_io.seek_us; + printf(" ..hunk %" PRIu32 "/%" PRIu32 " %.3f ms/hunk io %.3f ms/hunk heap used %d KB\n", + i, nhunks, + (double)(now - prog_t) / 1000.0 / BENCH_PROGRESS_EVERY, + (double)(io_now - prog_io) / 1000.0 / BENCH_PROGRESS_EVERY, + (int)((heap_at_entry - heap_caps_get_free_size(MALLOC_CAP_DEFAULT)) / 1024)); + prog_t = now; prog_io = io_now; + } +#endif + memset(buf, 0xAA, header->hunkbytes); /* poison, so a no-op decode is visible */ + if ((i & 63) == 0) { + size_t f = heap_caps_get_free_size(MALLOC_CAP_DEFAULT); + size_t lb = heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT); + if (f < heap_min_free) heap_min_free = f; + if (lb < largest_min) largest_min = lb; + } + h0 = esp_timer_get_time(); + err = chd_read(chd, i, buf); + h1 = esp_timer_get_time(); + if (err != CHDERR_NONE) { bad = 1; bad_hunk = i; break; } + lat_add(&seqlat, (uint64_t)(h1 - h0)); + } + + t1 = esp_timer_get_time(); + + if (bad) { + printf("%-48s READ FAILED at hunk %" PRIu32 "/%" PRIu32 ": %s\n", + name, bad_hunk, header->totalhunks, chd_error_string(err)); + printf(" dest[0..31] = "); + for (int k = 0; k < 32 && k < (int)header->hunkbytes; k++) printf("%02x", buf[k]); + printf("\n"); + free(buf); + chd_close(chd); + return r; + } + + /* only a full read has a meaningful compressed-input size to rate against */ + r.compressed_bytes = capped ? 0 : reported_len; + r.decompressed_bytes = (uint64_t)header->hunkbytes * nhunks; + r.elapsed_us = (uint64_t)(t1 - t0); + r.ok = 1; + + double secs = r.elapsed_us / 1e6; + double out_mbps = secs > 0 ? (r.decompressed_bytes / 1e6) / secs : 0; + double in_mbps = secs > 0 ? (r.compressed_bytes / 1e6) / secs : 0; + + if (capped) + printf("%-48s hunks=%" PRIu32 "/%-4" PRIu32 " (capped) hunkbytes=%-7" PRIu32 " %8.2f ms out=%7.2f MB/s\n", + name, nhunks, header->totalhunks, header->hunkbytes, secs * 1000.0, out_mbps); + else + printf("%-48s hunks=%-4" PRIu32 " hunkbytes=%-7" PRIu32 " %8.2f ms in=%6.2f MB/s out=%7.2f MB/s\n", + name, header->totalhunks, header->hunkbytes, secs * 1000.0, in_mbps, out_mbps); + + /* bottleneck split: everything not spent inside the storage callbacks is + * decode + CRC + map work, so this attributes the hunk time exactly */ + { + uint64_t io_us = g_io.read_us + g_io.seek_us; + double io_pct = r.elapsed_us ? 100.0 * (double)io_us / (double)r.elapsed_us : 0; + printf(" io: %6.2f%% of wall (%" PRIu64 " ms read + %" PRIu64 " ms seek, %" + PRIu64 " reads / %" PRIu64 " seeks (%" PRIu64 " elided), %" PRIu64 " KB, %.2f MB/s while reading)" + " cpu: %6.2f%%\n", + io_pct, g_io.read_us / 1000, g_io.seek_us / 1000, g_io.reads, g_io.seeks, g_io.seeks_elided, + g_io.read_bytes / 1024, + g_io.read_us ? (g_io.read_bytes / 1e6) / (g_io.read_us / 1e6) : 0, + 100.0 - io_pct); + printf(" heap: open %d KB, peak %d KB (incl. %" PRIu32 " KB dest buf), " + "smallest largest-free-block %d KB\n", + (int)((heap_at_entry - heap_after_open) / 1024), + (int)((heap_at_entry - heap_min_free) / 1024), + header->hunkbytes / 1024, + (int)(largest_min / 1024)); + { + uint64_t rh = 0, rm = 0; + chd_get_cache_stats(chd, &rh, &rm); + if (rh || rm) + printf(" readahead: %" PRIu64 " hits / %" PRIu64 " refills\n", rh, rm); + } + lat_print("hunk latency (seq)", &seqlat); + prof_print(); + } + + free(buf); + chd_close(chd); + return r; +} + + +/* -------------------------------------------------------------------------- + * BENCH_ECCPROBE: which ECC inner loop is actually fastest on this core. + * + * Two independent substitutions are available, and instruction counts on x86 + * predict the opposite of what an in-order single-issue core does, so measure: + * + * ecclow[256] is exactly xtime(x) in GF(2^8) with poly 0x11d - verified + * for all 256 entries. Table = xor/add/lbu in-loop (one load); + * arithmetic = 6 ALU ops, no load. + * poffsets/qoffsets are closed form - verified exactly: + * poffsets[r][c] = r + 86c + * qoffsets[r][c] = (86*(r>>1) + (r&1) + 88c) mod 2236 + * so the offset load can become an increment. Together the + * three tables are 8856 bytes of .rodata. + * -------------------------------------------------------------------------- */ +#ifndef BENCH_ECCPROBE +#define BENCH_ECCPROBE 0 +#endif + +#if BENCH_ECCPROBE +#include "esp_cpu.h" + +#define EP_P_NUM 86 +#define EP_P_COMP 24 +#define EP_Q_NUM 52 +#define EP_Q_COMP 43 +#define EP_P_OFF 0x81c +#define EP_Q_OFF (EP_P_OFF + 2 * EP_P_NUM) + +static uint16_t ep_poff[EP_P_NUM][EP_P_COMP]; +static uint16_t ep_qoff[EP_Q_NUM][EP_Q_COMP]; +static uint8_t ep_ecclow[256]; +static uint8_t ep_sector[2352]; + +static void ep_init(void) +{ + int r, c, i; + for (r = 0; r < EP_P_NUM; r++) + for (c = 0; c < EP_P_COMP; c++) + ep_poff[r][c] = (uint16_t)(r + 86 * c); + for (r = 0; r < EP_Q_NUM; r++) + for (c = 0; c < EP_Q_COMP; c++) + ep_qoff[r][c] = (uint16_t)((86 * (r >> 1) + (r & 1) + 88 * c) % 2236); + for (i = 0; i < 256; i++) + ep_ecclow[i] = (uint8_t)((i << 1) ^ ((i >> 7) * 0x1d)); + for (i = 0; i < 2352; i++) + ep_sector[i] = (uint8_t)(i * 7 + (i >> 3)); + ep_sector[15] = 1; /* mode 1, so the mode-2 masking path is not taken */ +} + +#define EP_XTIME(x) ((uint8_t)(((x) << 1) ^ (((x) >> 7) * 0x1d))) + +/* A: table offsets + table xtime (what ships today) */ +static void ep_bytes_A(const uint8_t *sec, const uint16_t *row, int len, + uint8_t *o1, uint8_t *o2) +{ + const uint8_t *d = sec + 12; + uint8_t v1 = 0, v2 = 0; + int c; + for (c = 0; c < len; c++) { + const uint8_t b = d[row[c]]; + v1 = ep_ecclow[v1 ^ b]; + v2 ^= b; + } + *o1 = v1; *o2 = v2 ^ v1; +} + +/* B: table offsets + arithmetic xtime */ +static void ep_bytes_B(const uint8_t *sec, const uint16_t *row, int len, + uint8_t *o1, uint8_t *o2) +{ + const uint8_t *d = sec + 12; + uint8_t v1 = 0, v2 = 0; + int c; + for (c = 0; c < len; c++) { + const uint8_t b = d[row[c]]; + uint8_t t = v1 ^ b; + v1 = EP_XTIME(t); + v2 ^= b; + } + *o1 = v1; *o2 = v2 ^ v1; +} + +/* C: closed-form offsets + table xtime. P walks +86, Q walks +88 mod 2236. */ +static void ep_bytes_C(const uint8_t *sec, uint32_t off, uint32_t step, + uint32_t mod, int len, uint8_t *o1, uint8_t *o2) +{ + const uint8_t *d = sec + 12; + uint8_t v1 = 0, v2 = 0; + int c; + for (c = 0; c < len; c++) { + const uint8_t b = d[off]; + v1 = ep_ecclow[v1 ^ b]; + v2 ^= b; + off += step; + if (off >= mod) off -= mod; + } + *o1 = v1; *o2 = v2 ^ v1; +} + +/* D: closed-form offsets + arithmetic xtime */ +static void ep_bytes_D(const uint8_t *sec, uint32_t off, uint32_t step, + uint32_t mod, int len, uint8_t *o1, uint8_t *o2) +{ + const uint8_t *d = sec + 12; + uint8_t v1 = 0, v2 = 0; + int c; + for (c = 0; c < len; c++) { + const uint8_t b = d[off]; + uint8_t t = v1 ^ b; + v1 = EP_XTIME(t); + v2 ^= b; + off += step; + if (off >= mod) off -= mod; + } + *o1 = v1; *o2 = v2 ^ v1; +} + +static void ep_gen_AB(uint8_t *sec, int arith) +{ + int b; + for (b = 0; b < EP_P_NUM; b++) + (arith ? ep_bytes_B : ep_bytes_A)(sec, ep_poff[b], EP_P_COMP, + &sec[EP_P_OFF + b], &sec[EP_P_OFF + EP_P_NUM + b]); + for (b = 0; b < EP_Q_NUM; b++) + (arith ? ep_bytes_B : ep_bytes_A)(sec, ep_qoff[b], EP_Q_COMP, + &sec[EP_Q_OFF + b], &sec[EP_Q_OFF + EP_Q_NUM + b]); +} + +static void ep_gen_CD(uint8_t *sec, int arith) +{ + int b; + for (b = 0; b < EP_P_NUM; b++) + (arith ? ep_bytes_D : ep_bytes_C)(sec, (uint32_t)b, 86, 0xffffffffu, + EP_P_COMP, &sec[EP_P_OFF + b], &sec[EP_P_OFF + EP_P_NUM + b]); + for (b = 0; b < EP_Q_NUM; b++) + (arith ? ep_bytes_D : ep_bytes_C)(sec, + (uint32_t)(86 * (b >> 1) + (b & 1)), 88, 2236, + EP_Q_COMP, &sec[EP_Q_OFF + b], &sec[EP_Q_OFF + EP_Q_NUM + b]); +} + +/* E: P restructured so the 86 independent rows are the inner loop over + * contiguous bytes (poffsets[r][c] = r + 86c), which auto-vectorises - 9.7x on + * x86 SSE2, 17.7x AVX2, and NEON on aarch64. The P4 has no vector unit GCC can + * target, so this measures whether the restructure alone costs anything here. + * Q keeps the shipped scalar table path; its offsets are a diagonal and are not + * contiguous in either dimension. */ +#pragma GCC push_options +#pragma GCC optimize("O3","tree-vectorize") +static void ep_p_rows_vec(const uint8_t *sec, uint8_t *p1, uint8_t *p2) +{ + const uint8_t *d = sec + 12; + uint8_t v1[EP_P_NUM], v2[EP_P_NUM]; + int c, r; + memset(v1, 0, EP_P_NUM); memset(v2, 0, EP_P_NUM); + for (c = 0; c < EP_P_COMP; c++) { + const uint8_t *src = d + EP_P_NUM * c; + for (r = 0; r < EP_P_NUM; r++) { + uint8_t b = src[r]; + uint8_t t = v1[r] ^ b; + v1[r] = (uint8_t)((t << 1) ^ ((t >> 7) * 0x1d)); + v2[r] ^= b; + } + } + for (r = 0; r < EP_P_NUM; r++) { p1[r] = v1[r]; p2[r] = v2[r] ^ v1[r]; } +} +#pragma GCC pop_options + +static void ep_gen_E(uint8_t *sec) +{ + int b; + ep_p_rows_vec(sec, &sec[EP_P_OFF], &sec[EP_P_OFF + EP_P_NUM]); + for (b = 0; b < EP_Q_NUM; b++) + ep_bytes_A(sec, ep_qoff[b], EP_Q_COMP, + &sec[EP_Q_OFF + b], &sec[EP_Q_OFF + EP_Q_NUM + b]); +} + +static void run_eccprobe(void) +{ + static uint8_t ref[2352], tmp[2352]; + const char *names[5] = { "A table off + table xtime (ships)", + "B table off + arith xtime", + "C closed off + table xtime", + "D closed off + arith xtime", + "E P row-inner vectorisable + Q as-is" }; + double base = 0.0; + int v; + + ep_init(); + printf("=== ECC inner-loop variants (2352-byte sector, ecc_generate) ===\n"); + + /* correctness: every variant must produce the same parity bytes */ + memcpy(ref, ep_sector, 2352); ep_gen_AB(ref, 0); + for (v = 1; v < 5; v++) { + memcpy(tmp, ep_sector, 2352); + if (v == 1) ep_gen_AB(tmp, 1); + else if (v == 4) ep_gen_E(tmp); + else ep_gen_CD(tmp, v == 3); + printf(" variant %c parity %s\n", 'A' + v, + memcmp(ref, tmp, 2352) == 0 ? "MATCHES A" : "*** DIFFERS ***"); + } + + for (v = 0; v < 5; v++) { + const int reps = 2000; + uint32_t best = 0xffffffffu; + int rep, k; + for (rep = 0; rep < 5; rep++) { + uint32_t t0 = esp_cpu_get_cycle_count(); + for (k = 0; k < reps; k++) { + if (v < 2) ep_gen_AB(ep_sector, v); + else if (v == 4) ep_gen_E(ep_sector); + else ep_gen_CD(ep_sector, v == 3); + } + uint32_t t1 = esp_cpu_get_cycle_count(); + if ((t1 - t0) < best) best = t1 - t0; + } + { + double cyc = (double)best / reps; + if (v == 0) base = cyc; + printf(" %-34s %8.0f cycles/sector %5.2f us %5.3fx\n", + names[v], cyc, cyc / 400.0, base / cyc); + } + } + printf("=== eccprobe done ===\n"); +} +#endif /* BENCH_ECCPROBE */ + +/* -------------------------------------------------------------------------- + * BENCH_MULPROBE: dependent-chain latency for the integer multiplier. + * + * dr_flac's LPC prediction for CD audio always takes its 64-bit path + * (bitsPerSample + precision + ilog2(order) = 16+15+5 > 32), and a 32x32->64 + * multiply-accumulate on RV32 costs a mul plus a mulh where x86-64 spends one + * imul. If mulh is multi-cycle and unpipelined that is the whole reason FLAC + * costs far more per instruction here than LZMA, whose range decoder has no + * 64-bit multiplies at all. Measure it rather than assume. + * -------------------------------------------------------------------------- */ +#ifndef BENCH_MULPROBE +#define BENCH_MULPROBE 0 +#endif + +#if BENCH_MULPROBE +#include "esp_cpu.h" + +#define MULPROBE_ITERS 200000 + +/* Each chain is 8 back-to-back dependent ops, so the loop overhead is + * amortised and what is left is 8 x issue-to-use latency. */ +#define MULPROBE_CHAIN(name, insn) \ + static uint32_t mulprobe_##name(uint32_t iters, uint32_t seed) \ + { \ + uint32_t a = seed, b = 3, t0, t1; \ + t0 = esp_cpu_get_cycle_count(); \ + while (iters--) { \ + __asm__ volatile( \ + insn " %0, %0, %1\n\t" insn " %0, %0, %1\n\t" \ + insn " %0, %0, %1\n\t" insn " %0, %0, %1\n\t" \ + insn " %0, %0, %1\n\t" insn " %0, %0, %1\n\t" \ + insn " %0, %0, %1\n\t" insn " %0, %0, %1" \ + : "+r"(a) : "r"(b)); \ + } \ + t1 = esp_cpu_get_cycle_count(); \ + g_mulprobe_sink += a; \ + return t1 - t0; \ + } + +static volatile uint32_t g_mulprobe_sink; +MULPROBE_CHAIN(add, "add") +MULPROBE_CHAIN(mul, "mul") +MULPROBE_CHAIN(mulh, "mulh") +MULPROBE_CHAIN(xor, "xor") + +/* dr_flac allocates roughly one 32-64KB block per FLAC hunk (measured: 0.82 + * allocs and ~34.7KB per cdfl hunk, against 0.017 and 714 bytes for cdlz). + * glibc's malloc cost is inside the x86 instruction counts we model against, + * but ESP-IDF's capability-tagged multi-heap allocator is not, so time it here + * rather than infer it from a model residual. */ +static void run_mallocprobe(void) +{ + static const size_t sizes[] = { 256, 4096, 32768, 65536, 223668 }; + size_t i; + + printf("=== heap_caps_malloc + free latency ===\n"); + for (i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + const int reps = 2000; + uint32_t t0, t1, best = 0xffffffffu; + int rep, k; + + for (rep = 0; rep < 5; rep++) { + t0 = esp_cpu_get_cycle_count(); + for (k = 0; k < reps; k++) { + void *p = heap_caps_malloc(sizes[i], MALLOC_CAP_DEFAULT); + g_mulprobe_sink += (uint32_t)(uintptr_t)p; + heap_caps_free(p); + } + t1 = esp_cpu_get_cycle_count(); + if ((t1 - t0) < best) + best = t1 - t0; + } + printf(" %7u B: %8.1f cycles/(malloc+free) = %6.2f us\n", + (unsigned)sizes[i], (double)best / reps, + (double)best / reps / 400.0); + } + /* and what it costs to merely touch that much fresh memory once */ + { + const int reps = 200; + void *p = heap_caps_malloc(32768, MALLOC_CAP_DEFAULT); + uint32_t t0 = esp_cpu_get_cycle_count(); + int k; + for (k = 0; k < reps; k++) + memset(p, k, 32768); + uint32_t t1 = esp_cpu_get_cycle_count(); + printf(" memset 32KB: %.1f cycles = %.2f us\n", + (double)(t1 - t0) / reps, (double)(t1 - t0) / reps / 400.0); + heap_caps_free(p); + } + printf("=== mallocprobe done ===\n"); +} + +static void run_mulprobe(void) +{ + struct { const char *name; uint32_t (*fn)(uint32_t, uint32_t); } probes[] = { + { "add ", mulprobe_add }, + { "xor ", mulprobe_xor }, + { "mul ", mulprobe_mul }, + { "mulh", mulprobe_mulh }, + }; + size_t i; + + printf("=== integer op latency (dependent chain, %d iters x 8 ops) ===\n", + MULPROBE_ITERS); + for (i = 0; i < sizeof(probes) / sizeof(probes[0]); i++) { + uint32_t best = 0xffffffffu, r; + int rep; + /* best of 5: the cycle counter is shared with interrupts */ + for (rep = 0; rep < 5; rep++) { + r = probes[i].fn(MULPROBE_ITERS, 0x9e3779b9u + rep); + if (r < best) + best = r; + } + printf(" %s: %10u cycles for %u ops -> %.2f cycles/op\n", + probes[i].name, (unsigned)best, + (unsigned)(MULPROBE_ITERS * 8), + (double)best / (double)(MULPROBE_ITERS * 8)); + } + printf("=== mulprobe done ===\n"); +} +#endif /* BENCH_MULPROBE */ + +void app_main(void) +{ +#if BENCH_MULPROBE + run_mulprobe(); + run_mallocprobe(); +#if BENCH_ECCPROBE + run_eccprobe(); +#endif + return; +#endif + printf("=== libchdr ESP32-P4 real-hardware throughput benchmark ===\n"); + printf("free heap: %u bytes (largest block: %u)\n", + (unsigned)heap_caps_get_free_size(MALLOC_CAP_DEFAULT), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT)); + printf("free INTERNAL: %u (largest %u) free 8BIT: %u (largest %u)\n", + (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL), + (unsigned)heap_caps_get_free_size(MALLOC_CAP_8BIT), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_8BIT)); + + /* isolate: does a single ~220KB plain malloc() work on a totally fresh + * heap, before any codec has run? */ + { + void *probe = malloc(223668); + printf("cold-boot malloc(223668) = %p\n", probe); + void *probe_caps = heap_caps_malloc(223668, MALLOC_CAP_DEFAULT); + printf("cold-boot heap_caps_malloc(223668, DEFAULT) = %p\n", probe_caps); + if (probe) free(probe); + if (probe_caps) heap_caps_free(probe_caps); + } + printf("%-48s %-10s %-9s %13s %11s %14s\n", + "file", "hunks", "hunkbytes", "time", "in", "out"); + + size_t n = sizeof(g_corpus) / sizeof(g_corpus[0]); + uint64_t total_in = 0, total_out = 0, total_us = 0; + int total_ok = 0; + + printf("--- flash-embedded corpus (%zu files) ---\n", n); + for (size_t i = 0; i < n; i++) { + printf(" [before] free=%u largest=%u\n", + (unsigned)heap_caps_get_free_size(MALLOC_CAP_DEFAULT), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT)); + bool heap_ok_before = heap_caps_check_integrity_all(true); + membuf mb = { g_corpus[i].data, g_corpus[i].len, 0 }; + run_result r = run_one(g_corpus[i].name, &mem_callbacks, &mb, g_corpus[i].len); + bool heap_ok_after = heap_caps_check_integrity_all(true); + if (!heap_ok_before || !heap_ok_after) { + printf(" ^^^ heap corruption detected: before=%d after=%d (free=%u largest=%u)\n", + heap_ok_before, heap_ok_after, + (unsigned)heap_caps_get_free_size(MALLOC_CAP_DEFAULT), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT)); + } + if (r.ok) { + total_in += r.compressed_bytes; + total_out += r.decompressed_bytes; + total_us += r.elapsed_us; + total_ok++; + } + } + + double total_secs = total_us / 1e6; + printf("=== flash: %d/%zu files OK, %" PRIu64 " bytes in / %" PRIu64 " bytes out, " + "%.2f s total, avg in=%.2f MB/s out=%.2f MB/s ===\n", + total_ok, n, total_in, total_out, total_secs, + total_secs > 0 ? (total_in / 1e6) / total_secs : 0, + total_secs > 0 ? (total_out / 1e6) / total_secs : 0); + + /* ---- SD/mass-storage corpus: the actual deployment path (onboard TF + * card slot, not flash). Also re-tests every synthetic seed CHD - + * including hd_zlib.chd, the known-failing Class B repro - read through + * the real FATFS/SDMMC stack instead of a flash-mapped const array, to + * separate "read source" from "CPU execution" as candidate causes. ---- */ + printf("\n--- SD card corpus ---\n"); + sdmmc_card_t *card = mount_sdcard(); + if (!card) { + printf("=== SD: no card mounted, skipping SD corpus ===\n"); + return; + } + + static char *sd_paths[SD_MAX_FILES]; + int sd_n; +#if SD_USE_SAMPLE + sd_n = 0; + for (size_t si = 0; si < ARRAY_LEN(g_sd_sample) && sd_n < SD_MAX_FILES; si++) { + struct stat st; + if (stat(g_sd_sample[si], &st) != 0) { + printf("SAMPLE MISSING: %s\n", g_sd_sample[si]); + continue; + } + sd_paths[sd_n++] = strdup(g_sd_sample[si]); + } + printf("SD: characterized sample, %d of %zu entries present\n", sd_n, ARRAY_LEN(g_sd_sample)); +#else + sd_n = sd_scan_dir(SD_MOUNT_POINT, sd_paths, 0, 0); + printf("SD: found %d *.chd file(s) under %s\n", sd_n, SD_MOUNT_POINT); +#endif + +#if BENCH_RAWIO + /* Is the ~2 MB/s ceiling libchdr's or the storage stack's? Read a real + * file straight through fread() at several block sizes, no CHD parsing, + * no decode. If this also tops out around 2 MB/s then the limit is the + * card, the SDMMC driver or FATFS, and no amount of work inside libchdr + * moves it. Sweeping the block size separately answers whether the path + * is transaction-bound or bandwidth-bound. */ + /* Split the card and driver from the filesystem: sdmmc_read_sectors() + * talks to the card directly, so if this is also ~2 MB/s the limit is the + * card or the SDMMC driver, and if it is much faster the cost is in FATFS + * or the VFS/stdio layers above it. Read-only, so it cannot disturb the + * card contents. */ + printf("\n--- raw sector read, bypassing FATFS ---\n"); + { + static const size_t nsec[] = { 8, 64, 128, 512 }; /* 4KB .. 256KB */ + for (size_t i = 0; i < ARRAY_LEN(nsec); i++) { + size_t bytes = nsec[i] * 512; + uint8_t *buf = heap_caps_malloc(bytes, MALLOC_CAP_DMA); + if (!buf) { printf(" %4zu sectors (%3zu KB): DMA malloc failed\n", nsec[i], bytes/1024); continue; } + uint64_t total = 0; size_t sector = 40960; /* well inside the data area */ + int64_t t0 = esp_timer_get_time(); + while (total < 16u*1024*1024) { + if (sdmmc_read_sectors(card, buf, sector, nsec[i]) != ESP_OK) break; + sector += nsec[i]; total += bytes; + } + int64_t t1 = esp_timer_get_time(); + double secs = (t1 - t0) / 1e6; + printf(" %4zu sectors (%3zu KB): %6.2f MB in %5.2f s = %5.2f MB/s\n", + nsec[i], bytes/1024, total/1e6, secs, secs > 0 ? (total/1e6)/secs : 0); + heap_caps_free(buf); + } + } + + /* Middle layer: FatFs f_read() directly, skipping the VFS and newlib + * stdio that fread() goes through. Three points - sectors, f_read, fread - + * localise the 8.4x loss to one layer instead of "somewhere above the + * driver". */ + printf("\n--- FatFs f_read, bypassing VFS+stdio ---\n"); + { + static const size_t blks[] = { 4096, 32768, 131072 }; + const char *vfs = sd_n > 0 ? sd_paths[0] : NULL; + const char *ff = vfs; + if (ff && strncmp(ff, SD_MOUNT_POINT, strlen(SD_MOUNT_POINT)) == 0) + ff += strlen(SD_MOUNT_POINT); + for (size_t i = 0; ff && i < ARRAY_LEN(blks); i++) { + uint8_t *buf = malloc(blks[i]); + /* FIL embeds a sector buffer and is far too large for app_main's + * frame - a stack-allocated one tripped the stack protector */ + FIL *fp = malloc(sizeof(FIL)); + if (!buf || !fp) { free(buf); free(fp); printf(" %6zu KB: malloc failed\n", blks[i]/1024); continue; } + if (f_open(fp, ff, FA_READ) != FR_OK) { free(buf); free(fp); printf(" f_open failed\n"); break; } + uint64_t total = 0; int64_t t0 = esp_timer_get_time(); + while (total < 24u*1024*1024) { + UINT got = 0; + if (f_read(fp, buf, (UINT)blks[i], &got) != FR_OK || got == 0) break; + total += got; + } + int64_t t1 = esp_timer_get_time(); + double secs = (t1 - t0) / 1e6; + printf(" block %6zu KB: %7.2f MB in %6.2f s = %5.2f MB/s\n", + blks[i]/1024, total/1e6, secs, secs > 0 ? (total/1e6)/secs : 0); + f_close(fp); free(fp); free(buf); + } + } + + printf("\n--- raw SD read throughput (no libchdr) ---\n"); + { + static const size_t blks[] = { 4096, 16384, 32768, 131072, 524288 }; + const char *probe = NULL; + for (int i = 0; i < sd_n && probe == NULL; i++) probe = sd_paths[i]; + for (size_t bi = 0; bi < ARRAY_LEN(blks); bi++) { + uint8_t *buf = malloc(blks[bi]); + if (!buf) { printf(" %6zu KB block: malloc failed\n", blks[bi]/1024); continue; } + FILE *rf = fopen(probe, "rb"); + if (!rf) { free(buf); printf(" fopen failed\n"); break; } + uint64_t total = 0; int64_t t0 = esp_timer_get_time(); + /* 24MB is enough to be well past any caching and still quick */ + while (total < 24u*1024*1024) { + size_t got = fread(buf, 1, blks[bi], rf); + if (got == 0) break; + total += got; + } + int64_t t1 = esp_timer_get_time(); + double secs = (t1 - t0) / 1e6; + printf(" block %6zu KB: %7.2f MB in %6.2f s = %5.2f MB/s\n", + blks[bi]/1024, total/1e6, secs, (total/1e6)/secs); + fclose(rf); free(buf); + } + } +#endif + +#if BENCH_FSINFO + printf("\n--- filesystem geometry and fragmentation ---\n"); + fs_report((const char *const *)sd_paths, (size_t)sd_n); +#if BENCH_FSINFO > 1 + printf("\n=== BENCH COMPLETE ===\n"); /* report-only mode */ + return; +#endif +#endif + + g_max_hunks = SD_MAX_HUNKS_PER_FILE; + if (g_max_hunks) + printf("SD: reading at most %" PRIu32 " hunks per file\n", g_max_hunks); + + uint64_t sd_total_in = 0, sd_total_out = 0, sd_total_us = 0; + int sd_total_ok = 0; + + for (int i = 0; i < sd_n; i++) { +#if BENCH_FATFS_BACKEND + FIL *f = fffile_open(sd_paths[i]); + if (!f) { + printf("%-56s f_open FAILED\n", sd_paths[i]); + continue; + } +#else + FILE *f = fopen(sd_paths[i], "rb"); + if (!f) { + printf("%-56s fopen FAILED\n", sd_paths[i]); + continue; + } +#endif + bool heap_ok_before = heap_caps_check_integrity_all(true); +#if BENCH_FATFS_BACKEND + run_result r = run_one(sd_paths[i], &fffile_callbacks, f, fffile_fsize(f)); +#else + run_result r = run_one(sd_paths[i], &sdfile_callbacks, f, sdfile_fsize(f)); +#endif + bool heap_ok_after = heap_caps_check_integrity_all(true); + if (!heap_ok_before || !heap_ok_after) { + printf(" ^^^ heap corruption detected: before=%d after=%d\n", heap_ok_before, heap_ok_after); + } + /* run_one() already closes the chd_file (and thus calls + * sdfile_fclose on f via chd_close's core_fclose) on both the + * success and failure paths, so f is already closed here. */ + if (r.ok) { + sd_total_in += r.compressed_bytes; + sd_total_out += r.decompressed_bytes; + sd_total_us += r.elapsed_us; + sd_total_ok++; + } + } + + double sd_total_secs = sd_total_us / 1e6; + printf("=== SD: %d/%d files OK, %" PRIu64 " bytes in / %" PRIu64 " bytes out, " + "%.2f s total, avg in=%.2f MB/s out=%.2f MB/s ===\n", + sd_total_ok, sd_n, sd_total_in, sd_total_out, sd_total_secs, + sd_total_secs > 0 ? (sd_total_in / 1e6) / sd_total_secs : 0, + sd_total_secs > 0 ? (sd_total_out / 1e6) / sd_total_secs : 0); + +#if PASS_B_ENABLE + printf("\n--- Pass B: request granularity, amplification, random vs sequential ---\n"); + printf("libchdr reads whole hunks only (no sub-hunk API, no decoded-hunk cache),\n" + "so a sub-hunk request costs a full hunk decode. 'amp' is bytes decoded /\n" + "bytes the caller asked for.\n"); + for (int i = 0; i < sd_n; i++) + pass_b_file(sd_paths[i]); +#endif + + for (int i = 0; i < sd_n; i++) + free(sd_paths[i]); + + /* explicit end-of-run marker so the serial capture knows when to stop + * without guessing from the last section's heading */ + printf("\n=== BENCH COMPLETE ===\n"); +} diff --git a/contrib/esp32p4/idf-benchmark/partitions.csv b/contrib/esp32p4/idf-benchmark/partitions.csv new file mode 100644 index 0000000..6fcf3de --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/partitions.csv @@ -0,0 +1,4 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 6M, diff --git a/contrib/esp32p4/idf-benchmark/sdkconfig.defaults b/contrib/esp32p4/idf-benchmark/sdkconfig.defaults new file mode 100644 index 0000000..3911c6f --- /dev/null +++ b/contrib/esp32p4/idf-benchmark/sdkconfig.defaults @@ -0,0 +1,43 @@ +# Waveshare ESP32-P4-NANO: 16MB NOR flash, 32MB PSRAM (in-package, not used +# by this benchmark - everything here fits in internal 768KB HP L2MEM). +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + +# LZMA/zstd dictionaries and the FLAC/huffman decoders' local buffers can run +# deep call chains - default main task stack (3.5KB) is cut close for that, +# same margin tests/esp32p4/fw.c's QEMU probe already measured peak-heap +# against (this only affects *stack*, not the numbers printed there). +CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384 + +# esp_timer_get_time() is the only ESP-IDF facility this benchmark needs - +# everything else (Wi-Fi/BT co-processor, Ethernet, USB host, SD/MMC) stays +# off since none of it is exercised by the flash-embedded corpus. +CONFIG_ESP_TIMER_TASK_STACK_SIZE=2048 + +# Heap poisoning was a diagnostic for the decompression failures that turned +# out to be an ESP ROM miniz symbol collision (see components/libchdr/ +# CMakeLists.txt). Root-caused, so it is off: it costs on every malloc/free, +# and the CD-FLAC path allocates ~40KB per hunk. +CONFIG_HEAP_POISONING_DISABLED=y + +# Onboard TF/SD card slot (native SDMMC slot 0, GPIO43/44/39-42, LDO chan 4) +# - the real deployment storage path, not the flash-embedded corpus above. +CONFIG_FATFS_LFN_HEAP=y +CONFIG_FATFS_MAX_LFN=255 + +# Bottleneck sweep (Pass C) result: -O2 is worth 1.28x on pure decode and +# 1.10x end-to-end from SD; heap poisoning off adds 1.01x; raising the SD +# clock from the 20MHz default to 40MHz adds only 1.05x - the workload is +# decode-bound, not bus-bound. Cumulative 1.17x from SD, 1.28x decode-only. +CONFIG_COMPILER_OPTIMIZATION_PERF=y + +# FatFs walks the FAT cluster chain on every f_lseek, and restarts from the +# first cluster on any *backward* seek. libchdr's COMPRESSION_SELF hunks are +# always backward references (measured: 100% of them, across all 14 sample +# files), so a self-ref-heavy CHD pays a full chain walk per hunk. On a +# 271328-hunk, 29.2%-self-ref file this took the read from 2.7 to 85 ms/hunk +# and made the run look hung. The CLMT makes lseek O(1): same 14 minutes then +# covered hunk 25000 vs 245000, ~10x. Every file on this card is a single +# fragment, so the default 64-word table is ~20x larger than needed here. +CONFIG_FATFS_USE_FASTSEEK=y diff --git a/docs/perf-esp32p4-findings.md b/docs/perf-esp32p4-findings.md new file mode 100644 index 0000000..6d23fa9 --- /dev/null +++ b/docs/perf-esp32p4-findings.md @@ -0,0 +1,416 @@ +# libchdr on ESP32-P4: what costs what, and what does not help + +Everything here is measured, on a Waveshare ESP32-P4-NANO at 400 MHz, against +real CHD files. Where a number came from a desktop proxy that is said +explicitly. The point of the document is to stop good-sounding ideas being +re-tried: most of the things in the "does not help" section look obviously +correct on paper and are losses in practice. + +Companion to `contrib/esp32p4/idf-benchmark/README.md`, which covers the +storage stack. This file is about CPU. + +## The board + + -march=rv32imafc_zicsr_zifencei_xesppie -mabi=ilp32f, 400 MHz + single-issue, in-order, no Zbb + +Measured with a dependent-chain probe (`idf.py -DBENCH_MULPROBE=1`): + +| op | cycles | +|---|---| +| add, xor | 1.25 | +| mul, mulh | 2.13 | +| `heap_caps_malloc` + `heap_caps_free` | 4.08 us, flat 4 KB..224 KB | +| `memset` 32 KB | 30.9 us | + +`xesppie` is Espressif's 128-bit vector extension. The assembler accepts +`esp.vld.128.ip` / `esp.vadd.s8`, but **GCC exposes no intrinsics or builtins**, +and it needs CSR 0x7F2 enablement plus context-switch state. Any use means hand +written assembly. + +**ESP32-S3 is Xtensa LX7, not RISC-V.** None of the RV32 work transfers. Its +128-bit SIMD is a TIE extension GCC does not auto-vectorise to. + +## One model explains every codec + +Profiling the seed CHDs the board actually runs +(`tests/corpus/seeds/cd_{none,cdzs,cdzl,cdlz,cdfl}.chd` - same content, one +codec each), x86-64 instruction counts predict P4 wall clock with a single +slope: + +| codec | x86 Ir/hunk | predicted | measured | err | +|---|---|---|---|---| +| cd_none | 21,137 | 1.218 ms | 1.218 ms | fit | +| cd_cdzs | 180,877 | 1.736 | 1.901 | -8.7% | +| cd_cdzl | 209,849 | 1.830 | 1.945 | -5.9% | +| cd_cdlz | 341,315 | 2.257 | 2.303 | -2.0% | +| cd_cdfl | 1,213,474 | 5.087 | 5.087 | fit | + +**1.30 cycles per instruction at 400 MHz**, covering a 57x spread in cost. There +is no codec-specific penalty on this core. FLAC is not slow because of RISC-V; +it executes ~127 instructions per audio sample and that is all. + +## What shipped, and what it bought + +Measured on-board, ms/hunk, identical content, cumulative: + +| codec | before | + ECC SWAR | + CRC slice-4 | total | +|---|---|---|---|---| +| cd_cdzl | 1.945 | 1.926 | 1.656 | **1.175x** | +| cd_cdzs | 1.901 | 1.907 | 1.645 | **1.156x** | +| cd_cdlz | 2.303 | 2.304 | 2.038 | **1.130x** | +| cd_cdfl | 5.087 | 5.018 | 4.753 | **1.070x** | +| cd_none | 1.218 | 1.220 | 1.223 | 0.996x | + +Real discs: Pyramid Plunder 468.58 -> 409.54 ms (1.144x), Hawiian Island Girls +916.64 -> 847.72 ms (1.081x). + +`cd_none` does not move because an uncompressed CHD takes an early path that +reaches neither ECC nor the CRC. + +### crc16 slice-by-4 (commit d0ff583) - the biggest single lever + +`crc16_update` was a byte-at-a-time table walk: **12 instructions per byte on +RV32**, two of them purely truncating the accumulator back to `uint16_t`. It +runs over every decoded hunk under `VERIFY_BLOCK_CRC`, costing ~0.76 ms per +19584-byte hunk - *more than the zstd decode it was checking*. + +Slice-by-4 folds four bytes per iteration through three extra tables +(`s_table1/2/3` = `s_table` advanced one, two and three byte positions), so four +lookups XOR into one result. **6.25 instructions per byte**, +1536 B rodata. + +This beats the ECC work because it runs on **every compressed hunk of every +codec on every platform**, where ECC only fires on data sectors whose parity +chdman stripped. + +Verified identical for every length 0..4096 **and 256 different starting CRCs** - +the latter matters because the CHD v5 map CRC chains a running value rather than +restarting from 0xffff. + +### ECC P parity, SWAR (commit f1a0600) + +Three exact algebraic identities, verified over their full domain: + + ecclow[x] == xtime(x) in GF(2^8), poly 0x11d (all 256 entries) + poffsets[r][c] == r + 86c + qoffsets[r][c] == (86*(r>>1) + (r&1) + 88c) mod 2236 + +The P rows are independent and, for a fixed component, read consecutive bytes. +So four rows pack into one 32-bit accumulator and the per-component step goes +word-wide, using arithmetic `xtime` because a table lookup cannot be packed +(no gather). This is SWAR, not SIMD: no vector ISA, no intrinsics, portable. + +Measured `ecc_generate` over a 2352-byte sector: 47,776 -> 32,513 cycles, +**1.469x**. + +Bytes are assembled from four loads rather than read as a word: the P stride is +86 and `86 % 4 == 2`, so consecutive components alternate 4-aligned and +2-aligned whatever the caller's buffer alignment. **This cannot be padded away.** +Measured, the byte-built word beats both an `lw`/`lhu` split by alignment parity +and a padded aligned copy. + +**Q cannot be done this way** - its offsets are a diagonal, contiguous in +neither dimension, and it is the larger half (2236 of 4300 components/sector). +That caps any P-side work at ~1.76x on ECC. + +### Other shipped work + +- **dr_flac 0.13.3 -> 0.13.4** (commit 0765eec). Correctness, not speed. Fixes a + heap buffer overflow in `drflac__realloc_from_callbacks` (copied `szOld` bytes + into a smaller buffer when shrinking), and a discarded decode result that let + a failed subframe report success. Both matter: CHDs are attacker-supplied. +- **CMake now refuses `CHDR_WANT_RAW_DATA_SECTOR=OFF` with + `CHDR_VERIFY_BLOCK_CRC=ON`** (commit 86a4404). The stored CRC covers the + reconstituted hunk, so skipping ECC regeneration can never match it - and the + failure is *content-dependent* (a hunk holding only audio frames has no ECC to + regenerate and still verifies), so it read as sporadic file corruption. + +## Does not help - measured, do not re-try + +### ECC: every other rewrite is slower + +`idf.py -DBENCH_MULPROBE=1 -DBENCH_ECCPROBE=1`. Cycles per 2352-byte sector, all +producing identical parity: + +| variant | cycles | vs shipped | +|---|---|---| +| A table offsets + table xtime (was shipped) | 47,776 | 1.000x | +| B table offsets + arithmetic xtime | 64,635 | 0.739x | +| C closed-form offsets + table xtime | 50,543 | 0.945x | +| D both | 73,453 | 0.650x | +| E P row-inner, auto-vectorisable | 50,084 | 0.954x | +| **F P SWAR 4x, byte-built word** | **32,513** | **1.469x** | +| G P SWAR 4x, aligned lw/lhu | 32,850 | 1.454x | + +**One `lbu` from a hot 256-byte table beats six ALU ops** on a single-issue +in-order core. In-loop the table is 3 instructions (base hoisted); arithmetic +`xtime` is 6. x86 instruction counting predicts the opposite (+17.6% Ir for +arithmetic) and is the wrong metric for a load-latency question. + +The 8856 bytes of tables *are* still buyable: variant C costs 5.7% of ECC for +all of `poffsets`+`qoffsets` (8600 B). A size option for flash-tight targets, +never a speed change. + +**Variant E is target-divergent** and is the trap here. The same restructure +auto-vectorises to 9.7x on x86-64 SSE2, 17.7x with AVX2, 7.84x on aarch64 NEON - +but needs `-O3` (1.08x at `-O2`) and **loses** 4.6% on the P4 and 22% on aarch64 +at `-O2`, because without a vector unit the accumulators spill from registers to +a stack array. SWAR wins everywhere and needs no gating. + +### FLAC: six hypotheses, all dead + +1. **Not clz / missing Zbb.** dr_flac sets `DRFLAC_NO_CPUID` off x86/ARM, so + `gIsLZCNTSupported` stays false, GCC folds the branch, and the compiled RV32 + object contains **no `__clzsi2` reference at all**. It already uses the + inline software clz. +2. **Not SIMD.** Default x86-64 builds compile no sse41 rice variant either, so + desktop profiles were always the same `__scalar` path RISC-V takes. Forcing + `-msse4.1` buys 1.20x on the function, 1.06x overall. +3. **Not I-cache**, despite `rice__scalar` being 23,206 B on RV32 against + LzmaDec's 4,532. It is specialised by lpcOrder/riceParam so one call touches + a few KB. Cachegrind at a simulated 16 KB L1I: cdlz 0.02% miss rate, cdfl + 0.05%. +4. **Not multiply latency.** mul/mulh 2.13 vs add 1.25 cycles, 12.6% static + multiply density in the rice loop vs 2.9% in LzmaDec, predicting ~6.7%. +5. **Not FPU.** FLAC decode is pure integer; libchdr uses the s16 path. +6. **Not the allocator.** dr_flac allocates ~one 32-64 KB block per FLAC hunk + (0.82 allocs + 34.7 KB per cdfl hunk, against 0.017 allocs and 714 bytes for + cdlz - 39x the rate). At 4.08 us that is **0.15% of a cdfl hunk**, 0.28% of + the whole cdfl-to-cdlz gap. This also settles the reverted drflac arena: a + RAM fix, never a throughput one - a perfect arena returns under 8 us/hunk. +7. **Not the 64-bit LPC path.** I believed CD audio forced + `drflac__calculate_prediction_64`. Instrumenting the real decision point over + **4,729 subframes on three discs: 0.0%**. Max LPC order observed is 12; + chdman's precision keeps `bps + precision + ilog2(order)` <= 32. dr_flac + already uses the 32-bit path exclusively. + +### PIE (ESP32-P4 vector extension): not worth it + +Four independent reasons. The LPC recurrence is serial across samples; only the +8-12-tap dot product is parallel, and each sample needs an unaligned 16-byte +history load where `esp.vld.128` wants 16-byte alignment. Under mid-side coding +the side subframe is 17-bit while PIE multiplies are s16 lanes, so half the +subframes cannot use it. Rice extraction, the other ~40%, is bit-serial and +data-dependent. And it is hand assembly with no compiler support inside a +vendored file. Ceiling ~12-25% for a large, unportable, unmaintainable change. + +### DRFLAC_64BIT: expected negative + +`drflac_cache_t` is `uint32` on RV32 (`DRFLAC_64BIT` is only defined for LP64), +so the bit-reader cache refills twice as often as on x86-64. Forcing it grows +`rice__scalar` from 7,470 to 8,465 instructions, because every variable 64-bit +shift on RV32 becomes a branch (jump tables go 8 -> 29). The rice path does +three variable-width cache shifts per sample against 0.3 reloads per sample; the +arithmetic does not work. Untested on hardware, expected -5 to -15%. + +## Still open + +Both are in libchdr's own code, neither started. + +### 1. `DR_FLAC_NO_CRC` (~8% of cdfl, and 13 KB of flash) + +dr_flac CRCs every FLAC frame, on top of libchdr's own hunk CRC. Measured on +RV32: + +| | text | data | bss | +|---|---|---|---| +| CRC on | 64,859 | 80 | 0 | +| `DR_FLAC_NO_CRC` | 51,765 | 80 | 0 | + +**-13,094 B of flash (-20%)**; `rice__scalar` shrinks 23,206 -> 19,056; the +compiler drops all three CRC tables. Note `data`/`bss` are unchanged: this saves +**flash, not RAM**. + +**Gate it on `VERIFY_BLOCK_CRC`, not on `LOWRAM_TARGET`.** What makes it safe is +that libchdr re-checks the whole decoded hunk against chdman's CRC; that is +`VERIFY_BLOCK_CRC`. Gating on `LOWRAM_TARGET` would permit +`LOWRAM_TARGET=1 + VERIFY_BLOCK_CRC=0`, which removes the *only* integrity check +on FLAC data - the same bug class the RAW/CRC CMake guard just closed. And +`LOWRAM_TARGET` is a RAM-vs-CPU trade for the hunk map, so the name would lie +about a flash saving. + +Place it in `src/libchdr_flac.c` beside the existing `DR_FLAC_NO_STDIO`, which +keeps it out of the vendored header. + +Seeking is not a concern: libchdr supplies a seek *callback* but never calls +`drflac_seek_to_pcm_frame`. + +Open question for the reviewer: dr_flac's CRC catches corruption one level +earlier, before garbage is decoded. A CRC is an integrity check and not a +memory-safety barrier - dr_flac's bounds checks are independent - but the +trade is a judgement call. + +### 2. The cdfl wrapper copies the audio three times (~5-6%) + +Per hunk, 18,816 audio bytes are written three times: + + dr_flac -> stack buffer[2352] write #1 libchdr_flac.c + -> cdfl->buffer, byteswapped read+write #2 write_callback + -> dest, memcpy 2352 per sector read+write #3 cdfl:158 + +The stack buffer exists *only* so `flac_decoder_write_callback` has somewhere to +read from while byteswapping. The final memcpy exists because `dest` has a +2448-byte stride (2352 audio + 96 subcode) while decode output is contiguous - +but `flac_decoder_decode_interleaved` already chunks at exactly 588 frames = +one sector = 2352 bytes, so each chunk could be decoded straight into +`dest + framenum*CD_FRAME_SIZE`. + +The byteswap itself is also poor: a nested per-channel loop with `sampch` and +`shift` as runtime values, ~8-10 instructions per sample. Word-wise in place is +2: `w = ((w & 0x00FF00FF) << 8) | ((w >> 8) & 0x00FF00FF)`. + +Risks: `libchdr_codec_flac.c` and `libchdr_codec_avhuff.c` share this wrapper. +AVHuff uses the **non-interleaved** multi-stream branch and has the thinnest +test coverage (4 files). Subcode still needs placing at offset 2352 of each 2448 +frame. + +## Methodology notes, learned the hard way + +- **Never anchor a fit on the point under test.** The first version of the + cycles-per-instruction table anchored the slope on `cd_none` *and* `cd_cdfl`, + so cdfl's 0% error was true by construction and the fit could not have + detected a FLAC-specific cost. Refit on non-FLAC codecs only, cdfl + extrapolates to 5.332 ms against 5.087 measured. +- **Compare the same file.** A "2.4x unexplained" residual turned out to be an + artifact of comparing a self-made corpus against on-target numbers from a + *different* disc. Codec cost ratios are content dependent. +- **x86 instruction counts are the wrong metric for load-latency questions** on + an in-order core, and they cannot see the target's allocator, its byteswap + lowering, or its cache width. They predicted the opposite result for ECC. +- **Static instruction counts include dead specialisations.** The 553 `mul` + + 390 `mulh` that made the 64-bit LPC path look hot were in a branch that never + executes. +- **A flat callgrind profile can be an annotator artifact.** "No hot line" in + `rice__scalar` was because everything hot is inlined from elsewhere and + attributed to *those* source lines, outside the function's own range. +- **Synthetic seeds are not real content.** `tests/corpus/seeds/cd_*.chd` do + **0.0%** ECC work, so they cannot measure an ECC change; ECC only runs on + frames whose parity chdman stripped. Of 14 real discs on the test card, three + do 0% ECC (a PSP ISO and two hard-disk CHDs) and the rest range 2-35%. +- **Cap size hides defects.** A 600-hunk cap hid a huffman bug; a 3000-hunk cap + hid a missing FatFs cluster map. + +## Reproducing + + # op latency, allocator, ECC variants + idf.py -DBENCH_MULPROBE=1 -DBENCH_ECCPROBE=1 build flash monitor + + # decode-output equivalence over a CHD corpus + # (strided sampling - capping from hunk 0 never reaches the audio tracks + # that follow the data track, so a naive cap validates nothing about FLAC) + +Correctness bar used throughout: decoded output byte-identical over **287 CHDs**, +with `VERIFY_BLOCK_CRC` checking every hunk against chdman's own CRC, plus the +AVHuff regression suite (4/4). + +## End-to-end result on the full corpus + +Same 14 SD files, same FatFs + cluster-map + 64 KB read-ahead configuration, +both uncapped over 7.32 GB of decoded output: + +| | time | throughput | +|---|---|---| +| before the CPU work on this branch | 2307.0 s | 3.17 MB/s | +| after ECC SWAR + crc16 slice-by-4 | **1924.7 s** | **3.80 MB/s** | + +**1.20x end to end**, 14/14 files OK. Per file, uncapped: + +| file | hunks | MB/s | CPU | IO | +|---|---|---|---|---| +| Ikaruga | 68,645 | 12.71 | 97.1% | 2.9% | +| naomi vathlete | 68,645 | 9.55 | - | - | +| kinst2 | 111,737 | 6.49 | 68.1% | 31.9% | +| Sensible Soccer | 16,400 | 6.33 | - | - | +| Insanity | 11,825 | 4.74 | 79.5% | 20.5% | +| Shadowrun | 13,291 | 3.69 | - | - | +| Castlevania X | 271,328 | 3.07 | 30.7% | 69.3% | +| Surgical Strike | 25,707 | 1.87 | - | - | +| SS-parodius | 17,393 | 1.69 | 84.9% | 15.1% | + +Every CD image decodes faster than the drive it shipped on: Ikaruga 69x CD +(5.8x a Dreamcast GD-ROM), Insanity 25.8x (a 1x PC Engine CD drive), +SS-parodius 9.2x (4.6x a 2x Saturn drive), Castlevania X 1.7x a PSP UMD. + +## Robustness: three latent bugs, all pre-existing on master + +Found by fuzzing under ASan+UBSan. None affect well-formed files; all are +reachable from a malformed one, which matters because CHDs are attacker-supplied. + +1. **Signed overflow in the header parser** (149dc57). `get_bigendian_uint32_t` + did `base[0] << 24` without casting - uint8_t promotes to int, so any byte + >= 0x80 is UB. Real files never hit it because every field read through it + is a small count or an ASCII four-char codec tag. The uint48 and uint64 + readers already cast; this one was inconsistent. +2. **Unvalidated bit widths from the file** (56273ca). The v5 map header's + `lengthbits`/`selfbits`/`parentbits` are raw bytes that become the width + argument to `bitstream_read()`. Above 32 they make `bitstream_peek()` shift + by a negative amount. Now rejected as `CHDERR_INVALID_FILE` at parse time. +3. **Shift by 32 refilling an over-consumed bitstream** (598424f). `bits` goes + negative after over-consumption, so `24 - bits` reaches 32. Only reachable + with `LOWRAM_TARGET=OFF` - the lazy checkpointed map reaches the huffman + decoder differently - so fuzzing that covered only the low-RAM path missed + it. **Fuzz both map implementations.** + +Bar now established, all clean: + +- **3412 malformed inputs** (mutations of all 17 seed codecs across header, + map-region, whole-file and truncation strategies, plus 131 structure-aware + header-field cases) against **both** `LOWRAM_TARGET=ON` and `OFF`, on + **both 64-bit and 32-bit** builds. +- **546 metadata-targeted inputs** including explicit self-referential cycles + in the metadata chain, through `chd_get_metadata()`, on both map paths. +- **API misuse**: NULL handles, out-of-range hunks, cache budget 0/1/1TB, + budget toggled mid-stream (output identical), OOM path. No leaks. +- **Access-order equivalence**: HEAD vs master byte-identical over sequential, + reverse, random and scattered-sample orders. +- **Config matrix**: LOWRAM on/off, system zlib, system zstd and LTO all + produce byte-identical output; `WANT_RAW_DATA_SECTOR=OFF` differs as designed. +- **32-bit vs 64-bit** decode byte-identical. +- Compiles clean for **Cortex-M33** (68,816 B .text) and **Cortex-M0+** + (77,976 B), i.e. both RP2350 cores. + +Incidental: no CD image sampled from the corpus has any nonzero subcode, which +is why `CHDR_WANT_SUBCODE=OFF` is output-identical on them - 96 of every 2448 +bytes is zeros being decompressed and copied. + +## Self-reference locality, and why a bigger decoded-hunk cache is not worth it + +Castlevania X is 29.2% self-referential over its 271,328 hunks - but only +**10.0% over the first 2500**, which is why a capped run reads 4.61 MB/s and the +full disc 3.07. Not a regression; the capped number was unrepresentative. + +LRU simulation over the real 79,305-self-reference stream: + +| entries | RAM at 4 KB hunks | hit % of self-refs | +|---|---|---| +| **1** | **4 KB** | **21.1%** | +| 16 | 64 KB | 25.3% | +| 64 | 256 KB | 27.0% | +| 256 | 1 MB | 31.6% | +| 4096 | 16 MB | 44.4% | +| 65536 | 256 MB | 99.7% | + +One entry already captures most of the benefit because **24.0% of +self-references point at the same target as the previous one**. 99.2% are more +than 16 hunks back, with the mode at 2^14-2^15 - 16k to 64k hunks, 64-256 MB +into the file - so no small cache can reach them. The single-entry cache that +ships under LOWRAM is at the knee. 16 MB of PSRAM would avoid 9.5% of reads for +perhaps 6-7% of wall time. + +## Portability of the two perf changes + +Measured, not extrapolated (aarch64 under qemu, so ratios not absolute times): + +| | x86-64 -O2 | x86-64 -O3 | aarch64 -O2 | aarch64 -O3 | RV32 | +|---|---|---|---|---|---| +| crc16 slice-4 (shipped) | 3.55x | 3.52x | 1.97x | 2.42x | 1.9x | +| crc16 **slice-8** (not done) | **6.57x** | **6.42x** | 2.35x | 2.70x | n/a | +| ECC P SWAR32 (shipped) | 2.54x | 2.70x | 4.41x | 4.54x | 1.47x | +| ECC P row-inner (rejected) | 0.79x | 8.36x | 0.84x | 8.19x | 0.95x | + +SWAR32 wins on every target at both optimisation levels, which is why it +shipped; row-inner is far better at -O3 with a vector unit but **loses** at -O2 +and on RV32, and distro packages are frequently -O2. + +**Slice-by-8 is a free 1.85x over slice-4 on 64-bit** for +2 KB of rodata. Not +implemented - it only pays on 64-bit and would add a second code path. diff --git a/include/dr_libs/dr_flac.h b/include/dr_libs/dr_flac.h index 2891194..051f37c 100644 --- a/include/dr_libs/dr_flac.h +++ b/include/dr_libs/dr_flac.h @@ -1,6 +1,6 @@ /* FLAC audio decoder. Choice of public domain or MIT-0. See license statements at the end of this file. -dr_flac - v0.13.3 - 2026-01-17 +dr_flac - v0.13.4 - TBD David Reid - mackron@gmail.com @@ -126,7 +126,7 @@ extern "C" { #define DRFLAC_VERSION_MAJOR 0 #define DRFLAC_VERSION_MINOR 13 -#define DRFLAC_VERSION_REVISION 3 +#define DRFLAC_VERSION_REVISION 4 #define DRFLAC_VERSION_STRING DRFLAC_XSTRINGIFY(DRFLAC_VERSION_MAJOR) "." DRFLAC_XSTRINGIFY(DRFLAC_VERSION_MINOR) "." DRFLAC_XSTRINGIFY(DRFLAC_VERSION_REVISION) #include /* For size_t. */ @@ -709,6 +709,9 @@ onRead (in) onSeek (in) The function to call when the read position of the client data needs to move. +onTell (in) + The function to call when the read position of the client needs to be queried. + pUserData (in, optional) A pointer to application defined data that will be passed to onRead and onSeek. @@ -759,6 +762,9 @@ onRead (in) onSeek (in) The function to call when the read position of the client data needs to move. +onTell (in) + The function to call when the read position of the client needs to be queried. + container (in) Whether or not the FLAC stream is encapsulated using standard FLAC encapsulation or Ogg encapsulation. @@ -800,6 +806,9 @@ onRead (in) onSeek (in) The function to call when the read position of the client data needs to move. +onTell (in) + The function to call when the read position of the client needs to be queried. + onMeta (in) The function to call for every metadata block. @@ -1547,6 +1556,8 @@ static DRFLAC_INLINE drflac_bool32 drflac_has_sse41(void) #define DRFLAC_ZERO_OBJECT(p) DRFLAC_ZERO_MEMORY((p), sizeof(*(p))) #endif +#define DRFLAC_MIN(a, b) (((a) < (b)) ? (a) : (b)) + #define DRFLAC_MAX_SIMD_VECTOR_SIZE 64 /* 64 for AVX-512 in the future. */ /* Result Codes */ @@ -5367,6 +5378,7 @@ static drflac_bool32 drflac__decode_subframe(drflac_bs* bs, drflac_frame* frame, { drflac_subframe* pSubframe; drflac_uint32 subframeBitsPerSample; + drflac_bool32 decodeResult; DRFLAC_ASSERT(bs != NULL); DRFLAC_ASSERT(frame != NULL); @@ -5413,28 +5425,28 @@ static drflac_bool32 drflac__decode_subframe(drflac_bs* bs, drflac_frame* frame, { case DRFLAC_SUBFRAME_CONSTANT: { - drflac__decode_samples__constant(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); + decodeResult = drflac__decode_samples__constant(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); } break; case DRFLAC_SUBFRAME_VERBATIM: { - drflac__decode_samples__verbatim(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); + decodeResult = drflac__decode_samples__verbatim(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); } break; case DRFLAC_SUBFRAME_FIXED: { - drflac__decode_samples__fixed(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); + decodeResult = drflac__decode_samples__fixed(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); } break; case DRFLAC_SUBFRAME_LPC: { - drflac__decode_samples__lpc(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); + decodeResult = drflac__decode_samples__lpc(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); } break; - default: return DRFLAC_FALSE; + default: decodeResult = DRFLAC_FALSE; } - return DRFLAC_TRUE; + return decodeResult; } static drflac_bool32 drflac__seek_subframe(drflac_bs* bs, drflac_frame* frame, int subframeIndex) @@ -5549,6 +5561,7 @@ static drflac_result drflac__decode_flac_frame(drflac* pFlac) #endif /* This function should be called while the stream is sitting on the first byte after the frame header. */ + pFlac->currentFLACFrame.pcmFramesRemaining = 0; DRFLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes)); /* The frame block size must never be larger than the maximum block size defined by the FLAC stream. */ @@ -5596,6 +5609,7 @@ static drflac_result drflac__decode_flac_frame(drflac* pFlac) static drflac_result drflac__seek_flac_frame(drflac* pFlac) { + drflac_result result; int channelCount; int i; drflac_uint16 desiredCRC16; @@ -5603,16 +5617,20 @@ static drflac_result drflac__seek_flac_frame(drflac* pFlac) drflac_uint16 actualCRC16; #endif + pFlac->currentFLACFrame.pcmFramesRemaining = 0; + channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); for (i = 0; i < channelCount; ++i) { if (!drflac__seek_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i)) { - return DRFLAC_ERROR; + result = DRFLAC_ERROR; + goto error; } } /* Padding. */ if (!drflac__seek_bits(&pFlac->bs, DRFLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7)) { - return DRFLAC_ERROR; + result = DRFLAC_ERROR; + goto error; } /* CRC. */ @@ -5620,16 +5638,22 @@ static drflac_result drflac__seek_flac_frame(drflac* pFlac) actualCRC16 = drflac__flush_crc16(&pFlac->bs); #endif if (!drflac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) { - return DRFLAC_AT_END; + result = DRFLAC_AT_END; + goto error; } #ifndef DR_FLAC_NO_CRC if (actualCRC16 != desiredCRC16) { - return DRFLAC_CRC_MISMATCH; /* CRC mismatch. */ + result = DRFLAC_CRC_MISMATCH; /* CRC mismatch. */ + goto error; } #endif return DRFLAC_SUCCESS; + +error: + DRFLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes)); + return result; } static drflac_bool32 drflac__read_and_decode_next_flac_frame(drflac* pFlac) @@ -5950,6 +5974,22 @@ static drflac_bool32 drflac__seek_to_pcm_frame__binary_search_internal(drflac* p } for (;;) { + /* + If only two adjacent byte offsets remain, binary search cannot narrow the range any further. Seek to the closest frame before the target and decode + forward from there. + */ + if ((byteRangeHi - byteRangeLo) == 1) { + if (!drflac__seek_to_approximate_flac_frame_to_byte(pFlac, closestSeekOffsetBeforeTargetPCMFrame, closestSeekOffsetBeforeTargetPCMFrame, byteRangeHi, &lastSuccessfulSeekOffset)) { + break; + } + + if (pFlac->currentPCMFrame <= pcmFrameIndex && drflac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame)) { + return DRFLAC_TRUE; + } + + break; + } + if (drflac__seek_to_approximate_flac_frame_to_byte(pFlac, targetByte, byteRangeLo, byteRangeHi, &lastSuccessfulSeekOffset)) { /* We found a FLAC frame. We need to check if it contains the sample we're looking for. */ drflac_uint64 newPCMRangeLo; @@ -5980,8 +6020,6 @@ static drflac_bool32 drflac__seek_to_pcm_frame__binary_search_internal(drflac* p break; /* Failed to seek to FLAC frame. */ } } else { - const float approxCompressionRatio = (drflac_int64)(lastSuccessfulSeekOffset - pFlac->firstFLACFramePosInBytes) / ((drflac_int64)(pcmRangeLo * pFlac->channels * pFlac->bitsPerSample)/8.0f); - if (pcmRangeLo > pcmFrameIndex) { /* We seeked too far forward. We need to move our target byte backward and try again. */ byteRangeHi = lastSuccessfulSeekOffset; @@ -6004,12 +6042,14 @@ static drflac_bool32 drflac__seek_to_pcm_frame__binary_search_internal(drflac* p break; /* Failed to seek to FLAC frame. */ } } else { + const double approxCompressionRatio = (drflac_int64)(lastSuccessfulSeekOffset - pFlac->firstFLACFramePosInBytes) / ((drflac_int64)(pcmRangeLo * pFlac->channels * pFlac->bitsPerSample)/8.0); + byteRangeLo = lastSuccessfulSeekOffset; if (byteRangeHi < byteRangeLo) { byteRangeHi = byteRangeLo; } - targetByte = lastSuccessfulSeekOffset + (drflac_uint64)(((drflac_int64)((pcmFrameIndex-pcmRangeLo) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * approxCompressionRatio); + targetByte = lastSuccessfulSeekOffset + (drflac_uint64)(((drflac_int64)((pcmFrameIndex-pcmRangeLo) * pFlac->channels * pFlac->bitsPerSample)/8.0) * approxCompressionRatio); if (targetByte > byteRangeHi) { targetByte = byteRangeHi; } @@ -6402,7 +6442,7 @@ static void* drflac__realloc_from_callbacks(void* p, size_t szNew, size_t szOld, } if (p != NULL) { - DRFLAC_COPY_MEMORY(p2, p, szOld); + DRFLAC_COPY_MEMORY(p2, p, DRFLAC_MIN(szNew, szOld)); pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData); } @@ -6430,11 +6470,22 @@ static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, d We want to keep track of the byte position in the stream of the seektable. At the time of calling this function we know that we'll be sitting on byte 42. */ - drflac_uint64 runningFilePos = 42; - drflac_uint64 seektablePos = 0; - drflac_uint32 seektableSize = 0; + drflac_uint64 runningFilePos = 42; + drflac_uint64 seektablePos = 0; + drflac_uint32 seektableSize = 0; + drflac_int64 fileSize = 0; + drflac_bool32 hasKnownFileSize = DRFLAC_FALSE; - (void)onTell; + /* We'll be doing some memory allocations here against untrusted data. We'll do a basic validation check that they don't exceed the size of the file. */ + if (onTell != NULL && onSeek != NULL) { + if (onSeek(pUserData, 0, DRFLAC_SEEK_END)) { + if (onTell(pUserData, &fileSize)) { + hasKnownFileSize = DRFLAC_TRUE; + } + + onSeek(pUserData, (int)runningFilePos, DRFLAC_SEEK_SET); /* Safe cast because runningFilePos should always be 42 at this point. */ + } + } for (;;) { drflac_metadata metadata; @@ -6444,6 +6495,11 @@ static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, d if (drflac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize) == DRFLAC_FALSE) { return DRFLAC_FALSE; } + + if (hasKnownFileSize && (blockSize > ((drflac_uint64)fileSize - runningFilePos))) { + return DRFLAC_FALSE; /* Block size exceeds the size of the file. */ + } + runningFilePos += 4; metadata.type = blockType; @@ -6490,10 +6546,12 @@ static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, d drflac_uint32 seekpointCount; drflac_uint32 iSeekpoint; void* pRawData; + size_t rawDataSize; seekpointCount = blockSize/DRFLAC_SEEKPOINT_SIZE_IN_BYTES; + rawDataSize = seekpointCount * sizeof(drflac_seekpoint); - pRawData = drflac__malloc_from_callbacks(seekpointCount * sizeof(drflac_seekpoint), pAllocationCallbacks); + pRawData = drflac__malloc_from_callbacks(rawDataSize, pAllocationCallbacks); if (pRawData == NULL) { return DRFLAC_FALSE; } @@ -6514,7 +6572,7 @@ static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, d } metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; + metadata.rawDataSize = rawDataSize; metadata.data.seektable.seekpointCount = seekpointCount; metadata.data.seektable.pSeekpoints = (const drflac_seekpoint*)pRawData; @@ -6559,7 +6617,7 @@ static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, d drflac__free_from_callbacks(pRawData, pAllocationCallbacks); return DRFLAC_FALSE; } - metadata.data.vorbis_comment.vendor = pRunningData; pRunningData += metadata.data.vorbis_comment.vendorLength; + metadata.data.vorbis_comment.vendor = pRunningData; pRunningData += metadata.data.vorbis_comment.vendorLength; metadata.data.vorbis_comment.commentCount = drflac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4; /* Need space for 'commentCount' comments after the block, which at minimum is a drflac_uint32 per comment */ @@ -6747,13 +6805,18 @@ static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, d blockSizeRemaining -= 4; metadata.data.picture.mimeLength = drflac__be2host_32(metadata.data.picture.mimeLength); + if (blockSizeRemaining < metadata.data.picture.mimeLength) { + result = DRFLAC_FALSE; + goto done_flac; + } + pMime = (char*)drflac__malloc_from_callbacks(metadata.data.picture.mimeLength + 1, pAllocationCallbacks); /* +1 for null terminator. */ if (pMime == NULL) { result = DRFLAC_FALSE; goto done_flac; } - if (blockSizeRemaining < metadata.data.picture.mimeLength || onRead(pUserData, pMime, metadata.data.picture.mimeLength) != metadata.data.picture.mimeLength) { + if (onRead(pUserData, pMime, metadata.data.picture.mimeLength) != metadata.data.picture.mimeLength) { result = DRFLAC_FALSE; goto done_flac; } @@ -6769,13 +6832,18 @@ static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, d blockSizeRemaining -= 4; metadata.data.picture.descriptionLength = drflac__be2host_32(metadata.data.picture.descriptionLength); + if (blockSizeRemaining < metadata.data.picture.descriptionLength) { + result = DRFLAC_FALSE; + goto done_flac; + } + pDescription = (char*)drflac__malloc_from_callbacks(metadata.data.picture.descriptionLength + 1, pAllocationCallbacks); /* +1 for null terminator. */ if (pDescription == NULL) { result = DRFLAC_FALSE; goto done_flac; } - if (blockSizeRemaining < metadata.data.picture.descriptionLength || onRead(pUserData, pDescription, metadata.data.picture.descriptionLength) != metadata.data.picture.descriptionLength) { + if (onRead(pUserData, pDescription, metadata.data.picture.descriptionLength) != metadata.data.picture.descriptionLength) { result = DRFLAC_FALSE; goto done_flac; } @@ -8094,11 +8162,17 @@ static drflac* drflac_open_with_metadata_private(drflac_read_proc onRead, drflac return NULL; } + if ((0xFFFFFFFF - (seekpointCount * sizeof(drflac_seekpoint))) < allocationSize) { + #ifndef DR_FLAC_NO_OGG + drflac__free_from_callbacks(pOggbs, &allocationCallbacks); + #endif + return NULL; + } + allocationSize += seekpointCount * sizeof(drflac_seekpoint); } - - pFlac = (drflac*)drflac__malloc_from_callbacks(allocationSize, &allocationCallbacks); + pFlac = (drflac*)drflac__malloc_from_callbacks((size_t)allocationSize, &allocationCallbacks); if (pFlac == NULL) { #ifndef DR_FLAC_NO_OGG drflac__free_from_callbacks(pOggbs, &allocationCallbacks); @@ -9813,6 +9887,23 @@ static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_independent_stereo( } +static drflac_bool32 drflac__is_current_flac_frame_valid(drflac* pFlac) +{ + drflac_uint32 iChannel; + + if (pFlac->currentFLACFrame.header.blockSizeInPCMFrames > pFlac->maxBlockSizeInPCMFrames || pFlac->currentFLACFrame.pcmFramesRemaining > pFlac->currentFLACFrame.header.blockSizeInPCMFrames) { + return DRFLAC_FALSE; + } + + for (iChannel = 0; iChannel < pFlac->channels; iChannel += 1) { + if (pFlac->currentFLACFrame.subframes[iChannel].pSamplesS32 == NULL) { + return DRFLAC_FALSE; + } + } + + return DRFLAC_TRUE; +} + DRFLAC_API drflac_uint64 drflac_read_pcm_frames_s32(drflac* pFlac, drflac_uint64 framesToRead, drflac_int32* pBufferOut) { drflac_uint64 framesRead; @@ -11722,23 +11813,25 @@ DRFLAC_API drflac_bool32 drflac_seek_to_pcm_frame(drflac* pFlac, drflac_uint64 p } /* If the target sample and the current sample are in the same frame we just move the position forward. */ - if (pcmFrameIndex > pFlac->currentPCMFrame) { - /* Forward. */ - drflac_uint32 offset = (drflac_uint32)(pcmFrameIndex - pFlac->currentPCMFrame); - if (pFlac->currentFLACFrame.pcmFramesRemaining > offset) { - pFlac->currentFLACFrame.pcmFramesRemaining -= offset; - pFlac->currentPCMFrame = pcmFrameIndex; - return DRFLAC_TRUE; - } - } else { - /* Backward. */ - drflac_uint32 offsetAbs = (drflac_uint32)(pFlac->currentPCMFrame - pcmFrameIndex); - drflac_uint32 currentFLACFramePCMFrameCount = pFlac->currentFLACFrame.header.blockSizeInPCMFrames; - drflac_uint32 currentFLACFramePCMFramesConsumed = currentFLACFramePCMFrameCount - pFlac->currentFLACFrame.pcmFramesRemaining; - if (currentFLACFramePCMFramesConsumed > offsetAbs) { - pFlac->currentFLACFrame.pcmFramesRemaining += offsetAbs; - pFlac->currentPCMFrame = pcmFrameIndex; - return DRFLAC_TRUE; + if (drflac__is_current_flac_frame_valid(pFlac)) { + if (pcmFrameIndex > pFlac->currentPCMFrame) { + /* Forward. */ + drflac_uint32 offset = (drflac_uint32)(pcmFrameIndex - pFlac->currentPCMFrame); + if (pFlac->currentFLACFrame.pcmFramesRemaining > offset) { + pFlac->currentFLACFrame.pcmFramesRemaining -= offset; + pFlac->currentPCMFrame = pcmFrameIndex; + return DRFLAC_TRUE; + } + } else { + /* Backward. */ + drflac_uint32 offsetAbs = (drflac_uint32)(pFlac->currentPCMFrame - pcmFrameIndex); + drflac_uint32 currentFLACFramePCMFrameCount = pFlac->currentFLACFrame.header.blockSizeInPCMFrames; + drflac_uint32 currentFLACFramePCMFramesConsumed = currentFLACFramePCMFrameCount - pFlac->currentFLACFrame.pcmFramesRemaining; + if (currentFLACFramePCMFramesConsumed > offsetAbs) { + pFlac->currentFLACFrame.pcmFramesRemaining += offsetAbs; + pFlac->currentPCMFrame = pcmFrameIndex; + return DRFLAC_TRUE; + } } } @@ -12169,6 +12262,13 @@ DRFLAC_API drflac_bool32 drflac_next_cuesheet_track(drflac_cuesheet_track_iterat /* REVISION HISTORY ================ +v0.13.4 - TBD + - Add a bounds check when allocating memory during metadata processing. + - Fix a possible overflow error when parsing picture metadata. + - Fix an error with seek point parsing. + - Fix a possible deadlock when seeking. + - Fix an error where the decoder can be put into a bad state when seeking fails which then results in a crash when reading and seeking. + v0.13.3 - 2026-01-17 - Fix a compiler compatibility issue with some inlined assembly. - Fix a compilation warning. diff --git a/include/libchdr/chd.h b/include/libchdr/chd.h index c3330b9..5a0688b 100644 --- a/include/libchdr/chd.h +++ b/include/libchdr/chd.h @@ -388,6 +388,34 @@ CHD_EXPORT chd_error chd_open(const char *filename, int mode, chd_file *parent, /* precache underlying file */ CHD_EXPORT chd_error chd_precache(chd_file *chd); +/* Give libchdr a memory budget, in bytes, to spend on internal caching. + * + * 0 (the default) disables it entirely and reproduces the historical + * behaviour exactly. libchdr deliberately does not choose this number + * itself: how much memory is available is a property of the embedding + * system - a desktop, an ESP32 with or without PSRAM, an RP2350 - and not + * something a library can portably discover. + * + * Currently spent on a compressed read-ahead window, which collapses the + * one-seek-plus-one-read-per-hunk access pattern into far fewer, larger + * transfers. That matters when the per-transaction cost of the storage + * stack dominates its per-byte cost, which is the usual case for SD/eMMC + * behind a filesystem. Sequential reads transfer each byte exactly once + * regardless of the budget, so a larger budget trades memory for fewer + * transactions and never for redundant I/O. + * + * May be called at any time on an open file; lowering or zeroing it frees + * immediately. Returns CHDERR_OUT_OF_MEMORY if the budget could not be + * allocated, in which case caching stays off and the file remains fully + * usable. A budget below one hunk is rounded up, since a smaller window + * could never serve a read. */ +CHD_EXPORT chd_error chd_set_cache_budget(chd_file *chd, size_t bytes); +CHD_EXPORT size_t chd_get_cache_budget(const chd_file *chd); + +/* Read-ahead window hit/miss counts since the budget was last set. For + * tuning and diagnostics; either pointer may be NULL. */ +CHD_EXPORT void chd_get_cache_stats(const chd_file *chd, uint64_t *hits, uint64_t *misses); + /* close a CHD file */ CHD_EXPORT void chd_close(chd_file *chd); diff --git a/include/libchdr/flac.h b/include/libchdr/flac.h index 5022d1f..1ecf586 100644 --- a/include/libchdr/flac.h +++ b/include/libchdr/flac.h @@ -36,6 +36,7 @@ struct _flac_decoder { uint32_t uncompressed_offset; /* current position in uncompressed data */ uint32_t uncompressed_length; /* length of uncompressed data */ int uncompressed_swap; /* swap uncompressed sample data */ + int alloc_failed; /* set when the last reset() failed to allocate */ uint8_t custom_header[0x2a]; /* custom header */ }; diff --git a/src/codec_cdzs.h b/src/codec_cdzs.h index 2c81441..a34d33d 100644 --- a/src/codec_cdzs.h +++ b/src/codec_cdzs.h @@ -11,9 +11,40 @@ typedef struct _cdzs_codec_data cdzs_codec_data; struct _cdzs_codec_data { + /* One decompression context serves both the sector-data and subcode + * streams under LOWRAM_TARGET, halving the codec's context memory. + * + * Safe because the two streams are decoded strictly in sequence by + * cd_codec_decompress() - base to completion, error returns early, then + * subcode - never nested and never concurrently, into disjoint halves of + * `buffer`; and because the decompress entry point re-initialises the + * context on entry, so no state carries from one stream to the other. + * + * It is not free: alternating two differently-shaped streams through one + * context costs measurably more CPU than giving each its own, because the + * context's working set is rebuilt each way: measured on x86-64 over a + * whole file, +3.9%. A ZSTD_DCtx is ~94KB, so on a memory-constrained + * part that is a good trade and on a desktop it is not - hence + * LOWRAM_TARGET, which exists to make exactly this choice. A default + * build keeps two contexts and its previous speed. + * + * The same sharing was tried for cdzl and reverted: an inflate context is + * only ~8KB once the unused 32KB dictionary is gone, so paying 2.6% CPU + * for it is not worth it. + * + * NOTE for anyone adding threading (see the pread work in PR #162): + * codec state is already shared across concurrent chd_read() calls, but + * this removes even the accidental separation between base and subcode. + * Per-thread codec state has to mean per-thread codec instances. */ +#if LOWRAM_TARGET + zstd_codec_data base_decompressor; +#define cdzs_subcode_ctx(c) (&(c)->base_decompressor) +#else zstd_codec_data base_decompressor; #if WANT_SUBCODE zstd_codec_data subcode_decompressor; +#endif +#define cdzs_subcode_ctx(c) (&(c)->subcode_decompressor) #endif uint8_t* buffer; }; diff --git a/src/codec_zlib.h b/src/codec_zlib.h index e4f4baf..a186dae 100644 --- a/src/codec_zlib.h +++ b/src/codec_zlib.h @@ -1,41 +1,62 @@ -#ifndef LIBCHDR_CODEC_ZLIB_H -#define LIBCHDR_CODEC_ZLIB_H - -#include - -#if defined(__PS3__) || defined(__PSL1GHT__) -#define __MACTYPES__ -#endif -#ifdef CHDR_SYSTEM_ZLIB -#include -typedef uInt zlib_alloc_size; -#else -#include "../deps/miniz-3.1.2/miniz.h" -typedef size_t zlib_alloc_size; -#endif - -#include "../include/libchdr/chd.h" - -/* codec-private data for the ZLIB codec */ -#define MAX_ZLIB_ALLOCS 64 - -typedef struct _zlib_allocator zlib_allocator; -struct _zlib_allocator -{ - uint32_t * allocptr[MAX_ZLIB_ALLOCS]; - uint32_t * allocptr2[MAX_ZLIB_ALLOCS]; -}; - -typedef struct _zlib_codec_data zlib_codec_data; -struct _zlib_codec_data -{ - z_stream inflater; - zlib_allocator allocator; -}; - -/* zlib compression codec */ -chd_error zlib_codec_init(void *codec, uint32_t hunkbytes); -void zlib_codec_free(void *codec); -chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); - -#endif /* LIBCHDR_CODEC_ZLIB_H */ +#ifndef LIBCHDR_CODEC_ZLIB_H +#define LIBCHDR_CODEC_ZLIB_H + +#include + +#if defined(__PS3__) || defined(__PSL1GHT__) +#define __MACTYPES__ +#endif +#ifdef CHDR_SYSTEM_ZLIB +#include +typedef uInt zlib_alloc_size; +#else +#include "../deps/miniz-3.1.2/miniz.h" +typedef size_t zlib_alloc_size; +#endif + +#include "../include/libchdr/chd.h" + +/* codec-private data for the ZLIB codec */ +#define MAX_ZLIB_ALLOCS 64 + +typedef struct _zlib_allocator zlib_allocator; +struct _zlib_allocator +{ + uint32_t * allocptr[MAX_ZLIB_ALLOCS]; + uint32_t * allocptr2[MAX_ZLIB_ALLOCS]; +}; + +typedef struct _zlib_codec_data zlib_codec_data; +struct _zlib_codec_data +{ +#ifdef CHDR_SYSTEM_ZLIB + z_stream inflater; + zlib_allocator allocator; +#else + /* With bundled miniz we drive tinfl directly rather than going through + * the mz_inflate*() wrappers. Those allocate miniz's inflate_state, which + * embeds a fixed 32KB LZ dictionary (m_dict) - 41168 bytes per instance + * against tinfl_decompressor's 8376. + * + * The dictionary is dead weight here. libchdr always decompresses a hunk + * as one complete stream into a buffer large enough to hold all of it, so + * mz_inflate() was already passing TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF + * and tinfl was using the caller's output buffer as its own dictionary; + * m_dict was allocated and never touched. miniz also refuses any window + * size other than +/-15, so the wrappers give no way to ask for less. + * + * A CD-flavoured CHD instantiates this three or four times (cdzl needs one + * for sector data and one for subcode; cdlz and cdfl each need one for + * subcode), so dropping the dictionary saves ~32KB apiece - measured + * against a 254KB peak for a three-codec CD file. Heap-allocated rather + * than inline because chd_file embeds every codec's state by value. */ + tinfl_decompressor * inflater; +#endif +}; + +/* zlib compression codec */ +chd_error zlib_codec_init(void *codec, uint32_t hunkbytes); +void zlib_codec_free(void *codec); +chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +#endif /* LIBCHDR_CODEC_ZLIB_H */ diff --git a/src/libchdr_bitstream.c b/src/libchdr_bitstream.c index de67221..4fb1c63 100644 --- a/src/libchdr_bitstream.c +++ b/src/libchdr_bitstream.c @@ -51,8 +51,16 @@ uint32_t bitstream_peek(struct bitstream* bitstream, int numbits) { while (bitstream->bits <= 24) { - if (bitstream->doffset < bitstream->dlength) - bitstream->buffer |= (uint32_t)bitstream->read[bitstream->doffset] << (24 - bitstream->bits); + /* bits goes negative once a stream has been over-consumed, which + * malformed input can provoke, and then 24 - bits reaches 32 and + * the shift is undefined. A byte shifted that far lands entirely + * above bit 31, so contributing nothing is also the arithmetically + * correct result - well-formed streams keep bits >= 0 and never + * take this branch. */ + const int shift = 24 - bitstream->bits; + + if (bitstream->doffset < bitstream->dlength && shift < 32) + bitstream->buffer |= (uint32_t)bitstream->read[bitstream->doffset] << shift; bitstream->doffset++; bitstream->bits += 8; } @@ -71,7 +79,10 @@ uint32_t bitstream_peek(struct bitstream* bitstream, int numbits) void bitstream_remove(struct bitstream* bitstream, int numbits) { - bitstream->buffer <<= numbits; + /* buffer is 32 bits wide, so shifting by 32 is undefined even though + * consuming all 32 is a legitimate request - peek() already returns the + * whole buffer for that width. */ + bitstream->buffer = (numbits >= 32) ? 0 : (bitstream->buffer << numbits); bitstream->bits -= numbits; } diff --git a/src/libchdr_cdrom.c b/src/libchdr_cdrom.c index 02b883a..753d220 100644 --- a/src/libchdr_cdrom.c +++ b/src/libchdr_cdrom.c @@ -17,6 +17,9 @@ ***************************************************************************/ #include +#ifdef CHDR_DEBUG_ZLIB +#include +#endif #include "../include/libchdr/cdrom.h" @@ -301,19 +304,6 @@ static const uint16_t qoffsets[ECC_Q_NUM_BYTES][ECC_Q_COMP] = { 0x867,0x003,0x05b,0x0b3,0x10b,0x163,0x1bb,0x213,0x26b,0x2c3,0x31b,0x373,0x3cb,0x423,0x47b,0x4d3,0x52b,0x583,0x5db,0x633,0x68b,0x6e3,0x73b,0x793,0x7eb,0x843,0x89b,0x037,0x08f,0x0e7,0x13f,0x197,0x1ef,0x247,0x29f,0x2f7,0x34f,0x3a7,0x3ff,0x457,0x4af,0x507,0x55f } }; -/*------------------------------------------------- - * ecc_source_byte - return data from the sector - * at the given offset, masking anything - * particular to a mode - *------------------------------------------------- - */ - -static CHDR_INLINE uint8_t ecc_source_byte(const uint8_t *sector, uint32_t offset) -{ - /* in mode 2 always treat these as 0 bytes */ - return (sector[MODE_OFFSET] == 2 && offset < 4) ? 0x00 : sector[SYNC_OFFSET + SYNC_NUM_BYTES + offset]; -} - /** * @fn void ecc_compute_bytes(const uint8_t *sector, const uint16_t *row, int rowlen, uint8_t &val1, uint8_t &val2) * @@ -330,16 +320,43 @@ static CHDR_INLINE uint8_t ecc_source_byte(const uint8_t *sector, uint32_t offse void ecc_compute_bytes(const uint8_t *sector, const uint16_t *row, int rowlen, uint8_t *val1, uint8_t *val2) { + /* ecc_generate() points val1/val2 into the sector, so accumulating through + * them forces a spill and reload on every component in case the source read + * aliases the destination. It cannot: P reads stop at byte 2075 and write + * 2076..2247, Q reads stop at 2247 and write 2248..2351, and sector[MODE_OFFSET] + * is never a destination. So hoist the mode test, read each source byte once + * instead of twice, and keep the accumulators in registers. */ + const uint8_t *data = §or[SYNC_OFFSET + SYNC_NUM_BYTES]; + const int mode2 = (sector[MODE_OFFSET] == 2); + uint8_t v1 = 0, v2 = 0; int component; - *val1 = *val2 = 0; - for (component = 0; component < rowlen; component++) + + if (mode2) + { + /* in mode 2 always treat the first four bytes as 0 */ + for (component = 0; component < rowlen; component++) + { + const uint32_t offset = row[component]; + const uint8_t byte = (offset < 4) ? 0x00 : data[offset]; + + v1 = ecclow[v1 ^ byte]; + v2 ^= byte; + } + } + else { - *val1 ^= ecc_source_byte(sector, row[component]); - *val2 ^= ecc_source_byte(sector, row[component]); - *val1 = ecclow[*val1]; + for (component = 0; component < rowlen; component++) + { + const uint8_t byte = data[row[component]]; + + v1 = ecclow[v1 ^ byte]; + v2 ^= byte; + } } - *val1 = ecchigh[ecclow[*val1] ^ *val2]; - *val2 ^= *val1; + + v1 = ecchigh[ecclow[v1] ^ v2]; + *val1 = v1; + *val2 = v2 ^ v1; } /** @@ -388,14 +405,102 @@ int ecc_verify(const uint8_t *sector) * @param [in,out] sector If non-null, the sector. */ +static CHDR_INLINE uint8_t ecc_xtime(uint8_t x) +{ + return (uint8_t)((x << 1) ^ ((x >> 7) * 0x1d)); +} + +/*------------------------------------------------- + * ecc_p_generate_swar - compute the P parity four + * rows at a time, packed into one 32-bit word + *------------------------------------------------- + */ + +/* The P rows are independent of each other and, for a fixed component, they + * read consecutive bytes: poffsets[row][comp] is exactly row + 86*comp. So + * four rows fit in one 32-bit accumulator, and the whole per-component step + * becomes word-wide. + * + * The table lookup cannot come along - there is no gather - but it does not + * need to: ecclow[x] is exactly multiplication by x in GF(2^8) with polynomial + * 0x11d, verified over all 256 entries. Masking off the bits that would cross + * a byte boundary lets one word do four independent products. + * + * The bytes are assembled from four loads rather than read as a word: the P + * stride is 86 and 86 % 4 == 2, so consecutive components alternate 4-aligned + * and 2-aligned no matter how the caller aligned its buffer. Measured on + * ESP32-P4, doing it this way beats both a lw/lhu split and a padded aligned + * copy, and it is correct for any alignment. + */ + +static CHDR_INLINE uint32_t ecc_xtime4(uint32_t t) +{ + return ((t << 1) & 0xfefefefeu) ^ (((t >> 7) & 0x01010101u) * 0x1du); +} + +static void ecc_p_generate_swar(uint8_t *sector, int mode2) +{ + const uint8_t *data = §or[SYNC_OFFSET + SYNC_NUM_BYTES]; + int row, component; + + for (row = 0; row + 4 <= ECC_P_NUM_BYTES; row += 4) + { + uint32_t v1 = 0, v2 = 0, out; + const uint8_t *src = data + row; + + for (component = 0; component < ECC_P_COMP; component++) + { + const uint8_t *q = src + ECC_P_NUM_BYTES * component; + uint32_t x; + + /* offset = row + 86*component, so offset < 4 - the only case mode 2 + * masks - is exactly this group's first component */ + if (mode2 && row == 0 && component == 0) + { + x = 0; + } + else + { + x = (uint32_t)q[0] | ((uint32_t)q[1] << 8) | + ((uint32_t)q[2] << 16) | ((uint32_t)q[3] << 24); + } + + v1 = ecc_xtime4(v1 ^ x); + v2 ^= x; + } + + v1 = ((uint32_t)ecchigh[ecc_xtime( (uint8_t)v1 ) ^ (uint8_t)v2]) + | ((uint32_t)ecchigh[ecc_xtime((uint8_t)(v1 >> 8)) ^ (uint8_t)(v2 >> 8)] << 8) + | ((uint32_t)ecchigh[ecc_xtime((uint8_t)(v1 >> 16)) ^ (uint8_t)(v2 >> 16)] << 16) + | ((uint32_t)ecchigh[ecc_xtime((uint8_t)(v1 >> 24)) ^ (uint8_t)(v2 >> 24)] << 24); + out = v2 ^ v1; + + sector[ECC_P_OFFSET + row + 0] = (uint8_t)v1; + sector[ECC_P_OFFSET + row + 1] = (uint8_t)(v1 >> 8); + sector[ECC_P_OFFSET + row + 2] = (uint8_t)(v1 >> 16); + sector[ECC_P_OFFSET + row + 3] = (uint8_t)(v1 >> 24); + sector[ECC_P_OFFSET + ECC_P_NUM_BYTES + row + 0] = (uint8_t)out; + sector[ECC_P_OFFSET + ECC_P_NUM_BYTES + row + 1] = (uint8_t)(out >> 8); + sector[ECC_P_OFFSET + ECC_P_NUM_BYTES + row + 2] = (uint8_t)(out >> 16); + sector[ECC_P_OFFSET + ECC_P_NUM_BYTES + row + 3] = (uint8_t)(out >> 24); + } + + /* 86 is not a multiple of 4 - finish the tail scalar */ + for (; row < ECC_P_NUM_BYTES; row++) + ecc_compute_bytes(sector, poffsets[row], ECC_P_COMP, + §or[ECC_P_OFFSET + row], §or[ECC_P_OFFSET + ECC_P_NUM_BYTES + row]); +} + void ecc_generate(uint8_t *sector) { int byte; - /* first verify P bytes */ - for (byte = 0; byte < ECC_P_NUM_BYTES; byte++) - ecc_compute_bytes(sector, poffsets[byte], ECC_P_COMP, §or[ECC_P_OFFSET + byte], §or[ECC_P_OFFSET + ECC_P_NUM_BYTES + byte]); - /* then verify Q bytes */ + /* first the P bytes, four rows per word */ + ecc_p_generate_swar(sector, sector[MODE_OFFSET] == 2); + + /* then the Q bytes. These cannot be packed the same way: qoffsets[row][comp] + * is (86*(row>>1) + (row&1) + 88*comp) mod 2236, a diagonal that is + * contiguous in neither dimension. */ for (byte = 0; byte < ECC_Q_NUM_BYTES; byte++) ecc_compute_bytes(sector, qoffsets[byte], ECC_Q_COMP, §or[ECC_Q_OFFSET + byte], §or[ECC_Q_OFFSET + ECC_Q_NUM_BYTES + byte]); } @@ -458,12 +563,21 @@ chd_error cd_codec_decompress( /* reset and decode */ decomp_err = base_decompress(base_decompressor, &src[header_bytes], complen_base, &buffer[0], frames * CD_MAX_SECTOR_DATA); - if (decomp_err != CHDERR_NONE) + if (decomp_err != CHDERR_NONE) { +#ifdef CHDR_DEBUG_ZLIB + printf("cd_codec_decompress: stage=base complen_base=%u err=%d\n", (unsigned)complen_base, decomp_err); +#endif return decomp_err; + } #if WANT_SUBCODE decomp_err = subcode_decompress(subcode_decompressor, &src[header_bytes + complen_base], complen - complen_base - header_bytes, &buffer[frames * CD_MAX_SECTOR_DATA], frames * CD_MAX_SUBCODE_DATA); - if (decomp_err != CHDERR_NONE) + if (decomp_err != CHDERR_NONE) { +#ifdef CHDR_DEBUG_ZLIB + printf("cd_codec_decompress: stage=subcode complen_subcode=%u err=%d\n", + (unsigned)(complen - complen_base - header_bytes), decomp_err); +#endif return decomp_err; + } #endif /* reassemble the data */ diff --git a/src/libchdr_chd.c b/src/libchdr_chd.c index 0ed25e4..c232784 100644 --- a/src/libchdr_chd.c +++ b/src/libchdr_chd.c @@ -328,6 +328,50 @@ struct _chd_file uint8_t * file_cache; /* cache of underlying file */ + /* Compressed read-ahead. libchdr issues one seek+read per hunk, and + * compressed hunks are small - measured across a 14-CHD sample, 426 to + * 15223 bytes, averaging a few KB - so a large title costs hundreds of + * thousands of transactions whose fixed per-call cost (VFS dispatch, + * filesystem bookkeeping, controller command setup, DMA, interrupt) has + * nothing to do with their size. On an ESP32-P4 reading from SD that + * showed up as under 15% of the available bus bandwidth. + * + * This is only worth doing because hunk payloads turn out to be laid out + * strictly sequentially: across that same sample, 100% of hunks that + * touch the file begin exactly where the previous one ended, in a single + * run spanning the whole file. So one larger read serves many hunks. + * + * Off unless the caller sets a budget - see chd_set_cache_budget(). The + * library deliberately does not size this itself: how much memory is + * available is a property of the embedding system, not of libchdr. */ + uint8_t * ra_buf; /* NULL = disabled */ + size_t ra_capacity; + uint64_t ra_off; /* file offset of ra_buf[0] */ + size_t ra_valid; /* bytes currently held */ + uint64_t ra_hits, ra_misses; + + /* Decoded-hunk cache, used only by COMPRESSION_SELF back-references. + * A self-referencing hunk means "identical to hunk N", and the read path + * previously recursed into a full re-read *and* re-decode of hunk N every + * time. Measured across a 14-CHD corpus: 100% of self-references point + * backwards, so on FAT-backed storage each one also forces a filesystem + * seek that restarts its cluster-chain walk. + * + * Sized as a byte budget rather than an entry count on purpose: hunkbytes + * varies 8x across real content (2448 for a raw-sector CD image, 19584 for + * a normal CD, 4096 for a hard disk), so a fixed entry count would mean + * 4KB on one file and 1.2MB on another. Simulation over that corpus showed + * a single entry already captures 97-100% of self-references wherever they + * are clustered at all, and further entries buy tenths of a percent, so + * the default budget is deliberately small. + * + * Allocated lazily on the first self-reference actually encountered - CHDs + * with none (3 of the 14 measured) never pay for this. */ + uint8_t * selfcache_data; /* selfcache_entries * hunkbytes */ + uint32_t * selfcache_hunk; /* hunk in each slot, ~0 = empty */ + uint32_t selfcache_entries; + uint32_t selfcache_next; /* round-robin victim */ + #if LOWRAM_TARGET v5_lowram_map lowram_map; /* CHDv5 compressed-map lazy-decode state */ @@ -381,8 +425,89 @@ static int core_legacy_fseek(void* file, int64_t offset, int whence); static chd_error header_read(chd_file *chd); /* internal hunk read/write */ +/* Byte budget for the decoded-hunk cache described in struct _chd_file. + * Entries = max(1, budget / hunkbytes). LOWRAM targets get a single entry, + * which the measurements show is where nearly all of the benefit already is. */ +#ifndef CHDR_SELF_CACHE_BYTES +#if LOWRAM_TARGET +#define CHDR_SELF_CACHE_BYTES 0 +#else +#define CHDR_SELF_CACHE_BYTES 65536 +#endif +#endif + static chd_error hunk_read_into_memory(chd_file *chd, uint32_t hunknum, uint8_t *dest); +/*------------------------------------------------- + selfcache_* - decoded-hunk cache for + COMPRESSION_SELF back-references +-------------------------------------------------*/ + +static void selfcache_free(chd_file *chd) +{ + if (chd->selfcache_data != NULL) { free(chd->selfcache_data); chd->selfcache_data = NULL; } + if (chd->selfcache_hunk != NULL) { free(chd->selfcache_hunk); chd->selfcache_hunk = NULL; } + chd->selfcache_entries = 0; + chd->selfcache_next = 0; +} + +/* returns 1 once a cache exists (or already existed), 0 if it could not be + * allocated - callers must treat failure as "just don't cache", never fatal */ +static int selfcache_ensure(chd_file *chd) +{ + uint32_t n, i; + + if (chd->selfcache_entries != 0) + return 1; + if (chd->header.hunkbytes == 0) + return 0; + + n = CHDR_SELF_CACHE_BYTES / chd->header.hunkbytes; + if (n == 0) + n = 1; + + chd->selfcache_data = (uint8_t *)malloc((size_t)n * chd->header.hunkbytes); + chd->selfcache_hunk = (uint32_t *)malloc((size_t)n * sizeof(uint32_t)); + if (chd->selfcache_data == NULL || chd->selfcache_hunk == NULL) { + selfcache_free(chd); + return 0; + } + for (i = 0; i < n; i++) + chd->selfcache_hunk[i] = (uint32_t)~0; + chd->selfcache_entries = n; + chd->selfcache_next = 0; + return 1; +} + +static int selfcache_lookup(chd_file *chd, uint32_t hunknum, uint8_t *dest) +{ + uint32_t i; + for (i = 0; i < chd->selfcache_entries; i++) { + if (chd->selfcache_hunk[i] == hunknum) { + memcpy(dest, chd->selfcache_data + (size_t)i * chd->header.hunkbytes, + chd->header.hunkbytes); + return 1; + } + } + return 0; +} + +static void selfcache_store(chd_file *chd, uint32_t hunknum, const uint8_t *src) +{ + uint32_t i, slot; + + if (chd->selfcache_entries == 0) + return; + for (i = 0; i < chd->selfcache_entries; i++) + if (chd->selfcache_hunk[i] == hunknum) + return; /* already resident */ + slot = chd->selfcache_next; + chd->selfcache_next = (slot + 1) % chd->selfcache_entries; + memcpy(chd->selfcache_data + (size_t)slot * chd->header.hunkbytes, src, + chd->header.hunkbytes); + chd->selfcache_hunk[slot] = hunknum; +} + /* internal map access */ static chd_error map_read(chd_file *chd); #if LOWRAM_TARGET @@ -635,7 +760,14 @@ static CHDR_INLINE void put_bigendian_uint48(uint8_t *base, uint64_t value) static CHDR_INLINE uint32_t get_bigendian_uint32_t(const uint8_t *base) { - return (base[0] << 24) | (base[1] << 16) | (base[2] << 8) | base[3]; + /* Cast before shifting: base[0] promotes to int, so base[0] << 24 is + * signed overflow - undefined - for any byte >= 0x80. Real files never hit + * it because every field read through here holds either a small count or a + * four-character codec tag, and those are ASCII, but a malformed file only + * has to set one high bit. The uint48 and uint64 readers above already + * cast for the same reason. */ + return ((uint32_t)base[0] << 24) | ((uint32_t)base[1] << 16) | + ((uint32_t)base[2] << 8) | (uint32_t)base[3]; } /*------------------------------------------------- @@ -767,8 +899,133 @@ static uint16_t crc16_update(uint16_t crc, const void *data, uint32_t length) 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0 }; + static const uint16_t s_table1[256] = + { + 0x0000, 0x3331, 0x6662, 0x5553, 0xccc4, 0xfff5, 0xaaa6, 0x9997, + 0x89a9, 0xba98, 0xefcb, 0xdcfa, 0x456d, 0x765c, 0x230f, 0x103e, + 0x0373, 0x3042, 0x6511, 0x5620, 0xcfb7, 0xfc86, 0xa9d5, 0x9ae4, + 0x8ada, 0xb9eb, 0xecb8, 0xdf89, 0x461e, 0x752f, 0x207c, 0x134d, + 0x06e6, 0x35d7, 0x6084, 0x53b5, 0xca22, 0xf913, 0xac40, 0x9f71, + 0x8f4f, 0xbc7e, 0xe92d, 0xda1c, 0x438b, 0x70ba, 0x25e9, 0x16d8, + 0x0595, 0x36a4, 0x63f7, 0x50c6, 0xc951, 0xfa60, 0xaf33, 0x9c02, + 0x8c3c, 0xbf0d, 0xea5e, 0xd96f, 0x40f8, 0x73c9, 0x269a, 0x15ab, + 0x0dcc, 0x3efd, 0x6bae, 0x589f, 0xc108, 0xf239, 0xa76a, 0x945b, + 0x8465, 0xb754, 0xe207, 0xd136, 0x48a1, 0x7b90, 0x2ec3, 0x1df2, + 0x0ebf, 0x3d8e, 0x68dd, 0x5bec, 0xc27b, 0xf14a, 0xa419, 0x9728, + 0x8716, 0xb427, 0xe174, 0xd245, 0x4bd2, 0x78e3, 0x2db0, 0x1e81, + 0x0b2a, 0x381b, 0x6d48, 0x5e79, 0xc7ee, 0xf4df, 0xa18c, 0x92bd, + 0x8283, 0xb1b2, 0xe4e1, 0xd7d0, 0x4e47, 0x7d76, 0x2825, 0x1b14, + 0x0859, 0x3b68, 0x6e3b, 0x5d0a, 0xc49d, 0xf7ac, 0xa2ff, 0x91ce, + 0x81f0, 0xb2c1, 0xe792, 0xd4a3, 0x4d34, 0x7e05, 0x2b56, 0x1867, + 0x1b98, 0x28a9, 0x7dfa, 0x4ecb, 0xd75c, 0xe46d, 0xb13e, 0x820f, + 0x9231, 0xa100, 0xf453, 0xc762, 0x5ef5, 0x6dc4, 0x3897, 0x0ba6, + 0x18eb, 0x2bda, 0x7e89, 0x4db8, 0xd42f, 0xe71e, 0xb24d, 0x817c, + 0x9142, 0xa273, 0xf720, 0xc411, 0x5d86, 0x6eb7, 0x3be4, 0x08d5, + 0x1d7e, 0x2e4f, 0x7b1c, 0x482d, 0xd1ba, 0xe28b, 0xb7d8, 0x84e9, + 0x94d7, 0xa7e6, 0xf2b5, 0xc184, 0x5813, 0x6b22, 0x3e71, 0x0d40, + 0x1e0d, 0x2d3c, 0x786f, 0x4b5e, 0xd2c9, 0xe1f8, 0xb4ab, 0x879a, + 0x97a4, 0xa495, 0xf1c6, 0xc2f7, 0x5b60, 0x6851, 0x3d02, 0x0e33, + 0x1654, 0x2565, 0x7036, 0x4307, 0xda90, 0xe9a1, 0xbcf2, 0x8fc3, + 0x9ffd, 0xaccc, 0xf99f, 0xcaae, 0x5339, 0x6008, 0x355b, 0x066a, + 0x1527, 0x2616, 0x7345, 0x4074, 0xd9e3, 0xead2, 0xbf81, 0x8cb0, + 0x9c8e, 0xafbf, 0xfaec, 0xc9dd, 0x504a, 0x637b, 0x3628, 0x0519, + 0x10b2, 0x2383, 0x76d0, 0x45e1, 0xdc76, 0xef47, 0xba14, 0x8925, + 0x991b, 0xaa2a, 0xff79, 0xcc48, 0x55df, 0x66ee, 0x33bd, 0x008c, + 0x13c1, 0x20f0, 0x75a3, 0x4692, 0xdf05, 0xec34, 0xb967, 0x8a56, + 0x9a68, 0xa959, 0xfc0a, 0xcf3b, 0x56ac, 0x659d, 0x30ce, 0x03ff + }; + static const uint16_t s_table2[256] = + { + 0x0000, 0x3730, 0x6e60, 0x5950, 0xdcc0, 0xebf0, 0xb2a0, 0x8590, + 0xa9a1, 0x9e91, 0xc7c1, 0xf0f1, 0x7561, 0x4251, 0x1b01, 0x2c31, + 0x4363, 0x7453, 0x2d03, 0x1a33, 0x9fa3, 0xa893, 0xf1c3, 0xc6f3, + 0xeac2, 0xddf2, 0x84a2, 0xb392, 0x3602, 0x0132, 0x5862, 0x6f52, + 0x86c6, 0xb1f6, 0xe8a6, 0xdf96, 0x5a06, 0x6d36, 0x3466, 0x0356, + 0x2f67, 0x1857, 0x4107, 0x7637, 0xf3a7, 0xc497, 0x9dc7, 0xaaf7, + 0xc5a5, 0xf295, 0xabc5, 0x9cf5, 0x1965, 0x2e55, 0x7705, 0x4035, + 0x6c04, 0x5b34, 0x0264, 0x3554, 0xb0c4, 0x87f4, 0xdea4, 0xe994, + 0x1dad, 0x2a9d, 0x73cd, 0x44fd, 0xc16d, 0xf65d, 0xaf0d, 0x983d, + 0xb40c, 0x833c, 0xda6c, 0xed5c, 0x68cc, 0x5ffc, 0x06ac, 0x319c, + 0x5ece, 0x69fe, 0x30ae, 0x079e, 0x820e, 0xb53e, 0xec6e, 0xdb5e, + 0xf76f, 0xc05f, 0x990f, 0xae3f, 0x2baf, 0x1c9f, 0x45cf, 0x72ff, + 0x9b6b, 0xac5b, 0xf50b, 0xc23b, 0x47ab, 0x709b, 0x29cb, 0x1efb, + 0x32ca, 0x05fa, 0x5caa, 0x6b9a, 0xee0a, 0xd93a, 0x806a, 0xb75a, + 0xd808, 0xef38, 0xb668, 0x8158, 0x04c8, 0x33f8, 0x6aa8, 0x5d98, + 0x71a9, 0x4699, 0x1fc9, 0x28f9, 0xad69, 0x9a59, 0xc309, 0xf439, + 0x3b5a, 0x0c6a, 0x553a, 0x620a, 0xe79a, 0xd0aa, 0x89fa, 0xbeca, + 0x92fb, 0xa5cb, 0xfc9b, 0xcbab, 0x4e3b, 0x790b, 0x205b, 0x176b, + 0x7839, 0x4f09, 0x1659, 0x2169, 0xa4f9, 0x93c9, 0xca99, 0xfda9, + 0xd198, 0xe6a8, 0xbff8, 0x88c8, 0x0d58, 0x3a68, 0x6338, 0x5408, + 0xbd9c, 0x8aac, 0xd3fc, 0xe4cc, 0x615c, 0x566c, 0x0f3c, 0x380c, + 0x143d, 0x230d, 0x7a5d, 0x4d6d, 0xc8fd, 0xffcd, 0xa69d, 0x91ad, + 0xfeff, 0xc9cf, 0x909f, 0xa7af, 0x223f, 0x150f, 0x4c5f, 0x7b6f, + 0x575e, 0x606e, 0x393e, 0x0e0e, 0x8b9e, 0xbcae, 0xe5fe, 0xd2ce, + 0x26f7, 0x11c7, 0x4897, 0x7fa7, 0xfa37, 0xcd07, 0x9457, 0xa367, + 0x8f56, 0xb866, 0xe136, 0xd606, 0x5396, 0x64a6, 0x3df6, 0x0ac6, + 0x6594, 0x52a4, 0x0bf4, 0x3cc4, 0xb954, 0x8e64, 0xd734, 0xe004, + 0xcc35, 0xfb05, 0xa255, 0x9565, 0x10f5, 0x27c5, 0x7e95, 0x49a5, + 0xa031, 0x9701, 0xce51, 0xf961, 0x7cf1, 0x4bc1, 0x1291, 0x25a1, + 0x0990, 0x3ea0, 0x67f0, 0x50c0, 0xd550, 0xe260, 0xbb30, 0x8c00, + 0xe352, 0xd462, 0x8d32, 0xba02, 0x3f92, 0x08a2, 0x51f2, 0x66c2, + 0x4af3, 0x7dc3, 0x2493, 0x13a3, 0x9633, 0xa103, 0xf853, 0xcf63 + }; + static const uint16_t s_table3[256] = + { + 0x0000, 0x76b4, 0xed68, 0x9bdc, 0xcaf1, 0xbc45, 0x2799, 0x512d, + 0x85c3, 0xf377, 0x68ab, 0x1e1f, 0x4f32, 0x3986, 0xa25a, 0xd4ee, + 0x1ba7, 0x6d13, 0xf6cf, 0x807b, 0xd156, 0xa7e2, 0x3c3e, 0x4a8a, + 0x9e64, 0xe8d0, 0x730c, 0x05b8, 0x5495, 0x2221, 0xb9fd, 0xcf49, + 0x374e, 0x41fa, 0xda26, 0xac92, 0xfdbf, 0x8b0b, 0x10d7, 0x6663, + 0xb28d, 0xc439, 0x5fe5, 0x2951, 0x787c, 0x0ec8, 0x9514, 0xe3a0, + 0x2ce9, 0x5a5d, 0xc181, 0xb735, 0xe618, 0x90ac, 0x0b70, 0x7dc4, + 0xa92a, 0xdf9e, 0x4442, 0x32f6, 0x63db, 0x156f, 0x8eb3, 0xf807, + 0x6e9c, 0x1828, 0x83f4, 0xf540, 0xa46d, 0xd2d9, 0x4905, 0x3fb1, + 0xeb5f, 0x9deb, 0x0637, 0x7083, 0x21ae, 0x571a, 0xccc6, 0xba72, + 0x753b, 0x038f, 0x9853, 0xeee7, 0xbfca, 0xc97e, 0x52a2, 0x2416, + 0xf0f8, 0x864c, 0x1d90, 0x6b24, 0x3a09, 0x4cbd, 0xd761, 0xa1d5, + 0x59d2, 0x2f66, 0xb4ba, 0xc20e, 0x9323, 0xe597, 0x7e4b, 0x08ff, + 0xdc11, 0xaaa5, 0x3179, 0x47cd, 0x16e0, 0x6054, 0xfb88, 0x8d3c, + 0x4275, 0x34c1, 0xaf1d, 0xd9a9, 0x8884, 0xfe30, 0x65ec, 0x1358, + 0xc7b6, 0xb102, 0x2ade, 0x5c6a, 0x0d47, 0x7bf3, 0xe02f, 0x969b, + 0xdd38, 0xab8c, 0x3050, 0x46e4, 0x17c9, 0x617d, 0xfaa1, 0x8c15, + 0x58fb, 0x2e4f, 0xb593, 0xc327, 0x920a, 0xe4be, 0x7f62, 0x09d6, + 0xc69f, 0xb02b, 0x2bf7, 0x5d43, 0x0c6e, 0x7ada, 0xe106, 0x97b2, + 0x435c, 0x35e8, 0xae34, 0xd880, 0x89ad, 0xff19, 0x64c5, 0x1271, + 0xea76, 0x9cc2, 0x071e, 0x71aa, 0x2087, 0x5633, 0xcdef, 0xbb5b, + 0x6fb5, 0x1901, 0x82dd, 0xf469, 0xa544, 0xd3f0, 0x482c, 0x3e98, + 0xf1d1, 0x8765, 0x1cb9, 0x6a0d, 0x3b20, 0x4d94, 0xd648, 0xa0fc, + 0x7412, 0x02a6, 0x997a, 0xefce, 0xbee3, 0xc857, 0x538b, 0x253f, + 0xb3a4, 0xc510, 0x5ecc, 0x2878, 0x7955, 0x0fe1, 0x943d, 0xe289, + 0x3667, 0x40d3, 0xdb0f, 0xadbb, 0xfc96, 0x8a22, 0x11fe, 0x674a, + 0xa803, 0xdeb7, 0x456b, 0x33df, 0x62f2, 0x1446, 0x8f9a, 0xf92e, + 0x2dc0, 0x5b74, 0xc0a8, 0xb61c, 0xe731, 0x9185, 0x0a59, 0x7ced, + 0x84ea, 0xf25e, 0x6982, 0x1f36, 0x4e1b, 0x38af, 0xa373, 0xd5c7, + 0x0129, 0x779d, 0xec41, 0x9af5, 0xcbd8, 0xbd6c, 0x26b0, 0x5004, + 0x9f4d, 0xe9f9, 0x7225, 0x0491, 0x55bc, 0x2308, 0xb8d4, 0xce60, + 0x1a8e, 0x6c3a, 0xf7e6, 0x8152, 0xd07f, 0xa6cb, 0x3d17, 0x4ba3 + }; + const uint8_t *src = (uint8_t*)data; + /* Slice-by-4. The byte-at-a-time form is 12 instructions per byte on RV32 + * (two of them just truncating back to uint16_t), and it runs over every + * decoded hunk under VERIFY_BLOCK_CRC, so it costs about 0.76 ms per + * 19584-byte hunk on an ESP32-P4 - more than the zstd decode it is + * checking. Folding four bytes per iteration takes it to 6.25. + * + * s_table1/2/3 are s_table advanced by one, two and three byte positions, + * so the four lookups can be XORed together. Verified identical to the + * byte-at-a-time result for every length 0..4096 and 256 starting CRCs. */ + while (length >= 4) + { + const uint16_t x = (uint16_t)((crc >> 8) ^ src[0]); + const uint16_t y = (uint16_t)((crc & 0xff) ^ src[1]); + + crc = (uint16_t)(s_table3[x] ^ s_table2[y] ^ s_table1[src[2]] ^ s_table[src[3]]); + src += 4; + length -= 4; + } + /* rip through the source data */ while (length-- != 0) crc = (crc << 8) ^ s_table[(crc >> 8) ^ *src++]; @@ -849,6 +1106,15 @@ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) selfbits = rawbuf[13]; parentbits = rawbuf[14]; + /* These three are raw bytes from the file and they become the bit width + * passed to bitstream_read() for every map entry. Anything above 32 makes + * bitstream_peek() shift by a negative amount, so a malformed file could + * reach undefined behaviour before any other check ran. chdman derives + * them from hunkbytes and the hunk count, so they never legitimately + * exceed 32. */ + if (lengthbits > 32 || selfbits > 32 || parentbits > 32) + return CHDERR_INVALID_FILE; + /* now read the map */ if ((header->mapoffset + mapbytes) < header->mapoffset || (header->mapoffset + mapbytes) >= chd->file_size) return CHDERR_INVALID_FILE; @@ -1909,6 +2175,57 @@ CHD_EXPORT chd_error chd_open_core_file_callbacks(const core_file_callbacks *cal return err; } +/*------------------------------------------------- + chd_set_cache_budget - give libchdr a memory + budget to spend on internal caching, or 0 to + disable it (the default) +-------------------------------------------------*/ + +CHD_EXPORT chd_error chd_set_cache_budget(chd_file *chd, size_t bytes) +{ + uint8_t *buf; + + if (chd == NULL) + return CHDERR_INVALID_PARAMETER; + + /* releasing is always possible */ + if (chd->ra_buf != NULL) + { + free(chd->ra_buf); + chd->ra_buf = NULL; + chd->ra_capacity = chd->ra_valid = 0; + chd->ra_off = 0; + } + if (bytes == 0) + return CHDERR_NONE; + + /* a window smaller than one hunk can never serve a read */ + if (bytes < chd->header.hunkbytes) + bytes = chd->header.hunkbytes; + + buf = (uint8_t *)malloc(bytes); + if (buf == NULL) + return CHDERR_OUT_OF_MEMORY; /* caching off; the file stays usable */ + + chd->ra_buf = buf; + chd->ra_capacity = bytes; + chd->ra_valid = 0; + chd->ra_off = 0; + chd->ra_hits = chd->ra_misses = 0; + return CHDERR_NONE; +} + +CHD_EXPORT size_t chd_get_cache_budget(const chd_file *chd) +{ + return (chd != NULL) ? chd->ra_capacity : 0; +} + +CHD_EXPORT void chd_get_cache_stats(const chd_file *chd, uint64_t *hits, uint64_t *misses) +{ + if (hits != NULL) *hits = (chd != NULL) ? chd->ra_hits : 0; + if (misses != NULL) *misses = (chd != NULL) ? chd->ra_misses : 0; +} + /*------------------------------------------------- chd_precache - precache underlying file in memory @@ -2098,6 +2415,15 @@ CHD_EXPORT void chd_close(chd_file *chd) if (chd->file.callbacks != NULL) core_fclose(&chd->file); + selfcache_free(chd); + + if (chd->ra_buf != NULL) + { + free(chd->ra_buf); + chd->ra_buf = NULL; + chd->ra_capacity = chd->ra_valid = 0; + } + if (chd->file_cache) free(chd->file_cache); @@ -2306,7 +2632,7 @@ CHD_EXPORT chd_error chd_get_metadata(chd_file *chd, uint32_t searchtag, uint32_ uint32_t faux_length; /* fill in the faux metadata */ - snprintf(faux_metadata, sizeof(faux_metadata), HARD_DISK_METADATA_FORMAT, chd->header.obsolete_cylinders, chd->header.obsolete_heads, chd->header.obsolete_sectors, (chd->header.obsolete_hunksize != 0) ? (chd->header.hunkbytes / chd->header.obsolete_hunksize) : 0); + snprintf(faux_metadata, sizeof(faux_metadata), HARD_DISK_METADATA_FORMAT, (int)chd->header.obsolete_cylinders, (int)chd->header.obsolete_heads, (int)chd->header.obsolete_sectors, (int)((chd->header.obsolete_hunksize != 0) ? (chd->header.hunkbytes / chd->header.obsolete_hunksize) : 0)); faux_length = (uint32_t)strlen(faux_metadata) + 1; /* copy the metadata itself */ @@ -2614,6 +2940,83 @@ static uint8_t* hunk_read_compressed(chd_file *chd, uint64_t offset, size_t size } #endif + /* Serve from the read-ahead window when the caller has given us a + * budget. On a miss, any bytes already held that sit at or after the + * requested offset are kept and slid to the front, so the refill + * reads only what is genuinely new: sequential access therefore + * transfers each byte exactly once, and the window is a pure + * reduction in transaction count rather than a trade against + * re-reading. A window is never used for a hunk larger than the + * window itself. */ + if (chd->ra_buf != NULL && size <= chd->ra_capacity) + { + int have = (offset >= chd->ra_off && + offset + size <= chd->ra_off + chd->ra_valid); + /* Only a forward-progressing miss refills the window. A backward + * read - which in practice means a COMPRESSION_SELF reference + * reaching back to an earlier hunk - is served directly and + * leaves the window untouched, so an excursion cannot throw away + * data already prefetched for the sequential stream it will + * return to. Without this the window is evicted and refilled + * around every self-reference, and the same bytes are fetched + * more than once. */ + if (!have && offset < chd->ra_off) + { + chd->ra_misses++; + } + else if (!have) + { + size_t keep = 0; + size_t want; + uint64_t fill_at; + + if (offset >= chd->ra_off && offset < chd->ra_off + chd->ra_valid) + { + keep = (size_t)(chd->ra_off + chd->ra_valid - offset); + memmove(chd->ra_buf, chd->ra_buf + (offset - chd->ra_off), keep); + } + fill_at = offset + keep; + want = chd->ra_capacity - keep; + if (want > 0 && fill_at < chd->file_size) + { + uint64_t avail = chd->file_size - fill_at; + if (avail > (uint64_t)want) + avail = (uint64_t)want; + if (seek_and_read(chd, fill_at, chd->ra_buf + keep, (size_t)avail)) + { + chd->ra_off = offset; + chd->ra_valid = keep + (size_t)avail; + chd->ra_misses++; + have = (offset + size <= chd->ra_off + chd->ra_valid); + } + else + { + /* a failed refill must not fail the read - drop the + * window and fall through to the direct path */ + chd->ra_valid = 0; + have = 0; + } + } + else + { + chd->ra_valid = keep; + chd->ra_off = offset; + have = (size <= keep); + } + } + else + chd->ra_hits++; + + if (have) + { + /* copy out rather than handing back a pointer into the + * window: the caller holds this across the decompress call, + * and a later refill would move the bytes underneath it */ + memcpy(chd->compressed, chd->ra_buf + (offset - chd->ra_off), size); + return chd->compressed; + } + } + if (!seek_and_read(chd, offset, chd->compressed, size)) return NULL; return chd->compressed; @@ -2636,6 +3039,18 @@ static chd_error hunk_read_uncompressed(chd_file *chd, uint64_t offset, size_t s } else { + /* Uncompressed hunks share the file's sequential layout with the + * compressed ones, so they must consume the same read-ahead window - + * otherwise the window prefetches their bytes and they then read the + * same range again directly, and a file with a meaningful fraction of + * uncompressed hunks transfers noticeably more than it needs to. */ + if (chd->ra_buf != NULL && offset >= chd->ra_off && + offset + size <= chd->ra_off + chd->ra_valid) + { + memcpy(dest, chd->ra_buf + (offset - chd->ra_off), size); + chd->ra_hits++; + return CHDERR_NONE; + } if (!seek_and_read(chd, offset, dest, size)) return CHDERR_READ_ERROR; } @@ -2851,7 +3266,23 @@ static chd_error hunk_read_into_memory(chd_file *chd, uint32_t hunknum, uint8_t return CHDERR_NONE; case COMPRESSION_SELF: - return hunk_read_into_memory(chd, blockoffs, dest); + { + uint32_t target = (uint32_t)blockoffs; + + /* the whole point of the cache: a self-reference otherwise + * costs a backward seek plus a full re-decode of a hunk that + * was very often decoded moments ago */ + if (chd->selfcache_entries != 0 && selfcache_lookup(chd, target, dest)) + return CHDERR_NONE; + + selfcache_ensure(chd); /* lazy: only files with self-refs pay */ + + err = hunk_read_into_memory(chd, target, dest); + if (err != CHDERR_NONE) + return err; + selfcache_store(chd, target, dest); + return CHDERR_NONE; + } case COMPRESSION_PARENT: { diff --git a/src/libchdr_codec_cdfl.c b/src/libchdr_codec_cdfl.c index f0a94c9..c553d92 100644 --- a/src/libchdr_codec_cdfl.c +++ b/src/libchdr_codec_cdfl.c @@ -3,9 +3,32 @@ #include #include #include +#ifdef CHDR_DEBUG_ZLIB +#include +#endif #include "../include/libchdr/cdrom.h" +/* CHDR_PROFILE_CDFL: attribute cdfl hunk time across its three stages. + * cdfl measured ~3.6x cdlz's codec-only time per hunk on ESP32-P4, which is + * the opposite of what raw FLAC vs LZMA decode cost would predict - the + * suspicion being flac_decoder_reset()'s per-hunk drflac_open_with_metadata() + * (full decoder teardown + rebuild, ~40KB alloc, STREAMINFO reparse) rather + * than the audio decode itself. The host supplies the clock so this stays + * free of any platform dependency. */ +#ifdef CHDR_PROFILE_CDFL +extern int64_t chdr_prof_now_us(void); +uint64_t chdr_prof_flac_reset_us; +uint64_t chdr_prof_flac_decode_us; +uint64_t chdr_prof_subcode_us; +uint64_t chdr_prof_cdfl_hunks; +#define PROF_T0() int64_t prof_t = chdr_prof_now_us() +#define PROF_ACC(acc) do { acc += (uint64_t)(chdr_prof_now_us() - prof_t); } while (0) +#else +#define PROF_T0() do {} while (0) +#define PROF_ACC(acc) do {} while (0) +#endif + static uint32_t cdfl_codec_blocksize(uint32_t bytes) { /* for CDs it seems that CD_MAX_SECTOR_DATA is the right target */ @@ -71,21 +94,49 @@ chd_error cdfl_codec_decompress(void *codec, const uint8_t *src, uint32_t comple /* reset and decode */ uint32_t frames = destlen / CD_FRAME_SIZE; - if (!flac_decoder_reset(&cdfl->decoder, 44100, 2, cdfl_codec_blocksize(frames * CD_MAX_SECTOR_DATA), src, complen)) - return CHDERR_DECOMPRESSION_ERROR; + { PROF_T0(); + if (!flac_decoder_reset(&cdfl->decoder, 44100, 2, cdfl_codec_blocksize(frames * CD_MAX_SECTOR_DATA), src, complen)) { +#ifdef CHDR_DEBUG_ZLIB + printf("cdfl_codec_decompress: stage=flac_reset complen=%u alloc_failed=%d\n", + (unsigned)complen, cdfl->decoder.alloc_failed); +#endif + /* reset() allocates a fresh ~40KB drflac decoder per hunk; on a + * small-RAM target that, not the stream, is what usually fails */ + return cdfl->decoder.alloc_failed ? CHDERR_OUT_OF_MEMORY : CHDERR_DECOMPRESSION_ERROR; + } + PROF_ACC(chdr_prof_flac_reset_us); } buffer = &cdfl->buffer[0]; - if (!flac_decoder_decode_interleaved(&cdfl->decoder, (int16_t *)(buffer), frames * CD_MAX_SECTOR_DATA/4, cdfl->swap_endian)) - return CHDERR_DECOMPRESSION_ERROR; + { PROF_T0(); + if (!flac_decoder_decode_interleaved(&cdfl->decoder, (int16_t *)(buffer), frames * CD_MAX_SECTOR_DATA/4, cdfl->swap_endian)) { +#ifdef CHDR_DEBUG_ZLIB + printf("cdfl_codec_decompress: stage=flac_decode complen=%u alloc_failed=%d\n", + (unsigned)complen, cdfl->decoder.alloc_failed); +#endif + /* alloc_failed is cleared by the reset() above, so it can only be + * set here by an allocation the decode itself attempted */ + return cdfl->decoder.alloc_failed ? CHDERR_OUT_OF_MEMORY : CHDERR_DECOMPRESSION_ERROR; + } + PROF_ACC(chdr_prof_flac_decode_us); } #if WANT_SUBCODE /* inflate the subcode data */ + { PROF_T0(); offset = flac_decoder_finish(&cdfl->decoder); ret = zlib_codec_decompress(&cdfl->subcode_decompressor, src + offset, complen - offset, &cdfl->buffer[frames * CD_MAX_SECTOR_DATA], frames * CD_MAX_SUBCODE_DATA); - if (ret != CHDERR_NONE) + if (ret != CHDERR_NONE) { +#ifdef CHDR_DEBUG_ZLIB + printf("cdfl_codec_decompress: stage=subcode offset=%u complen_subcode=%u err=%d\n", + (unsigned)offset, (unsigned)(complen - offset), ret); +#endif return ret; + } + PROF_ACC(chdr_prof_subcode_us); } #else flac_decoder_finish(&cdfl->decoder); #endif +#ifdef CHDR_PROFILE_CDFL + chdr_prof_cdfl_hunks++; +#endif /* reassemble the data */ for (framenum = 0; framenum < frames; framenum++) diff --git a/src/libchdr_codec_cdzs.c b/src/libchdr_codec_cdzs.c index 9efbd58..abb455a 100644 --- a/src/libchdr_codec_cdzs.c +++ b/src/libchdr_codec_cdzs.c @@ -21,11 +21,11 @@ chd_error cdzs_codec_init(void* codec, uint32_t hunkbytes) if (ret != CHDERR_NONE) return ret; -#if WANT_SUBCODE +#if WANT_SUBCODE && !LOWRAM_TARGET ret = zstd_codec_init(&cdzs->subcode_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SUBCODE_DATA); +#endif if (ret != CHDERR_NONE) return ret; -#endif if (hunkbytes % CD_FRAME_SIZE != 0) return CHDERR_CODEC_ERROR; @@ -38,7 +38,7 @@ void cdzs_codec_free(void* codec) cdzs_codec_data* cdzs = (cdzs_codec_data*) codec; free(cdzs->buffer); zstd_codec_free(&cdzs->base_decompressor); -#if WANT_SUBCODE +#if WANT_SUBCODE && !LOWRAM_TARGET zstd_codec_free(&cdzs->subcode_decompressor); #endif } @@ -50,7 +50,7 @@ chd_error cdzs_codec_decompress(void *codec, const uint8_t *src, uint32_t comple return cd_codec_decompress(cdzs->buffer, &cdzs->base_decompressor, zstd_codec_decompress, #if WANT_SUBCODE - &cdzs->subcode_decompressor, zstd_codec_decompress, + cdzs_subcode_ctx(cdzs), zstd_codec_decompress, #else NULL, NULL, #endif diff --git a/src/libchdr_codec_flac.c b/src/libchdr_codec_flac.c index d144dd4..85ba0a5 100644 --- a/src/libchdr_codec_flac.c +++ b/src/libchdr_codec_flac.c @@ -56,9 +56,9 @@ chd_error flac_codec_decompress(void *codec, const uint8_t *src, uint32_t comple return CHDERR_DECOMPRESSION_ERROR; if (!flac_decoder_reset(&flac->decoder, 44100, 2, flac_codec_blocksize(destlen), src + 1, complen - 1)) - return CHDERR_DECOMPRESSION_ERROR; + return flac->decoder.alloc_failed ? CHDERR_OUT_OF_MEMORY : CHDERR_DECOMPRESSION_ERROR; if (!flac_decoder_decode_interleaved(&flac->decoder, (int16_t *)(dest), destlen/4, swap_endian)) - return CHDERR_DECOMPRESSION_ERROR; + return flac->decoder.alloc_failed ? CHDERR_OUT_OF_MEMORY : CHDERR_DECOMPRESSION_ERROR; flac_decoder_finish(&flac->decoder); return CHDERR_NONE; diff --git a/src/libchdr_codec_zlib.c b/src/libchdr_codec_zlib.c index c498589..e6ed1d4 100644 --- a/src/libchdr_codec_zlib.c +++ b/src/libchdr_codec_zlib.c @@ -1,180 +1,273 @@ -#include "codec_zlib.h" - -#include -#include -#include - -static voidpf zlib_fast_alloc(voidpf opaque, zlib_alloc_size items, zlib_alloc_size size); -static void zlib_fast_free(voidpf opaque, voidpf address); -static void zlib_allocator_free(voidpf opaque); - -/*------------------------------------------------- - zlib_codec_init - initialize the ZLIB codec --------------------------------------------------*/ - -chd_error zlib_codec_init(void *codec, uint32_t hunkbytes) -{ - int zerr; - chd_error err; - zlib_codec_data *data = (zlib_codec_data*)codec; - - (void)hunkbytes; - - /* clear the buffers */ - memset(data, 0, sizeof(zlib_codec_data)); - - /* init the inflater first */ - data->inflater.next_in = (Bytef *)data; /* bogus, but that's ok */ - data->inflater.avail_in = 0; - data->inflater.zalloc = zlib_fast_alloc; - data->inflater.zfree = zlib_fast_free; - data->inflater.opaque = &data->allocator; - zerr = inflateInit2(&data->inflater, -MAX_WBITS); - - /* convert errors */ - if (zerr == Z_MEM_ERROR) - err = CHDERR_OUT_OF_MEMORY; - else if (zerr != Z_OK) - err = CHDERR_CODEC_ERROR; - else - err = CHDERR_NONE; - - return err; -} - -/*------------------------------------------------- - zlib_codec_free - free data for the ZLIB - codec --------------------------------------------------*/ - -void zlib_codec_free(void *codec) -{ - zlib_codec_data *data = (zlib_codec_data *)codec; - - /* deinit the streams */ - if (data != NULL) - { - inflateEnd(&data->inflater); - - /* free our fast memory */ - zlib_allocator_free(&data->allocator); - } -} - -/*------------------------------------------------- - zlib_codec_decompress - decompress data using - the ZLIB codec --------------------------------------------------*/ - -chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) -{ - zlib_codec_data *data = (zlib_codec_data *)codec; - int zerr; - - /* reset the decompressor */ - data->inflater.next_in = (Bytef *)src; - data->inflater.avail_in = complen; - data->inflater.total_in = 0; - data->inflater.next_out = (Bytef *)dest; - data->inflater.avail_out = destlen; - data->inflater.total_out = 0; - zerr = inflateReset(&data->inflater); - if (zerr != Z_OK) - return CHDERR_DECOMPRESSION_ERROR; - - /* do it */ - zerr = inflate(&data->inflater, Z_FINISH); - if (data->inflater.total_out != destlen) - return CHDERR_DECOMPRESSION_ERROR; - - return CHDERR_NONE; -} - -/*------------------------------------------------- - zlib_fast_alloc - fast malloc for ZLIB, which - allocates and frees memory frequently --------------------------------------------------*/ - -/* Huge alignment values for possible SIMD optimization by compiler (NEON, SSE, AVX) */ -#define ZLIB_MIN_ALIGNMENT_BITS 512 -#define ZLIB_MIN_ALIGNMENT_BYTES (ZLIB_MIN_ALIGNMENT_BITS / 8) - -static voidpf zlib_fast_alloc(voidpf opaque, zlib_alloc_size items, zlib_alloc_size size) -{ - zlib_allocator *alloc = (zlib_allocator *)opaque; - uintptr_t paddr = 0; - uint32_t *ptr; - int i; - - /* compute the size, rounding to the nearest 1k */ - size = (size * items + 0x3ff) & ~0x3ff; - - /* reuse a hunk if we can */ - for (i = 0; i < MAX_ZLIB_ALLOCS; i++) - { - ptr = alloc->allocptr[i]; - if (ptr && size == *ptr) - { - /* set the low bit of the size so we don't match next time */ - *ptr |= 1; - - /* return aligned block address */ - return (voidpf)(alloc->allocptr2[i]); - } - } - - /* alloc a new one */ - ptr = (uint32_t *)malloc(size + sizeof(uint32_t) + ZLIB_MIN_ALIGNMENT_BYTES); - if (!ptr) - return NULL; - - /* put it into the list */ - for (i = 0; i < MAX_ZLIB_ALLOCS; i++) - if (!alloc->allocptr[i]) - { - alloc->allocptr[i] = ptr; - paddr = (((uintptr_t)ptr) + sizeof(uint32_t) + (ZLIB_MIN_ALIGNMENT_BYTES-1)) & (~(ZLIB_MIN_ALIGNMENT_BYTES-1)); - alloc->allocptr2[i] = (uint32_t*)paddr; - break; - } - - /* set the low bit of the size so we don't match next time */ - *ptr = size | 1; - - /* return aligned block address */ - return (voidpf)paddr; -} - -/*------------------------------------------------- - zlib_fast_free - fast free for ZLIB, which - allocates and frees memory frequently --------------------------------------------------*/ - -static void zlib_fast_free(voidpf opaque, voidpf address) -{ - zlib_allocator *alloc = (zlib_allocator *)opaque; - uint32_t *ptr = (uint32_t *)address; - int i; - - /* find the hunk */ - for (i = 0; i < MAX_ZLIB_ALLOCS; i++) - if (ptr == alloc->allocptr2[i]) - { - /* clear the low bit of the size to allow matches */ - *(alloc->allocptr[i]) &= ~1; - return; - } -} - -/*------------------------------------------------- - zlib_allocator_free --------------------------------------------------*/ -static void zlib_allocator_free(voidpf opaque) -{ - zlib_allocator *alloc = (zlib_allocator *)opaque; - int i; - - for (i = 0; i < MAX_ZLIB_ALLOCS; i++) - if (alloc->allocptr[i]) - free(alloc->allocptr[i]); -} +#include "codec_zlib.h" + +#include +#include +#include +#ifdef CHDR_DEBUG_ZLIB +#include +#endif + +#ifdef CHDR_SYSTEM_ZLIB +static voidpf zlib_fast_alloc(voidpf opaque, zlib_alloc_size items, zlib_alloc_size size); +static void zlib_fast_free(voidpf opaque, voidpf address); +static void zlib_allocator_free(voidpf opaque); +#endif + +/*------------------------------------------------- + zlib_codec_init - initialize the ZLIB codec +-------------------------------------------------*/ + +#ifndef CHDR_SYSTEM_ZLIB + +/* ---- bundled miniz: drive tinfl directly, no 32KB dictionary ---- */ + +chd_error zlib_codec_init(void *codec, uint32_t hunkbytes) +{ + zlib_codec_data *data = (zlib_codec_data *)codec; + + (void)hunkbytes; + + memset(data, 0, sizeof(zlib_codec_data)); + data->inflater = (tinfl_decompressor *)malloc(sizeof(tinfl_decompressor)); + if (data->inflater == NULL) + return CHDERR_OUT_OF_MEMORY; + tinfl_init(data->inflater); + return CHDERR_NONE; +} + +void zlib_codec_free(void *codec) +{ + zlib_codec_data *data = (zlib_codec_data *)codec; + + if (data != NULL && data->inflater != NULL) + { + free(data->inflater); + data->inflater = NULL; + } +} + +chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) +{ + zlib_codec_data *data = (zlib_codec_data *)codec; + size_t in_bytes = complen; + size_t out_bytes = destlen; + tinfl_status status; + + if (data->inflater == NULL) + return CHDERR_DECOMPRESSION_ERROR; + + /* one hunk == one complete raw-deflate stream, decoded in a single call + * into a buffer that holds all of it: no zlib header, no further input to + * come, and the output buffer doubles as the dictionary */ + tinfl_init(data->inflater); + status = tinfl_decompress(data->inflater, (const mz_uint8 *)src, &in_bytes, + (mz_uint8 *)dest, (mz_uint8 *)dest, &out_bytes, + TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF); + + if (status != TINFL_STATUS_DONE || out_bytes != destlen) { +#ifdef CHDR_DEBUG_ZLIB + printf("zlib_codec_decompress: FAILED complen=%u destlen=%u status=%d in_used=%u out=%u src=", + (unsigned)complen, (unsigned)destlen, (int)status, + (unsigned)in_bytes, (unsigned)out_bytes); + for (uint32_t dbg_i = 0; dbg_i < complen; dbg_i++) + printf("%02x", src[dbg_i]); + printf("\n"); +#endif + return CHDERR_DECOMPRESSION_ERROR; + } + + return CHDERR_NONE; +} + +#else /* CHDR_SYSTEM_ZLIB */ + +chd_error zlib_codec_init(void *codec, uint32_t hunkbytes) +{ + int zerr; + chd_error err; + zlib_codec_data *data = (zlib_codec_data*)codec; + + (void)hunkbytes; + + /* clear the buffers */ + memset(data, 0, sizeof(zlib_codec_data)); + + /* init the inflater first */ + data->inflater.next_in = (Bytef *)data; /* bogus, but that's ok */ + data->inflater.avail_in = 0; + data->inflater.zalloc = zlib_fast_alloc; + data->inflater.zfree = zlib_fast_free; + data->inflater.opaque = &data->allocator; + zerr = inflateInit2(&data->inflater, -MAX_WBITS); + + /* convert errors */ + if (zerr == Z_MEM_ERROR) + err = CHDERR_OUT_OF_MEMORY; + else if (zerr != Z_OK) + err = CHDERR_CODEC_ERROR; + else + err = CHDERR_NONE; + + return err; +} + +/*------------------------------------------------- + zlib_codec_free - free data for the ZLIB + codec +-------------------------------------------------*/ + +void zlib_codec_free(void *codec) +{ + zlib_codec_data *data = (zlib_codec_data *)codec; + + /* deinit the streams */ + if (data != NULL) + { + inflateEnd(&data->inflater); + + /* free our fast memory */ + zlib_allocator_free(&data->allocator); + } +} + +/*------------------------------------------------- + zlib_codec_decompress - decompress data using + the ZLIB codec +-------------------------------------------------*/ + +chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) +{ + zlib_codec_data *data = (zlib_codec_data *)codec; + int zerr; + + /* reset the decompressor */ + data->inflater.next_in = (Bytef *)src; + data->inflater.avail_in = complen; + data->inflater.total_in = 0; + data->inflater.next_out = (Bytef *)dest; + data->inflater.avail_out = destlen; + data->inflater.total_out = 0; + zerr = inflateReset(&data->inflater); + if (zerr != Z_OK) { +#ifdef CHDR_DEBUG_ZLIB + printf("zlib_codec_decompress: inflateReset FAILED zerr=%d\n", zerr); +#endif + return CHDERR_DECOMPRESSION_ERROR; + } + + /* do it */ + zerr = inflate(&data->inflater, Z_FINISH); + if (data->inflater.total_out != destlen) { +#ifdef CHDR_DEBUG_ZLIB + /* only dump on the failure path - an unconditional per-call hex + * dump of every compressed block is too slow/UART-heavy to run + * across a real multi-hundred-file corpus (was previously seen to + * destabilize a full run outright). */ + printf("zlib_codec_decompress: FAILED complen=%u destlen=%u zerr=%d total_out=%u avail_in=%u avail_out=%u data=%p inflater.state=%p src=", + (unsigned)complen, (unsigned)destlen, zerr, (unsigned)data->inflater.total_out, + (unsigned)data->inflater.avail_in, (unsigned)data->inflater.avail_out, + (void*)data, (void*)data->inflater.state); + for (uint32_t dbg_i = 0; dbg_i < complen; dbg_i++) + printf("%02x", src[dbg_i]); + printf("\n"); +#endif + return CHDERR_DECOMPRESSION_ERROR; + } + + return CHDERR_NONE; +} + +#endif /* CHDR_SYSTEM_ZLIB */ + +#ifdef CHDR_SYSTEM_ZLIB + +/*------------------------------------------------- + zlib_fast_alloc - fast malloc for ZLIB, which + allocates and frees memory frequently +-------------------------------------------------*/ + +/* Huge alignment values for possible SIMD optimization by compiler (NEON, SSE, AVX) */ +#define ZLIB_MIN_ALIGNMENT_BITS 512 +#define ZLIB_MIN_ALIGNMENT_BYTES (ZLIB_MIN_ALIGNMENT_BITS / 8) + +static voidpf zlib_fast_alloc(voidpf opaque, zlib_alloc_size items, zlib_alloc_size size) +{ + zlib_allocator *alloc = (zlib_allocator *)opaque; + uintptr_t paddr = 0; + uint32_t *ptr; + int i; + + /* compute the size, rounding to the nearest 1k */ + size = (size * items + 0x3ff) & ~0x3ff; + + /* reuse a hunk if we can */ + for (i = 0; i < MAX_ZLIB_ALLOCS; i++) + { + ptr = alloc->allocptr[i]; + if (ptr && size == *ptr) + { + /* set the low bit of the size so we don't match next time */ + *ptr |= 1; + + /* return aligned block address */ + return (voidpf)(alloc->allocptr2[i]); + } + } + + /* alloc a new one */ + ptr = (uint32_t *)malloc(size + sizeof(uint32_t) + ZLIB_MIN_ALIGNMENT_BYTES); + if (!ptr) + return NULL; + + /* put it into the list */ + for (i = 0; i < MAX_ZLIB_ALLOCS; i++) + if (!alloc->allocptr[i]) + { + alloc->allocptr[i] = ptr; + paddr = (((uintptr_t)ptr) + sizeof(uint32_t) + (ZLIB_MIN_ALIGNMENT_BYTES-1)) & (~(ZLIB_MIN_ALIGNMENT_BYTES-1)); + alloc->allocptr2[i] = (uint32_t*)paddr; + break; + } + + /* set the low bit of the size so we don't match next time */ + *ptr = size | 1; + + /* return aligned block address */ + return (voidpf)paddr; +} + +/*------------------------------------------------- + zlib_fast_free - fast free for ZLIB, which + allocates and frees memory frequently +-------------------------------------------------*/ + +static void zlib_fast_free(voidpf opaque, voidpf address) +{ + zlib_allocator *alloc = (zlib_allocator *)opaque; + uint32_t *ptr = (uint32_t *)address; + int i; + + /* find the hunk */ + for (i = 0; i < MAX_ZLIB_ALLOCS; i++) + if (ptr == alloc->allocptr2[i]) + { + /* clear the low bit of the size to allow matches */ + *(alloc->allocptr[i]) &= ~1; + return; + } +} + +/*------------------------------------------------- + zlib_allocator_free +-------------------------------------------------*/ +static void zlib_allocator_free(voidpf opaque) +{ + zlib_allocator *alloc = (zlib_allocator *)opaque; + int i; + + for (i = 0; i < MAX_ZLIB_ALLOCS; i++) + if (alloc->allocptr[i]) + free(alloc->allocptr[i]); +} + +#endif /* CHDR_SYSTEM_ZLIB */ diff --git a/src/libchdr_flac.c b/src/libchdr_flac.c index d0f29d7..533c126 100644 --- a/src/libchdr_flac.c +++ b/src/libchdr_flac.c @@ -8,12 +8,34 @@ ***************************************************************************/ +#include #include +#include "../include/libchdr/chdconfig.h" #include "../include/libchdr/flac.h" #include "../include/libchdr/macros.h" #define DR_FLAC_IMPLEMENTATION #define DR_FLAC_NO_STDIO + +/* dr_flac CRC-checks every FLAC frame it decodes. When libchdr is also + * verifying each decoded hunk against the CRC chdman stored, that is the same + * data checked twice: a corrupt frame that dr_flac would reject instead decodes + * to garbage, and the hunk CRC rejects it one level up with the same + * CHDERR_DECOMPRESSION_ERROR. Dropping the inner check is worth ~8% of a CD-FLAC + * hunk and 13 KB of text on RV32, where the compiler can then discard dr_flac's + * CRC-8 and CRC-16 tables entirely. + * + * Deliberately tied to VERIFY_BLOCK_CRC and not to any "small target" switch: + * VERIFY_BLOCK_CRC is exactly the thing that makes it safe. Without it the + * frame CRC is the only integrity check FLAC data gets. + * + * DR_FLAC_NO_CRC also disables binary-search seeking, which libchdr never uses + * - each hunk is opened as a complete stream and read straight through, and + * drflac_seek_to_pcm_frame() is never called. */ +#if VERIFY_BLOCK_CRC +#define DR_FLAC_NO_CRC +#endif + #include "../include/dr_libs/dr_flac.h" /*************************************************************************** @@ -52,6 +74,7 @@ int flac_decoder_init(flac_decoder *decoder) decoder->uncompressed_offset = 0; decoder->uncompressed_length = 0; decoder->uncompressed_swap = 0; + decoder->alloc_failed = 0; return 0; } @@ -74,14 +97,53 @@ void flac_decoder_free(flac_decoder* decoder) *------------------------------------------------- */ +/* drflac_open_with_metadata() allocates the decoder plus a decoded-sample + * buffer sized from the STREAMINFO block (for a CD-FLAC hunk that is ~40KB), + * and reset() is called once per hunk - so on a small-RAM target this is the + * single most likely thing to fail here. Route it through callbacks that + * record an allocation failure, so callers can report CHDERR_OUT_OF_MEMORY + * instead of lumping it in with a genuine CHDERR_DECOMPRESSION_ERROR. */ + +static void *flac_decoder_malloc_callback(size_t sz, void *userdata) +{ + flac_decoder *decoder = (flac_decoder *)userdata; + void *ptr = malloc(sz); + if (ptr == NULL) + decoder->alloc_failed = 1; + return ptr; +} + +static void *flac_decoder_realloc_callback(void *ptr, size_t sz, void *userdata) +{ + flac_decoder *decoder = (flac_decoder *)userdata; + void *newptr = realloc(ptr, sz); + if (newptr == NULL) + decoder->alloc_failed = 1; + return newptr; +} + +static void flac_decoder_free_callback(void *ptr, void *userdata) +{ + (void)userdata; + free(ptr); +} + static int flac_decoder_internal_reset(flac_decoder* decoder) { + drflac_allocation_callbacks callbacks; + + callbacks.pUserData = decoder; + callbacks.onMalloc = flac_decoder_malloc_callback; + callbacks.onRealloc = flac_decoder_realloc_callback; + callbacks.onFree = flac_decoder_free_callback; + decoder->compressed_offset = 0; + decoder->alloc_failed = 0; flac_decoder_free(decoder); decoder->decoder = drflac_open_with_metadata( flac_decoder_read_callback, flac_decoder_seek_callback, flac_decoder_tell_callback, flac_decoder_metadata_callback, - decoder, NULL); + decoder, &callbacks); return (decoder->decoder != NULL); } diff --git a/src/libchdr_huffman.c b/src/libchdr_huffman.c index b22380b..16d96b5 100644 --- a/src/libchdr_huffman.c +++ b/src/libchdr_huffman.c @@ -582,6 +582,18 @@ enum huffman_error huffman_build_lookup_table(struct huffman_decoder* decoder) for (i = 0; i < l1size; i++) prefix_subid[i] = -1; + /* The whole lookup is rebuilt from scratch here, so the subtable arena + * starts empty every time. Without this reset the count carried over + * between calls - a decoder is created once per codec instance + * (huff_codec_init) and reused for every hunk, so subtable_count grew + * monotonically across hunks until it tripped the 2048 guard below and + * every subsequent huffman hunk failed with HUFFERR_TOO_MANY_CONTEXTS. + * That surfaced as a history-dependent CHDERR_DECOMPRESSION_ERROR: + * byte-identical input decoded fine or failed depending only on how many + * huffman-coded hunks had been read before it. The allocation itself is + * kept and grown by realloc, so resetting the count costs nothing. */ + decoder->subtable_count = 0; + /* Canonical-code coverage of the table is only guaranteed complete when * every assigned length participates in the Kraft-equality check in * huffman_assign_canonical_codes() (length 1 is exempt there, and a