LOWRAM_TARGET tuning: two-level huffman table, demand-grown scratch buffer, checkpoint stride, sequential resume-cache

.github/workflows/lowram-correctness.yml

@@ -0,0 +1,61 @@
+name: LOWRAM_TARGET correctness
+
+# Builds libchdr twice (default config and CHDR_LOWRAM_TARGET=ON) and diffs
+# decoded output byte-for-byte across the full corpus in several read
+# orders, instead of only relying on the internal per-hunk CRC check.
+#
+# This exists because of a real bug: LOWRAM_TARGET's sequential resume-cache
+# (v5_map_get_entry's fast path) silently corrupted the decode at every
+# LOWRAM_TARGET_CHECKPOINT_STRIDE checkpoint boundary until a next_boundary
+# check was added. The existing synthetic corpus is too small to ever cross
+# a stride-2048 boundary, so it did not catch this - only a manual sweep
+# against large real-world CHDs did. tests/corpus/generate.sh's
+# raw_boundary.chd (2,200 hunks of 64 B, deliberately crossing the default
+# stride) closes that gap; this workflow is what actually runs it on every
+# push/PR.
+
+on: [push, pull_request]
+
+jobs:
+  lowram-correctness:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install chdman
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends mame-tools
+
+      - name: Generate corpus seeds
+        run: tests/corpus/generate.sh
+
+      - name: Build (default config)
+        run: |
+          cmake -B build-default -DCMAKE_BUILD_TYPE=Release
+          cmake --build build-default --target chd_dump_order -j$(nproc)
+
+      - name: Build (CHDR_LOWRAM_TARGET=ON)
+        run: |
+          cmake -B build-lowram -DCMAKE_BUILD_TYPE=Release -DCHDR_LOWRAM_TARGET=ON
+          cmake --build build-lowram --target chd_dump_order -j$(nproc)
+
+      - name: Byte-identical decode, every seed x every read order
+        run: |
+          set -euo pipefail
+          D=build-default/tests/chd_dump_order
+          L=build-lowram/tests/chd_dump_order
+          fail=0
+          total=0
+          for f in tests/corpus/seeds/*.chd; do
+            for order in sequential reverse random:1 random:42; do
+              total=$((total+1))
+              "$D" "$f" "$order" > /tmp/d.bin
+              "$L" "$f" "$order" > /tmp/l.bin
+              if ! cmp -s /tmp/d.bin /tmp/l.bin; then
+                echo "MISMATCH: $(basename "$f") $order"
+                fail=$((fail+1))
+              fi
+            done
+          done
+          echo "$total checks, $fail mismatches"
+          [ "$fail" -eq 0 ]

.github/workflows/rv32-ram-budget.yml

@@ -11,16 +11,16 @@ name: RV32 RAM budget
 # saves another ~3% via cross-TU dead-code elimination - both measured and
 # CRC-verified correct under qemu-system-riscv32.
 #
-# Also builds with CHDR_LOWRAM_MAP=ON: replaces the fully-materialized
+# Also builds with CHDR_LOWRAM_TARGET=ON: replaces the fully-materialized
 # per-hunk map (12B/hunk for CHDv5, ~24B/hunk legacy - scales with total
 # hunk count, independent of codec/hunkbytes) with a checkpointed on-demand
 # decode. Barely visible on this workflow's tiny synthetic corpus (few
 # hunks/file), but on real full-size discs it's the dominant RAM cost -
 # measured 48-74% peak-heap reduction on real GD-ROM/UMD CHDs (naomi,
-# dreamcast, psp), CRC-verified byte-identical against the non-LOWRAM_MAP
+# dreamcast, psp), CRC-verified byte-identical against the non-LOWRAM_TARGET
 # build across sequential/reverse/random-order reads. Real-ROM validation
 # isn't reproducible in CI (copyrighted files), so this workflow's job is
-# proving LOWRAM_MAP=ON doesn't regress correctness or blow past budget on
+# proving LOWRAM_TARGET=ON doesn't regress correctness or blow past budget on
 # what CI *can* see - the real-file numbers were measured locally.
 
 on: [push, pull_request]
@@ -44,7 +44,7 @@ jobs:
           cmake -B build-rv32
           -DCMAKE_TOOLCHAIN_FILE=${{github.workspace}}/cmake/toolchain-rv32imafc.cmake
           -DBUILD_SHARED_LIBS=OFF -DINSTALL_STATIC_LIBS=OFF
-          -DCMAKE_BUILD_TYPE=MinSizeRel -DCHDR_LOWRAM_MAP=ON
+          -DCMAKE_BUILD_TYPE=MinSizeRel -DCHDR_LOWRAM_TARGET=ON
 
       - name: Build chdr-static
         run: cmake --build build-rv32 --target chdr-static -j$(nproc)

CMakeLists.txt

@@ -11,7 +11,7 @@ option(WITH_SYSTEM_ZSTD "Use system provided zstd library" OFF)
 option(CHDR_WANT_RAW_DATA_SECTOR "Output ECC data and sync header" ON)
 option(CHDR_WANT_SUBCODE "Output CD subchannel data" ON)
 option(CHDR_VERIFY_BLOCK_CRC "Verify integrity of decoded data" ON)
-option(CHDR_LOWRAM_MAP "Trade CPU for RAM on the per-hunk map (checkpointed on-demand decode instead of fully materializing it at chd_open) - for memory-constrained targets" OFF)
+option(CHDR_LOWRAM_TARGET "Trade CPU for RAM on the per-hunk map (checkpointed on-demand decode instead of fully materializing it at chd_open) - for memory-constrained targets" OFF)
 option(CHDR_WANT_TESTS "Build tests for the library" ON)
 
 option(BUILD_LTO "Compile libchdr with link-time optimization if supported" OFF)
@@ -93,10 +93,10 @@ else()
   list(APPEND CHDR_DEFINES VERIFY_BLOCK_CRC=0)
 endif()
 
-if(CHDR_LOWRAM_MAP)
-  list(APPEND CHDR_DEFINES LOWRAM_MAP=1)
+if(CHDR_LOWRAM_TARGET)
+  list(APPEND CHDR_DEFINES LOWRAM_TARGET=1)
 else()
-  list(APPEND CHDR_DEFINES LOWRAM_MAP=0)
+  list(APPEND CHDR_DEFINES LOWRAM_TARGET=0)
 endif()
 
 #--------------------------------------------------

include/libchdr/bitstream.h

@@ -40,7 +40,7 @@ void 				bitstream_remove(struct bitstream* bitstream, int numbits);
 uint32_t 			bitstream_flush(struct bitstream* bitstream);
 
 /* exact bit-granular position/seek, independent of byte alignment - used to
- * checkpoint/resume mid-stream decode (see LOWRAM_MAP in libchdr_chd.c) */
+ * checkpoint/resume mid-stream decode (see LOWRAM_TARGET in libchdr_chd.c) */
 uint64_t 			bitstream_position_bits(struct bitstream* bitstream);
 void 				bitstream_seek_bits(struct bitstream* bitstream, uint64_t bitpos);
 

include/libchdr/chdconfig.h

@@ -15,19 +15,42 @@
 #define VERIFY_BLOCK_CRC        1
 #endif
 
-/* Trade CPU for RAM on the per-hunk map: instead of fully materializing it
- * at chd_open() (12 bytes/hunk for CHDv5, ~24 bytes/hunk for legacy v1-v4 -
- * scales with total hunk count, independent of codec/hunkbytes choice, and
- * can reach multiple MB on full-size CD/GD-ROM/UMD images), keep only a
- * sparse checkpoint index and re-derive individual entries on demand. For
- * memory-constrained targets. See LOWRAM_MAP_CHECKPOINT_STRIDE to tune the
- * RAM/CPU tradeoff. */
-#ifndef LOWRAM_MAP
-#define LOWRAM_MAP              0
+/* Trade CPU for RAM across several independent levers, for
+ * memory-constrained targets (e.g. the BL616/RV32 port, 480KB SRAM):
+ *
+ * 1. Per-hunk map: instead of fully materializing it at chd_open()
+ *    (12 bytes/hunk for CHDv5, ~24 bytes/hunk for legacy v1-v4 - scales
+ *    with total hunk count, independent of codec/hunkbytes choice, and
+ *    can reach multiple MB on full-size CD/GD-ROM/UMD images), keep only
+ *    a sparse checkpoint index and re-derive individual entries on
+ *    demand. See LOWRAM_TARGET_CHECKPOINT_STRIDE to tune the RAM/CPU
+ *    tradeoff.
+ * 2. Each huffman_decoder's lookup table: a two-level table instead of
+ *    a full 2^maxbits direct table - see LOWRAM_TARGET_HUFFMAN_L1BITS
+ *    below.
+ * 3. The compressed-hunk scratch buffer (chd->compressed,
+ *    src/libchdr_chd.c): grown on demand to the largest hunk actually
+ *    read instead of preallocated to header.hunkbytes at chd_open().
+ */
+#ifndef LOWRAM_TARGET
+#define LOWRAM_TARGET              0
 #endif
 
-#ifndef LOWRAM_MAP_CHECKPOINT_STRIDE
-#define LOWRAM_MAP_CHECKPOINT_STRIDE 512
+#ifndef LOWRAM_TARGET_CHECKPOINT_STRIDE
+#define LOWRAM_TARGET_CHECKPOINT_STRIDE 2048
+#endif
+
+/* Under LOWRAM_TARGET, also replace each huffman_decoder's full 2^maxbits
+ * direct-lookup table (e.g. 128 KiB at maxbits=16, as used by AVHuff's
+ * Y/Cb/Cr contexts and the CHD huffman codec) with a two-level table: a
+ * 2^L1BITS first-level table, escaping to small per-prefix subtables only
+ * for the rare codes longer than L1BITS. Huffman assigns short codes to
+ * common symbols by construction, so the fast (non-escaping) path is
+ * unchanged; only long, rare codes pay an extra indirection. Total RAM
+ * drops to a few KiB per decoder regardless of maxbits.
+ */
+#ifndef LOWRAM_TARGET_HUFFMAN_L1BITS
+#define LOWRAM_TARGET_HUFFMAN_L1BITS 10
 #endif
 
 #endif

include/libchdr/huffman.h

@@ -14,6 +14,7 @@
 #define __HUFFMAN_H__
 
 #include "bitstream.h"
+#include "chdconfig.h"
 
 
 /***************************************************************************
@@ -59,9 +60,15 @@ struct huffman_decoder
 	uint8_t				maxbits;           /* maximum bits per code */
 	uint8_t 			prevdata;             /* value of the previous data (for delta-RLE encoding) */
 	int             	rleremaining;         /* number of RLE bytes remaining (for delta-RLE encoding) */
-	lookup_value *  	lookup;               /* pointer to the lookup table */
+	lookup_value *  	lookup;               /* pointer to the lookup table (full 2^maxbits table,
+	                                             or under LOWRAM_TARGET, the 2^l1bits first-level table) */
 	struct node_t *     huffnode;             /* array of nodes */
 	uint32_t *      	datahisto;            /* histogram of data values */
+#if LOWRAM_TARGET
+	uint8_t				l1bits;               /* first-level table width in bits, MIN(maxbits, LOWRAM_TARGET_HUFFMAN_L1BITS) */
+	lookup_value *		subtable;             /* concatenated second-level subtables, one per escaping first-level prefix */
+	uint32_t			subtable_count;       /* number of subtables currently allocated */
+#endif
 
 	/* array versions of the info we need */
 #if 0

src/libchdr_bitstream.c

@@ -52,7 +52,7 @@ uint32_t bitstream_peek(struct bitstream* bitstream, int numbits)
 		while (bitstream->bits <= 24)
 		{
 			if (bitstream->doffset < bitstream->dlength)
-				bitstream->buffer |= bitstream->read[bitstream->doffset] << (24 - bitstream->bits);
+				bitstream->buffer |= (uint32_t)bitstream->read[bitstream->doffset] << (24 - bitstream->bits);
 			bitstream->doffset++;
 			bitstream->bits += 8;
 		}