diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 58a208670..f666f2fcb 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -45,6 +45,7 @@ jobs: - name: build shell: bash run: | + [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22 CC="${{ matrix.compiler }}" ./build.sh "${{ matrix.target }}" "${{ matrix.mode }}" build-windows-2022: @@ -103,6 +104,7 @@ jobs: - name: Run shell: bash run: | + [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22 CC="${{ matrix.compiler }}" ./run_tests.sh clang "${{ matrix.mode }}" -- \* $TORTURE_FLAGS - name: Upload Artifacts if: failure() diff --git a/build.bat b/build.bat index e51a529eb..2aa2245dd 100644 --- a/build.bat +++ b/build.bat @@ -27,6 +27,7 @@ cd /D "%~dp0" :: - `ubsan`: enable undefined-behavior sanitizer :: - `telemetry`: enable RAD telemetry profiling support :: - `spall`: enable spall profiling support +:: - `oodle`: enable Oodle-compressed OBJ support in radlink (requires OODLE_SDK_DIR) :: --- Unpack Arguments ------------------------------------------------------- for %%a in (%*) do set "%%~a=1" @@ -53,13 +54,15 @@ if "%pgo%"=="1" ( where llvm-profdata /q || echo llvm-profdata is not in the PATH || exit /b 1 if "%clang%"=="1" ( if "%pgo_run%" == "1" ( - call llvm-profdata merge %LLVM_PROFILE_FILE% -output=%~dp0build\build.profdata || exit /b 1 + call llvm-profdata merge %~dp0build\pgo_raw\*.profraw -output=%~dp0build\build.profdata || exit /b 1 set auto_compile_flags=%auto_compile_flags% -fprofile-use=%~dp0build\build.profdata set pgo_run=0 ) else ( echo [pgo enabled] set auto_compile_flags=%auto_compile_flags% -fprofile-generate -mllvm -vp-counters-per-site=5 - set LLVM_PROFILE_FILE=%~dp0build\build.profraw + if not exist %~dp0build\pgo_raw mkdir %~dp0build\pgo_raw + del /q %~dp0build\pgo_raw\*.profraw 2>nul + set LLVM_PROFILE_FILE=%~dp0build\pgo_raw\build.%%p.profraw set pgo_run=1 ) ) else ( @@ -98,6 +101,10 @@ if "%msvc%"=="1" set rc=call rc if "%clang%"=="1" set rc=call llvm-rc if "%msvc%"=="1" set link_dll=/link /DLL if "%clang%"=="1" set link_dll=-Xlinker -DLL +if "%msvc%"=="1" set asm=call ml64 -nologo -c -Zi -Fo +if "%clang%"=="1" set asm=call llvm-ml -m64 /c /Fo +if "%msvc%"=="1" set mklib=call lib -nologo +if "%clang%"=="1" set mklib=call llvm-lib :: --- Choose Compile/Link Lines ---------------------------------------------- if "%msvc%"=="1" set compile_debug=%cl_debug% @@ -138,12 +145,43 @@ if "%no_meta%"=="" if exist metagen.exe ( ) popd +:: --- Assemble BLAKE3 -------------------------------------------------------- +if not exist build\blake3.lib if "%VSCMD_ARG_TGT_ARCH%" equ "x64" ( + echo [assembling blake3] + pushd build + %asm% blake3_sse2_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_sse2_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_sse41_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_sse41_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_avx2_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_avx2_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_avx512_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_avx512_x86-64_windows_msvc.asm || exit /b 1 + %mklib% -out:blake3.lib blake3_*_msvc.obj || exit /b 1 + popd +) + :: --- Build Everything (@build_targets) -------------------------------------- pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 if "%raddbg_non_graphical%"=="1" set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1 if "%com_shim%"=="1" set didbuild=1 && %compile% ..\src\com_shim\com_shim_main.c %compile_link% %out%com_shim.exe || exit /b 1 -if "%radlink%"=="1" set didbuild=1 && %compile% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +set radlink_oodle_flags= +set radlink_oodle_lib= +set rad_obj_compress_oodle_flags= +set rad_obj_compress_oodle_lib= +if "%oodle%"=="1" if "%OODLE_SDK_DIR%"=="" ( + echo OODLE_SDK_DIR must name an Oodle SDK when building radlink with oodle + exit /b 1 +) +if "%oodle%"=="1" if "%msvc%"=="1" set radlink_oodle_flags=/I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1 +if "%oodle%"=="1" if "%clang%"=="1" set radlink_oodle_flags=-I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1 +if "%oodle%"=="1" set radlink_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib" +if "%rad_obj_compress%"=="1" if "%OODLE_SDK_DIR%"=="" ( + echo OODLE_SDK_DIR must name an Oodle SDK when building rad_obj_compress + exit /b 1 +) +if "%rad_obj_compress%"=="1" if "%msvc%"=="1" set rad_obj_compress_oodle_flags=/I"%OODLE_SDK_DIR%\include" +if "%rad_obj_compress%"=="1" if "%clang%"=="1" set rad_obj_compress_oodle_flags=-I"%OODLE_SDK_DIR%\include" +if "%rad_obj_compress%"=="1" set rad_obj_compress_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib" +if "%radlink%"=="1" set didbuild=1 && %compile% %radlink_oodle_flags% ..\src\linker\lnk.c %radlink_oodle_lib% %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +if "%rad_obj_compress%"=="1" set didbuild=1 && %compile% %rad_obj_compress_oodle_flags% ..\src\linker\rad_obj_compress.c %rad_obj_compress_oodle_lib% %compile_link% %out%rad_obj_compress.exe || exit /b 1 if "%radbin%"=="1" set didbuild=1 && %compile% ..\src\radbin\radbin_main.c %compile_link% %out%radbin.exe || exit /b 1 if "%raddump%"=="1" set didbuild=1 && %compile% ..\src\raddump\raddump_main.c %compile_link% %out%raddump.exe || exit /b 1 if "%ryan_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c %compile_link% %out%ryan_scratch.exe || exit /b 1 @@ -184,6 +222,10 @@ if "%pgo_run%"=="1" ( if "%radlink%"=="1" ( pushd local\lyra_pgo call %~dp0build\radlink @lyra.rsp || exit /b 1 + rem lyra.rsp trains with /OPT:NOREF /OPT:NOICF; run a second link with REF+ICF on + rem (later switches win) so the profile also covers the /OPT:REF walk and the ICF + rem refinement -- otherwise PGO marks them cold and the build regresses on real links + call %~dp0build\radlink @lyra.rsp /OPT:REF /OPT:ICF || exit /b 1 popd ) goto restart diff --git a/build.sh b/build.sh index 3724a40d4..a3d119505 100755 --- a/build.sh +++ b/build.sh @@ -42,8 +42,8 @@ fi cc_icon="-DLNX_WM_ICON=1" # --- Choose Compile/Link Lines ----------------------------------------------- -if [[ "${gcc:-0}" == "1" ]]; then compiler="${CC:-gcc} $cc_cflags_gcc"; echo "[gcc compile]"; -elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; echo "[clang compile]"; +if [[ "${gcc:-0}" == "1" ]]; then compiler="${CC:-gcc} $cc_cflags_gcc"; ar="${AR:-ar}"; echo "[gcc compile]"; +elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; ar="${AR:-llvm-ar}"; echo "[clang compile]"; fi if [[ "${release:-0}" == "1" ]]; then echo "[release mode]"; compile="$compiler $cc_release"; elif [[ "${debug:-1}" == "1" ]]; then echo "[debug mode]"; compile="$compiler $cc_debug"; @@ -62,12 +62,23 @@ then cd .. fi +# --- Assemble BLAKE3 --------------------------------------------------------- +if [[ ! -f "build/blake3.a" ]] +then + echo "[assembling blake3]" + $compiler -c -g -o build/blake3_sse2_x86-64_unix.o src/third_party/blake3/blake3_sse2_x86-64_unix.S + $compiler -c -g -o build/blake3_sse41_x86-64_unix.o src/third_party/blake3/blake3_sse41_x86-64_unix.S + $compiler -c -g -o build/blake3_avx2_x86-64_unix.o src/third_party/blake3/blake3_avx2_x86-64_unix.S + $compiler -c -g -o build/blake3_avx512_x86-64_unix.o src/third_party/blake3/blake3_avx512_x86-64_unix.S + $ar rs build/blake3.a build/blake3_*_unix.o +fi + # --- Build Everything (@build_targets) --------------------------------------- cd build if [[ "${raddbg:-0}" == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c $cc_icon $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg; fi if [[ "${raddbg_non_graphical:-0}" == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg_non_graphical; fi if [[ "${radbin:-0}" == "1" ]]; then didbuild=1 && $compile ../src/radbin/radbin_main.c $cc_link -o radbin; fi -if [[ "${radlink:-0}" == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c $cc_link -o radlink; fi +if [[ "${radlink:-0}" == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c $cc_link -o radlink blake3.a; fi if [[ "${torture:-0}" == "1" ]]; then didbuild=1 && $compile ../src/torture/torture_main.c $cc_link $cc_os_gfx $cc_render $cc_font_provider -o torture; fi cd .. diff --git a/docs/compressed_obj_format.md b/docs/compressed_obj_format.md new file mode 100644 index 000000000..b4ba6ada5 --- /dev/null +++ b/docs/compressed_obj_format.md @@ -0,0 +1,695 @@ +# RAD Link portable compressed OBJ format + +This document is the implementation contract for producing portable compressed OBJ files that the +current RAD Link compressed-OBJ reader accepts. It describes format version 1, identified by the +eight-byte ASCII magic `RLOBJ001`. + +The intended reader and reference writer are: + +- `src/linker/lnk_compressed_obj_format.h` +- `src/linker/lnk_compressed_obj.c` +- `src/linker/rad_obj_compress.c` + +If this document and the reader ever disagree, the reader is authoritative. A format change must +use a new version or magic; silently changing version 1 would invalidate cached objects. + +## 1. Implementation summary + +For the tested production profile, the writer should: + +1. Finish producing one normal COFF or BigObj byte stream in memory. +2. Split that byte stream into independent 512 KiB logical segments. +3. Compress each segment independently with Oodle Kraken, level Normal, and + `spaceSpeedTradeoffBytes = 256`. +4. Store a segment raw when compression fails or does not make it smaller. +5. Emit the 64-byte `RLOBJ001` header, the 24-byte-per-segment directory, the performance + sidecars described below, and then the segment payloads. +6. Verify every compressed segment by decoding it and comparing it with the original bytes. +7. Publish through a temporary file plus an atomic rename. + +The following settings produced the measured 22.97 GiB FortniteClient and 24.39 GiB UEFN corpora: + +| Setting | Required/recommended value | +|---|---| +| Container | `RLOBJ001`, version 1 | +| Byte order | Little-endian | +| Segment size | 512 KiB (`524288`) | +| Oodle compressor | Kraken (`OodleLZ_Compressor_Kraken`, value 8 in Oodle 2.9.16) | +| Oodle level | Normal | +| `spaceSpeedTradeoffBytes` | 256 | +| Segment dependencies | None; every segment is a separate Oodle stream | +| Incompressible data | Store raw | +| Filesystem features | None required; ordinary portable file | + +All compressed OBJ inputs in a single RAD Link invocation **must use exactly the same segment +size**. The cache is initialized from the first compressed OBJ and rejects a later compressed OBJ +with a different segment size. Use 512 KiB consistently for the entire corpus. + +This is **not** the UBA 12-byte-header format with an eight-byte size prefix between segments. UBA +payloads can use the same Oodle codec, but they must be repackaged into the `RLOBJ001` header and +directory layout described here before this RAD Link reader can consume them. + +## 2. Compatibility levels + +There are two useful conformance levels. + +### 2.1 Core container: correct but not performance-equivalent + +A minimal compatible file needs only: + +- the fixed header; +- the segment directory; +- independently compressed or raw-stored segment payloads; and +- the `PORTABLE_RAW_MAP` header flag. + +With no sidecar flags, set `header.reserved` to zero. RAD Link can reconstruct the original OBJ +view and link it correctly, but it must fault or decode more debug information through the generic +cache. This level is useful for bringing up and validating a new writer, but it is not expected to +match the benchmarked link time or memory use. + +### 2.2 Optimized container: the measured production profile + +To reproduce the measured behavior, also emit: + +- the packed `.debug$T`/`.debug$P` leaf index; +- the complete-UDT hash index; +- the AMD64 base-relocation candidate index; +- the `.debug$S` subsection index; and +- the `.debug$S` summaries. + +These sidecars are optimization hints derived entirely from the original OBJ. They do not replace +the original bytes. Incorrect sidecar data can change linker behavior, so omit an optional sidecar +until its implementation is known to be correct rather than emitting an approximation. + +## 3. General encoding rules + +- All integers are unsigned and little-endian. +- File offsets are absolute offsets from byte zero of the compressed file. +- Logical or raw offsets refer to offsets in the original uncompressed OBJ byte stream. +- `align_up(x, n)` means `(x + n - 1) & ~(n - 1)`, with `n` a power of two. +- All fields named `reserved` must be written as zero unless this document assigns them a meaning. +- Do not serialize the records with `#pragma pack(1)`. Use the byte offsets below, or use the C + declarations with their normal layout and assert every size. +- The ordinary output file length must cover all metadata, stored payloads, and required raw-run + padding. The format does not depend on sparse files, filesystem compression, bundles, or another + OBJ. +- Direct compressed inputs must currently use the `.obj` or `.o` extension. Use `.obj` for the + Windows toolchain. RAD Link's command-line parser classifies the path by extension before reading + the magic, so a path ending in `.radobj` is rejected as an unknown file format even when its + contents are a valid `RLOBJ001` container. The stored bytes are a RAD Link container rather than + a COFF file, and other linkers will not understand it. + +Recommended compile-time checks for a C or C++ implementation are: + +```cpp +static_assert(sizeof(LNK_CObjHeader) == 64); +static_assert(sizeof(LNK_CObjSegment) == 24); +static_assert(sizeof(LNK_CObjTypeIndex) == 40); +static_assert(sizeof(LNK_CObjUdtHashIndex) == 16); +static_assert(sizeof(LNK_CObjBaseRelocIndex) == 16); +static_assert(sizeof(LNK_CObjBaseRelocEntry) == 16); +static_assert(sizeof(LNK_CObjDebugSIndex) == 16); +static_assert(sizeof(LNK_CObjDebugSEntry) == 16); +static_assert(sizeof(LNK_CObjDebugSSummary) == 16); +``` + +## 4. Top-level file layout + +The reference writer uses this order: + +```text +0 ++-------------------------------+ +| 64-byte header | ++-------------------------------+ 8-byte aligned +| segment directory | segment_count * 24 bytes ++-------------------------------+ 8-byte aligned +| optional sidecar directories | ++-------------------------------+ 8-byte aligned +| optional sidecar payloads | ++-------------------------------+ 8-byte aligned +| segment payloads and padding | ++-------------------------------+ file length +``` + +Sidecar and segment offsets make the format relocatable inside the file. A production writer +should nevertheless use the canonical order above, keep ranges non-overlapping, and monotonically +advance a checked 64-bit layout cursor. + +## 5. Fixed header + +The header is exactly 64 bytes. + +| Offset | Type | Field | Value and meaning | +|---:|---:|---|---| +| 0 | `U64` | `magic` | `0x3130304A424F4C52`; bytes spell `RLOBJ001` | +| 8 | `U32` | `version` | `1` | +| 12 | `U32` | `header_size` | `64` | +| 16 | `U64` | `raw_size` | Exact byte length of the original OBJ; must be nonzero | +| 24 | `U32` | `segment_size` | Power of two, at least 64 KiB; use 512 KiB | +| 28 | `U32` | `segment_count` | `ceil(raw_size / segment_size)` | +| 32 | `U64` | `directory_offset` | Absolute offset of the segment directory; canonical value is 64 | +| 40 | `U64` | `data_offset` | Canonical first segment-payload position before payload alignment | +| 48 | `U32` | `compressor` | Oodle compressor enum; write 8 for Kraken | +| 52 | `U32` | `flags` | Container flags plus type-index count in bits 16-31 | +| 56 | `U64` | `reserved` | When sidecars exist, absolute offset of the sidecar directory block; otherwise zero | + +`data_offset` and `compressor` are descriptive in the version-1 reader; individual directory +entries locate payloads and Oodle identifies its stream internally. Writers must still populate +both fields correctly so future readers, diagnostics, and validators can rely on them. + +### 5.1 Header flags + +| Value | Name | Meaning | +|---:|---|---| +| `0x00000001` | `TYPE_INDEX` | One or more type-index directory records are present | +| `0x00000002` | retired | Never set; the portable reader rejects it | +| `0x00000004` | `UDT_HASH_INDEX` | One UDT-hash directory follows each type-index directory | +| `0x00000008` | `BASE_RELOC_INDEX` | A base-relocation directory is present | +| `0x00000010` | `PORTABLE_RAW_MAP` | Required for every version-1 portable file | +| `0x00000040` | `PACKED_TYPE_SIDECAR` | Type sidecars use the packed representation | +| `0x00000080` | `PACKED_TYPE_OFFSETS_V2` | Packed offset representation is version 2 | +| `0x00002000` | `DEBUG_S_INDEX` | A `.debug$S` directory is present | +| `0x00004000` | `DEBUG_S_SUMMARY` | One summary follows each `.debug$S` entry | +| `0xffff0000` | type count | Number of type-index records, shifted left by 16 | + +All unlisted bits must be zero. The reader rejects unknown bits. + +Flag dependencies are: + +- `UDT_HASH_INDEX` requires `TYPE_INDEX`. +- `DEBUG_S_SUMMARY` requires `DEBUG_S_INDEX`. +- `PACKED_TYPE_SIDECAR` and `PACKED_TYPE_OFFSETS_V2` must either both be set or both be clear. +- A nonzero type count requires `TYPE_INDEX`. +- `TYPE_INDEX` requires a nonzero type count, with a maximum of 65535. + +The packed flags have no purpose without a type index; a new writer should leave them clear when +there are no type records. + +## 6. Segment directory + +The segment directory contains `segment_count` consecutive 24-byte entries. Entry `i` describes +logical bytes beginning at `i * segment_size` in the original OBJ. + +| Entry offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `file_offset` | Absolute offset of the stored payload | +| 8 | `U32` | `stored_size` | Number of meaningful bytes stored at `file_offset` | +| 12 | `U32` | `raw_size` | Number of original bytes represented by this segment | +| 16 | `U32` | `flags` | Zero for Oodle data; `0x1` for raw data | +| 20 | `U32` | `reserved` | Zero | + +The only segment flag is: + +```text +LNK_COBJ_SEGMENT_RAW = 0x00000001 +``` + +For every entry: + +```text +raw_offset = i * segment_size +raw_size = min(segment_size, header.raw_size - raw_offset) +``` + +`stored_size` must be nonzero, and `[file_offset, file_offset + stored_size)` must be inside the +file. If `RAW` is set, `stored_size` must equal `raw_size`. If `RAW` is clear, the stored bytes must +be one complete Oodle stream that decodes to exactly `raw_size` bytes. + +The writer should choose raw storage when `OodleLZ_Compress` returns a nonpositive length or a +length greater than or equal to `raw_size`. This prevents compression from growing the file. + +### 6.1 Oodle call contract + +The tested writer initializes options from: + +```cpp +OodleLZ_Compressor compressor = OodleLZ_Compressor_Kraken; +OodleLZ_CompressionLevel level = OodleLZ_CompressionLevel_Normal; +OodleLZ_CompressOptions options = + *OodleLZ_CompressOptions_GetDefault(compressor, level); +options.spaceSpeedTradeoffBytes = 256; +OodleLZ_CompressOptions_Validate(&options); +``` + +It calls `OodleLZ_Compress` once per segment, with no dictionary base, long-range matcher, or shared +state. Do not continue an Oodle stream from one segment into the next. RAD Link may request any +segment first and may decode several segments concurrently. + +RAD Link decodes with fuzz safety enabled, quantum CRC checking disabled, no caller scratch, and +`OodleLZ_Decode_Unthreaded`. The writer currently does not request Oodle quantum CRCs. Integrity of +the complete portable object should be provided by the cache system's normal content hash. + +Do not mix Oodle codecs within one object. Version 1 has only one header-level compressor field, +even though the current Oodle decode entry point can identify a stream without consulting it. + +### 6.2 Raw-segment placement + +Raw segments are mapped directly from the portable file into RAD Link's logical OBJ address range. +Their layout therefore has additional requirements beyond the basic directory validation. + +For each maximal run of consecutive raw segments: + +1. Align the first raw segment's `file_offset` to 64 KiB. +2. Store full raw segments contiguously at `segment_size` intervals. +3. Because `segment_size` is a power of two and at least 64 KiB, every segment in the run remains + 64 KiB aligned. +4. If the final OBJ segment is raw and shorter than `segment_size`, extend the physical file through + `align_up(raw_size, 64 KiB)` bytes for that final segment. Padding bytes may be zero and are not + part of `stored_size`. + +Canonical payload-cursor logic is: + +```text +in_raw_run = false +cursor = header.data_offset + +for each segment: + if storing raw: + if !in_raw_run: + cursor = align_up(cursor, 64 KiB) + entry.file_offset = cursor + write raw_size bytes + cursor += is_final_segment + ? align_up(raw_size, 64 KiB) + : segment_size + in_raw_run = true + else: + in_raw_run = false + cursor = align_up(cursor, 8) + entry.file_offset = cursor + write stored_size compressed bytes + cursor += stored_size +``` + +The final file length is the final cursor, including raw-run padding. Do not rely on a sparse range +to provide the padding. + +## 7. Optional sidecar directory block + +When any sidecar is present, `header.reserved` points to one tightly packed directory block. The +block has no separate header. Its record order is determined entirely by `header.flags`: + +```text +LNK_CObjTypeIndex[type_count] if TYPE_INDEX +LNK_CObjUdtHashIndex[type_count] if UDT_HASH_INDEX +LNK_CObjBaseRelocIndex if BASE_RELOC_INDEX +LNK_CObjDebugSIndex if DEBUG_S_INDEX +``` + +There is no padding between these directory record arrays. Align the end of the complete block to +8 bytes before writing sidecar payload arrays. Sidecar payload offsets are absolute and each +payload should begin at an 8-byte-aligned file offset. + +## 8. Type leaf sidecar + +Emit one `LNK_CObjTypeIndex` for each `.debug$T` or `.debug$P` COFF section that has at least the +four-byte CodeView signature. Preserve COFF section order. + +The record is 40 bytes: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `raw_section_offset` | Original OBJ offset of the first byte after the 4-byte CodeView signature | +| 8 | `U32` | `raw_section_size` | Section raw size minus 4 | +| 12 | `U32` | `leaf_count` | Number of valid leaf records indexed | +| 16 | `U64` | `offsets_file_offset` | Offset array or packed-offset block | +| 24 | `U64` | `sizes_file_offset` | Size array, or packed kind dictionary | +| 32 | `U64` | `kinds_file_offset` | Kind array, or packed kind-code array | + +For a section whose `PointerToRawData` is `foff` and `SizeOfRawData` is `fsize`: + +```text +raw_section_offset = foff + 4 +raw_section_size = fsize - 4 +``` + +Starting at `raw_section_offset`, parse leaves as follows: + +```text +cursor = 0 +while cursor + 4 <= raw_section_size: + size = read_u16(cursor + 0) + kind = read_u16(cursor + 2) + stride = size + 2 + if size < 2 or stride > raw_section_size - cursor: + stop + append offset=cursor, size=size, kind=kind + cursor += stride +``` + +Leaf offsets are relative to the first byte after the CodeView signature, not relative to the OBJ +or compressed container. The stored 16-bit `size` is the CodeView record size field: it excludes +the size field itself and includes the kind and payload. + +### 8.1 Normal type representation + +When the packed flags are clear, write three separate arrays: + +```text +offsets_file_offset -> U32 offsets[leaf_count] +sizes_file_offset -> U16 sizes[leaf_count] +kinds_file_offset -> U16 kinds[leaf_count] +``` + +### 8.2 Packed type representation, version 2 + +The tested profile uses the packed representation for every type section if every section has at +most 256 distinct leaf kinds. If any type section exceeds that limit, clear both packed flags and +use the normal representation for all type sections in that object. + +Packed offsets use groups of 512 leaves: + +```text +group_size = 512 +group_count = ceil(leaf_count / 512) +``` + +At `offsets_file_offset`, write `group_count` pairs of little-endian `U32` values: + +```text +struct Group { + U32 absolute_base; + U32 payload_offset_and_width; +}; +``` + +The group directory is followed by padding to an eight-byte boundary relative to +`offsets_file_offset`, then the delta payload: + +```text +group_bytes = group_count * 8 +payload_start = offsets_file_offset + align_up(group_bytes, 8) +``` + +For each group: + +- `absolute_base` is the first leaf offset in that group. +- Select a two-byte delta if the last offset minus the base is at most `0xffff`; otherwise select a + three-byte delta. Offsets are monotonically increasing, so checking the last delta is sufficient. +- `payload_offset_and_width & ~1` is the byte offset from `payload_start` to this group's deltas. +- Bit zero is 0 for two-byte deltas and 1 for three-byte deltas. +- Write one little-endian two- or three-byte delta per leaf, including a zero delta for the first + leaf. +- A three-byte delta must not exceed `0xffffff`. + +The second and third packed arrays repurpose the legacy field names: + +```text +sizes_file_offset -> U16 kind_dictionary[256] // exactly 512 bytes +kinds_file_offset -> U8 kind_code[leaf_count] +``` + +Assign dictionary codes in first-seen leaf order. Zero-fill unused dictionary entries. A leaf's +kind is `kind_dictionary[kind_code[i]]`. Leaf sizes are derived from adjacent offsets; the final +leaf ends at `raw_section_size`. + +## 9. Complete-UDT hash sidecar + +When `UDT_HASH_INDEX` is set, write exactly `type_count` consecutive 16-byte directory records +immediately after all `LNK_CObjTypeIndex` records: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `hashes_file_offset` | Absolute offset of `U64` hashes, or zero when count is zero | +| 8 | `U32` | `hash_count` | Number of hashes | +| 12 | `U32` | `reserved` | Zero | + +Each record corresponds by index to the type-index record at the same index. Hashes are appended in +leaf order, but only for complete UDT definitions with nonempty unique names. + +The reference algorithm operates on the leaf payload after the two-byte kind field: + +| Leaf kind | Minimum payload bytes | Properties field | Cursor after fixed header | +|---:|---:|---|---:| +| `0x1504`, `0x1505`, `0x1519` | 16 | `U16` at `payload + 2` | 16 | +| `0x1506` | 8 | `U16` at `payload + 2` | 8 | +| `0x1507` | 12 | `U16` at `payload + 2` | 12 | +| `0x1608`, `0x1609` | 20 | `U32` at `payload + 0` | 20 | + +Ignore other kinds. Require property bit `0x0200` and reject property bit `0x0080`. For every kind +except `0x1507`, skip one CodeView numeric value at the cursor. Then skip the first NUL-terminated +name and hash the following nonempty NUL-terminated unique name. + +CodeView numeric length is: + +- two bytes when the first `U16` is below `0x8000`; +- otherwise two bytes for the numeric tag plus payload sizes: `0x8000:1`, + `0x8001/0x8002/0x801c:2`, `0x8003/0x8004/0x8005:4`, + `0x8006/0x8009/0x800a/0x800c:8`, `0x8007:10`, `0x8008:16`, `0x800b:6`, + `0x800d:16`, `0x800e:20`, `0x800f:32`, and `0x8017/0x8018:16`. + +Hash the unique-name bytes with wrapping 64-bit arithmetic: + +```text +h = 5381 +for byte in unique_name: + h = ((h << 5) + h) XOR byte +h = h OR 1 +``` + +The `OR 1` is mandatory because zero is the linker's empty-set sentinel. + +## 10. AMD64 base-relocation sidecar + +This optional sidecar is supported for AMD64 COFF machine `0x8664`. Omit the flag on unsupported +machines. + +The 16-byte directory is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `entries_file_offset` | Absolute entry-array offset, or zero when count is zero | +| 8 | `U32` | `entry_count` | Number of candidate relocations | +| 12 | `U32` | `reserved` | Zero | + +Each 16-byte entry is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U32` | `sect_idx` | Zero-based COFF section index | +| 4 | `U32` | `apply_off` | Relocation's section-relative application offset | +| 8 | `U32` | `isymbol` | COFF symbol-table index | +| 12 | `U8` | `addr_size` | 8 for `IMAGE_REL_AMD64_ADDR64`, 4 for `IMAGE_REL_AMD64_ADDR32` | +| 13 | `U8[3]` | `reserved` | Zero | + +Scan sections in COFF order and relocations in table order. Retain only AMD64 relocation type 1 +(`ADDR64`) and type 2 (`ADDR32`). RAD Link still applies symbol interpretation and section +liveness checks after reading this candidate list. + +Support `IMAGE_SCN_LNK_NRELOC_OVFL` (`0x01000000`) in the usual COFF way. When the section header's +relocation count is `0xffff`, the first relocation is the overflow counter; require its +`VirtualAddress`/`apply_off` to be nonzero, use `apply_off - 1` real entries, and skip the counter. + +## 11. `.debug$S` sidecar + +The 16-byte directory is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `entries_file_offset` | Absolute entry-array offset, or zero when count is zero | +| 8 | `U32` | `entry_count` | Number of retained C13 subsections | +| 12 | `U32` | `reserved` | Zero | + +Each 16-byte entry is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U32` | `raw_section_offset` | Original OBJ offset of the `.debug$S` section, including its signature | +| 4 | `U32` | `raw_payload_offset` | Original OBJ offset of the subsection payload | +| 8 | `U32` | `raw_payload_size` | Payload length, clamped to bytes remaining in the section | +| 12 | `U32` | `kind` | C13 subsection kind | + +Scan `.debug$S` sections in COFF section order. Skip the four-byte CodeView signature and parse +subsections as: + +```text +cursor = 0 +while cursor + 8 <= bytes_after_signature: + kind = read_u32(cursor + 0) + payload_size = read_u32(cursor + 4) + payload_rel = cursor + 8 + clamped_size = min(payload_size, bytes_after_signature - payload_rel) + + if (kind & 0x80000000) == 0: + append entry for this payload + + cursor = align_up(payload_rel + payload_size, 4) + stop on arithmetic overflow or failure to advance +``` + +Unknown kinds with bit 31 clear are retained. Kinds with bit 31 set are ignored. Entries must be +grouped by and nondecreasing in `raw_section_offset`, which naturally follows from scanning COFF +sections in order. + +### 11.1 `.debug$S` summary array + +When `DEBUG_S_SUMMARY` is set, one 16-byte summary exists for every debug-S entry. It begins at: + +```text +summary_offset = align_up(entries_file_offset + entry_count * 16, 8) +``` + +There is no summary offset in the directory; the reader derives it using this formula. + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U32` | `module_symbol_size` | Four-byte-aligned size of symbols retained in the PDB module stream | +| 4 | `U32` | `gsi_candidate_count` | Number of candidate GSI records | +| 8 | `U32` | `proc_ref_count` | Number of procedure-reference candidates | +| 12 | `U32` | `flags` | Bit 0 means the subsection contains local data | + +Write an all-zero summary for subsection kinds other than `0xF1` (symbols). For an `0xF1` payload, +walk byte-packed symbol records. Each record starts with `U16 record_size, U16 kind`; its input +stride is `record_size + 2`, while its contribution to `module_symbol_size` is that stride aligned +up to four bytes. Stop at a record smaller than four bytes or one that overruns the payload. + +Use these kind sets: + +```text +global = {1107,0102,0202,1008,110d,020e,100f,1113} +typedef = {0004,1003,1108} +scope = {1110,110f,1103,1102,114d,115d,1104,1132,1147,1146} +end = {0006,114f,114e} +procedure-reference = {1110,110f,1147,1146} +has-locals = {113e,1111,110c,110d,1112,1113,1153,1107} +``` + +For `gsi_candidate_count`, begin each symbol subsection with candidate scope depth zero and active +state true. Count a global kind, or a typedef at candidate depth zero, while active. Scope kinds +increment candidate depth. An end kind decrements a positive depth; an end at depth zero makes the +candidate state inactive for the rest of that subsection. + +Maintain a separate module scope depth across all symbol subsections in the OBJ. A symbol belongs +to the module stream unless it is kind `0x0007`, a global kind, a typedef at module depth zero, or +kind `0x1176`. For a retained symbol, update module depth for scope/end kinds and add +`align_up(record_size + 2, 4)` to `module_symbol_size`. + +Set summary flag bit `0x1` if any symbol kind is in the `has-locals` set. + +## 12. Canonical layout algorithm + +The following order matches the reference writer: + +```text +cursor = align_up(64, 8) + +header.directory_offset = cursor +cursor = align_up(cursor + segment_count * 24, 8) + +if any sidecar directory is present: + header.reserved = cursor + cursor += type_count * 40 if TYPE_INDEX + cursor += type_count * 16 if UDT_HASH_INDEX + cursor += 16 if BASE_RELOC_INDEX + cursor += 16 if DEBUG_S_INDEX + cursor = align_up(cursor, 8) +else: + header.reserved = 0 + +for each type index: + assign offsets block; align cursor to 8 + assign sizes/dictionary block; align cursor to 8 + assign kinds/codes block; align cursor to 8 + assign UDT hashes block if nonempty; align cursor to 8 + +assign base-relocation entries if nonempty; align cursor to 8 +assign debug-S entries if nonempty; align cursor to 8 +assign debug-S summaries; align cursor to 8 + +header.data_offset = cursor +write segment payloads using the rules in section 6 +rewrite the completed segment directory +truncate the file to the final payload cursor +``` + +Use checked 64-bit addition and multiplication at every step. Reject any value that cannot be +represented by its destination field. + +## 13. Reader acceptance rules + +Before publishing a container, validate at least the same conditions RAD Link applies: + +### Header and directory + +- File size is at least 64 bytes. +- Magic, version, and header size are exact. +- `raw_size` is nonzero. +- `segment_size` is a power of two and at least 64 KiB. +- `segment_count == ceil(raw_size / segment_size)` without arithmetic overflow. +- The directory range is in the file. +- `PORTABLE_RAW_MAP` is set, no unknown flags are set, and all flag dependencies hold. +- Every segment has the expected raw size, a nonzero stored size, known flags, and an in-file + payload range. +- Every raw segment has `stored_size == raw_size`. +- Every raw run satisfies the 64 KiB mapping rules in section 6.2. + +### Type sidecars + +- The sidecar directory and every payload array are in the file. +- Every indexed raw section range is inside the original OBJ. +- Packed group directories and payload ranges are in bounds. +- Every packed descriptor selects width 2 or 3 through bit zero, points inside the delta payload, + and has enough bytes for its group. +- Offsets are monotonic, inside the indexed raw section, and reproduce the actual leaf boundaries. +- Packed kind codes resolve to the intended dictionary entry. + +### Other sidecars + +- Every UDT hash, base-relocation entry, debug-S entry, and summary array is in the file. +- Debug-S payload ranges are inside the original OBJ. +- Debug-S entries are nondecreasing by `raw_section_offset`. +- Sidecar contents are regenerated from the exact raw OBJ bytes and tested against a link without + sidecars. + +Finally, decode every compressed segment with the same Oodle decode options as RAD Link and compare +it byte-for-byte with the corresponding original segment. + +## 14. Publication and cache integrity + +Write to a temporary path in the destination directory. Flush and close it, complete all validation, +then atomically replace the destination. A failed conversion must never leave a file beginning with +`RLOBJ001` at the final path. + +Version 1 has no container checksum and does not enable Oodle quantum CRCs. The artifact cache must +continue to verify its normal whole-file content hash on upload and download. That hash covers the +header, sidecars, compressed payloads, raw payloads, and alignment padding. + +The artifact key must distinguish at least: + +- container magic and version; +- segment size; +- codec and compression level; +- compression options, including space/speed tradeoff; and +- sidecar schema/implementation version. + +Even when two settings decode to the same original OBJ, they are different stored artifacts. + +## 15. Bring-up and correctness test plan + +Use this order for a new writer: + +1. Emit a segment-only container with all segments compressed, avoiding raw mapping initially. +2. Decode every segment in a standalone validator and reconstruct the original OBJ byte-for-byte. +3. Link a small target from raw OBJs and compressed OBJs; compare image and PDB hashes. +4. Enable raw fallback and test files whose first, middle, and final segments are raw, including a + short final raw segment. +5. Link a response file mixing raw OBJs, compressed direct OBJ inputs, and ordinary `.lib` files. +6. Add sidecars one at a time, comparing image and PDB hashes after each addition. +7. Convert a complete large corpus with one uniform 512 KiB segment size and run warm and cold-cache + performance tests. +8. Corrupt each metadata field in validator tests and confirm rejection rather than an out-of-bounds + read or a partially accepted container. + +Compressed direct OBJ inputs can coexist with raw direct OBJ inputs and ordinary `.lib` files. +Compressed members inside a `.lib` archive are not supported by the current reader. + +The repository's reference smoke path is: + +```bat +scripts\build_cobj_test.bat +scripts\run_cobj_smoke.bat +``` + +For corpus conversion and deterministic image/PDB comparisons, see +`docs/compressed_obj_workflow.md` and the scripts under `scripts/`. diff --git a/docs/compressed_obj_workflow.md b/docs/compressed_obj_workflow.md new file mode 100644 index 000000000..6afccb046 --- /dev/null +++ b/docs/compressed_obj_workflow.md @@ -0,0 +1,200 @@ +# Portable compressed OBJ files + +## Overview + +RAD Link can optionally consume independently compressed, self-contained OBJ files. The format is +intended for targets whose raw OBJ corpus is large enough that disk traffic and file-backed pages +dominate link time and peak working set. + +Each source OBJ remains one ordinary portable file. The format does not use sparse allocation, +filesystem compression, bundles, shared manifests, or dependencies between OBJ files. Raw OBJ +files, compressed OBJ files, and ordinary `.lib` inputs can coexist in one link. + +Oodle support is opt-in. This repository does not contain or distribute Oodle headers, libraries, +or binaries. An Oodle-enabled build consumes a separately supplied SDK through `OODLE_SDK_DIR`. +The ordinary RAD Link release build has no Oodle compile-time or link-time dependency. + +## Measured results + +These are process-start-to-process-exit measurements from the final reviewed reader. File sizes are +ordinary file lengths on disk; neither corpus uses filesystem compression or sparse allocation. +Peak commit and peak working set are process peaks sampled at 50 ms intervals. + +| Target and input | Warm wall | Peak commit | Peak working set | OBJ corpus | +|---|---:|---:|---:|---:| +| FortniteClient, raw mmap | 15.36 s | 18.12 GiB | 95.50 GiB | 81.01 GiB | +| FortniteClient, compressed | 13.27 s | 15.06 GiB | 35.90 GiB | 22.97 GiB | +| UEFN DLL, raw mmap | 23.55 s | 28.45 GiB | 111.58 GiB | 89.22 GiB | +| UEFN DLL, compressed | 21.92 s | 22.27 GiB | 47.86 GiB | 24.39 GiB | + +For the cold-cache comparison, the OS file cache was purged separately before each link: + +| Target | Raw mmap | Compressed | +|---|---:|---:| +| FortniteClient | 28.51 s | 14.87 s | +| UEFN DLL | 30.56 s | 21.26 s | + +The compressed format reduced the tested corpora to roughly 28% of their original size and cut +peak working set by 57-62%. Warm results vary by roughly 0.5-1.0 seconds because PDB output is +asynchronous; a fully cached raw mmap link can occasionally be faster. The cold-cache and memory +improvements were consistent. + +Correctness was checked against raw-OBJ control links using the same output paths: + +- FortniteClient EXE SHA-256 (raw and compressed): + `1E126B7C8FAA2B730C380D9CF61131CE2E9175FAEC37A833D474952C5168D8F3` +- FortniteClient PDB SHA-256 (raw and compressed): + `39E040E711381CE48CDDE5238E7909F2083DE5DF785BF9FAAA819061AE6F56D7` +- UEFN DLL SHA-256 (raw and compressed): + `71EFFECB950BBF6E6DC6CBFFD20BA434E69388F024CAC21E083F0EA48DC7D1B5` + +## Build configurations + +Build the ordinary release without Oodle: + +```bat +build.bat radlink release +``` + +This succeeds without an Oodle SDK even if `OODLE_SDK_DIR` is unset. If a compressed OBJ is passed +to this binary it exits with a clear unsupported-input error. + +To opt into compressed OBJ support, point at an external SDK and pass the explicit build option: + +```bat +set OODLE_SDK_DIR=C:\path\to\OodleDataCompression\Sdk +build.bat radlink release oodle +``` + +The standalone reference writer also requires the external SDK: + +```bat +build.bat rad_obj_compress release +``` + +From an x64 Visual Studio developer command prompt, `scripts\build_cobj_test.bat` builds both +Oodle-enabled tools and copies them to `out_cobj`. `scripts\run_cobj_smoke.bat` compresses a small +OBJ, links both versions, and byte-compares the resulting executables. + +## Selected format + +The measured writer settings are: + +| Setting | Value | +|---|---:| +| Codec | Kraken | +| Compression level | Normal | +| Independent segment size | 512 KiB | +| Space/speed tradeoff | 256 | +| Incompressible segments | Stored raw | + +Each file contains: + +1. a fixed header describing the original COFF size and segment geometry; +2. a segment directory with stored offsets, stored sizes, raw sizes, and raw/compressed flags; +3. compact indexes for type leaves, complete UDT hashes, base relocations, and `.debug$S` data; +4. independently decodable segment payloads. + +The declarations are in `src\linker\lnk_compressed_obj_format.h`. The complete writer-facing byte +layout and sidecar contract are documented in `docs\compressed_obj_format.md`. All integers are +little-endian, and all file offsets and sidecar ranges are validated before the logical OBJ view is +published. + +The 512 KiB segment size was the measured knee for both targets. Smaller segments reduced sparse +decode amplification but added too many decoder calls; larger segments saved little disk space and +decoded too many unused bytes. + +## Reader design + +The reader preserves RAD Link's pointer-based COFF parsers while bounding private commit: + +1. Input classification checks one magic value. Ordinary OBJs stay on the existing mmap path. +2. `VirtualAlloc2` reserves a logical range matching the original OBJ using placeholders. +3. Raw-stored segments map directly from the portable file at their logical offsets. +4. A read fault on a compressed segment decodes into a bounded pagefile-backed slot and maps that + slot into the placeholder. A pinned write-copy path preserves rare in-place COFF patches. +5. Type and `.debug$S` sidecars let dominant debug passes stream compressed ranges directly into + their final destinations instead of repeatedly faulting the generic cache. +6. At the PDB boundary, the first cache generation is frozen and later misses use a smaller second + generation. This keeps published pointers stable and eliminates measured generic redecodes. + +The best measured UEFN configuration used a 17 GiB first generation and a 6 GiB second generation. +The equivalent explicit RAD Link options are: + +```bat +/RAD_COBJ_CACHE_GIB:17 +/RAD_COBJ_CACHE_SHRINK_GIB:6 +/RAD_COBJ_CACHE_FREEZE +/RAD_COBJ_TRIM_WS +/RAD_COBJ_ONE_SHOT +``` + +FortniteClient needed 27,727 first-generation segments (13.54 GiB) and ran with a 14+1 GiB +configuration. Cache capacity is not part of the file format. When these options are absent, RAD +Link derives a bounded capacity from physical memory and available commit, then caps it to the +logical size of the compressed inputs. Environment variables remain available as development +overrides; precedence is command line, environment, then the adaptive policy. + +`/RAD_COBJ_ONE_SHOT` is only appropriate for `radlink.exe` because the process is about to exit. +Leave it unset in embedded or long-lived hosts. `/RAD_LOG:TIMERS` prints the final cache, decode, +eviction, and timing statistics when diagnostics are needed; ordinary links remain silent. + +## Create and link a corpus + +Convert the direct OBJ entries in a response file while leaving switches and `.lib` paths intact: + +```powershell +& .\scripts\cobj_compress_rsp.ps1 ` + -ResponseFile C:\path\target.rsp ` + -OutputDirectory D:\scratch\compressed-objs ` + -Compressor .\out_cobj\rad_obj_compress.exe ` + -Workers 16 ` + -SegmentKiB 512 ` + -Codec kraken ` + -SpaceSpeedTradeoff 256 ` + -CompressionLevel normal +``` + +The output directory receives one independently portable file per OBJ and a rewritten +`compressed.rsp`. Use a fresh directory when changing the format or writer settings. + +Measure a link with process wall time and peak memory: + +```powershell +& .\scripts\bench_cobj.ps1 ` + -Rsp D:\scratch\compressed-objs\compressed.rsp ` + -Output D:\scratch\link\target.exe ` + -WorkingDirectory C:\path\to\link\working-directory ` + -CacheGiB 17 ` + -CacheShrinkGiB 6 ` + -Tag compressed_warm +``` + +`scripts\stage_raw_rsp.ps1`, `scripts\warm_rsp.ps1`, and `scripts\retarget_rsp.ps1` support fair +raw/compressed comparisons. `scripts\test_link_determinism.ps1` checks both image and PDB hashes +across repeated links. + +## Writer and integrity invariants + +A production writer must preserve these rules: + +1. Each output is self-contained and has ordinary non-sparse file length. +2. Segment size is a power of two and every segment is independently decodable. +3. Directory and sidecar offsets are aligned, ordered where required, and bounds-checked. +4. Type offsets are relative to the first byte after the CodeView signature. +5. The `.debug$S` index follows the reader's accepted-subsection rules exactly. +6. Incompressible segments may be stored raw without changing logical segment geometry. +7. Write a temporary file, validate every compressed round trip, and atomically rename it. + +The reference writer decodes and byte-compares every compressed segment before publishing. The +container does not currently enable Oodle quantum CRCs, and raw-stored segments bypass the decoder. +Production caches should retain their normal whole-file content hash verification when uploading +and downloading these objects; that covers metadata, compressed payloads, and raw payloads without +adding work to the linker's hot path. + +## Current limitations + +- The lazy placeholder reader is Windows-only. +- Oodle must be supplied and licensed separately; it is not part of this repository. +- Direct compressed OBJ inputs are supported. Compressed members inside `.lib` archives are not. +- The cache is process-global and tuned for the one-link command-line executable. diff --git a/scripts/bench_cobj.ps1 b/scripts/bench_cobj.ps1 new file mode 100644 index 000000000..3a0c4b9b1 --- /dev/null +++ b/scripts/bench_cobj.ps1 @@ -0,0 +1,78 @@ +param( + [Parameter(Mandatory=$true)][string]$Rsp, + [Parameter(Mandatory=$true)][string]$Output, + [string]$Radlink = '', + [string]$WorkingDirectory = '', + [string]$LogDirectory = '', + [int]$CacheGiB = 17, + [int]$CacheShrinkGiB = 6, + [switch]$CacheFreeze = $true, + [ValidateSet('','types','except_debug_s','except_debug_s_runs','all')][string]$CacheFreezeTrim = '', + [int]$SkipCleanup = 1, + [string]$Tag = 'v2', + [string]$ExtraArgs = '', + [switch]$TrimWorkingSet = $true, + [switch]$TrimProcessWorkingSet, + [switch]$TraceMemory +) +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +if (!$Radlink) { $Radlink = Join-Path $root 'out_cobj\radlink.exe' } +if (!$WorkingDirectory) { $WorkingDirectory = (Get-Location).Path } +if (!$LogDirectory) { $LogDirectory = Join-Path $root 'cobj_bench_output' } +$Radlink = [IO.Path]::GetFullPath($Radlink) +$WorkingDirectory = [IO.Path]::GetFullPath($WorkingDirectory) +$LogDirectory = [IO.Path]::GetFullPath($LogDirectory) +[IO.Directory]::CreateDirectory($LogDirectory) | Out-Null +$log = Join-Path $LogDirectory "cobj_$Tag.log" +$phase = Join-Path $LogDirectory "cobj_$Tag.phase.log" +Remove-Item -LiteralPath $log,$phase -Force -ErrorAction SilentlyContinue +$psi = [Diagnostics.ProcessStartInfo]::new() +$psi.FileName = $Radlink +$psi.WorkingDirectory = $WorkingDirectory +$psi.Arguments = '@"' + [IO.Path]::GetFullPath($Rsp) + '" /RAD_LOG:timers ' + $ExtraArgs +$psi.UseShellExecute = $false +$psi.CreateNoWindow = $true +$psi.RedirectStandardOutput = $true +$psi.RedirectStandardError = $true +$psi.Environment['RAD_COBJ_CACHE_GIB'] = [string]$CacheGiB +$psi.Environment['RAD_COBJ_SKIP_CLEANUP'] = [string]$SkipCleanup +if ($CacheShrinkGiB -gt 0) { $psi.Environment['RAD_COBJ_CACHE_SHRINK_GIB'] = [string]$CacheShrinkGiB } +else { [void]$psi.Environment.Remove('RAD_COBJ_CACHE_SHRINK_GIB') } +if ($CacheFreeze) { $psi.Environment['RAD_COBJ_CACHE_FREEZE'] = '1' } +else { [void]$psi.Environment.Remove('RAD_COBJ_CACHE_FREEZE') } +if ($CacheFreezeTrim) { + $psi.Environment['RAD_COBJ_CACHE_FREEZE_TRIM'] = if ($CacheFreezeTrim -eq 'all') { '1' } else { $CacheFreezeTrim } +} else { [void]$psi.Environment.Remove('RAD_COBJ_CACHE_FREEZE_TRIM') } +if ($TrimWorkingSet) { $psi.Environment['RAD_COBJ_TRIM_WS'] = '1' } +if ($TrimProcessWorkingSet) { $psi.Environment['RAD_COBJ_TRIM_WS'] = '2' } +$psi.Environment['RADLINK_PHASE_LOG'] = $phase +$p = [Diagnostics.Process]::new(); $p.StartInfo = $psi +$sw = [Diagnostics.Stopwatch]::StartNew(); [void]$p.Start() +$stdoutTask = $p.StandardOutput.ReadToEndAsync() +$stderrTask = $p.StandardError.ReadToEndAsync() +$peakPrivate = 0L; $peakCommit = 0L; $peakWs = 0L +$memoryTrace = [Collections.Generic.List[string]]::new() +if ($TraceMemory) { $memoryTrace.Add('ElapsedSeconds,PrivateGiB,WorkingSetGiB') } +while (!$p.HasExited) { + try { + $p.Refresh() + $private = $p.PrivateMemorySize64 + $commit = $p.PeakPagedMemorySize64 + $ws = $p.WorkingSet64 + $peakPrivate=[Math]::Max($peakPrivate,$private) + $peakCommit=[Math]::Max($peakCommit,$commit) + $peakWs=[Math]::Max($peakWs,$ws) + if ($TraceMemory) { + $memoryTrace.Add(('{0:F3},{1:F6},{2:F6}' -f $sw.Elapsed.TotalSeconds,($private/1GB),($ws/1GB))) + } + } catch {} + Start-Sleep -Milliseconds 50 +} +$stdout=$stdoutTask.GetAwaiter().GetResult(); $stderr=$stderrTask.GetAwaiter().GetResult(); $sw.Stop() +[IO.File]::WriteAllText($log, $stdout + $stderr) +if ($TraceMemory) { [IO.File]::WriteAllLines((Join-Path $LogDirectory "cobj_$Tag.memory.csv"), $memoryTrace) } +$p.Refresh(); $peakPrivate=[Math]::Max($peakPrivate,$p.PrivateMemorySize64); $peakCommit=[Math]::Max($peakCommit,$p.PeakPagedMemorySize64); $peakWs=[Math]::Max($peakWs,$p.WorkingSet64) +$hash = if ($p.ExitCode -eq 0 -and (Test-Path -LiteralPath $Output)) { (Get-FileHash -LiteralPath $Output -Algorithm SHA256).Hash } else { '' } +[pscustomobject]@{Tag=$Tag;ExitCode=$p.ExitCode;WallSeconds=$sw.Elapsed.TotalSeconds;PeakCommitGiB=$peakCommit/1GB;PeakPrivateGiB=$peakPrivate/1GB;PeakWorkingSetGiB=$peakWs/1GB;UserSeconds=$p.UserProcessorTime.TotalSeconds;KernelSeconds=$p.PrivilegedProcessorTime.TotalSeconds;Sha256=$hash;Log=$log;PhaseLog=$phase} | ConvertTo-Json -Compress +Get-Content -LiteralPath $log -Tail 30 diff --git a/scripts/build_cobj_test.bat b/scripts/build_cobj_test.bat new file mode 100644 index 000000000..33275495b --- /dev/null +++ b/scripts/build_cobj_test.bat @@ -0,0 +1,19 @@ +@echo off +setlocal +if "%OODLE_SDK_DIR%"=="" ( + echo OODLE_SDK_DIR must name an Oodle SDK + exit /b 1 +) +if not exist "%OODLE_SDK_DIR%\include\oodle2.h" ( + echo Oodle SDK not found at "%OODLE_SDK_DIR%" + exit /b 1 +) +where cl >nul 2>nul || ( + echo Run this script from an x64 Visual Studio developer command prompt + exit /b 1 +) +call "%~dp0..\build.bat" radlink rad_obj_compress release oodle || exit /b 1 +if not exist "%~dp0..\out_cobj" mkdir "%~dp0..\out_cobj" +copy /y "%~dp0..\build\radlink.exe" "%~dp0..\out_cobj\radlink.exe" >nul || exit /b 1 +copy /y "%~dp0..\build\radlink.pdb" "%~dp0..\out_cobj\radlink.pdb" >nul || exit /b 1 +copy /y "%~dp0..\build\rad_obj_compress.exe" "%~dp0..\out_cobj\rad_obj_compress.exe" >nul || exit /b 1 diff --git a/scripts/cobj_compress_rsp.ps1 b/scripts/cobj_compress_rsp.ps1 new file mode 100644 index 000000000..17efb37e4 --- /dev/null +++ b/scripts/cobj_compress_rsp.ps1 @@ -0,0 +1,107 @@ +param( + [Parameter(Mandatory=$true)][string]$ResponseFile, + [Parameter(Mandatory=$true)][string]$OutputDirectory, + [Parameter(Mandatory=$true)][string]$Compressor, + [int]$Workers = 16, + [int]$SegmentKiB = 512, + [ValidateSet('selkie','mermaid','kraken')][string]$Codec = 'kraken', + [int]$SpaceSpeedTradeoff = 256, + [ValidateSet('superfast','veryfast','fast','normal','optimal1','optimal2','optimal3','optimal4','optimal5')][string]$CompressionLevel = 'normal' +) + +$ErrorActionPreference = 'Stop' +Add-Type -TypeDefinition @' +using System; +using System.Runtime.InteropServices; +public static class CObjFileSize { + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] + public static extern uint GetCompressedFileSizeW(string path, out uint high); +} +'@ +$ResponseFile = [IO.Path]::GetFullPath($ResponseFile) +$OutputDirectory = [IO.Path]::GetFullPath($OutputDirectory) +$Compressor = [IO.Path]::GetFullPath($Compressor) +[IO.Directory]::CreateDirectory($OutputDirectory) | Out-Null + +$lines = [IO.File]::ReadAllLines($ResponseFile) +$work = New-Object System.Collections.Generic.List[object] +$rewritten = New-Object string[] $lines.Length +$objIndex = 0 +for ($lineIndex = 0; $lineIndex -lt $lines.Length; ++$lineIndex) { + $line = $lines[$lineIndex] + $path = $line.Trim().Trim('"') + if ($path -match '(?i)\.obj$' -and [IO.File]::Exists($path)) { + $leaf = [IO.Path]::GetFileName($path) + $dest = Join-Path $OutputDirectory ('{0:D5}_{1}' -f $objIndex, $leaf) + $work.Add([pscustomobject]@{ Source=$path; Destination=$dest; Index=$objIndex }) + $rewritten[$lineIndex] = '"' + ($dest -replace '\\','/') + '"' + ++$objIndex + } else { + $rewritten[$lineIndex] = $line + } +} +[IO.File]::WriteAllLines((Join-Path $OutputDirectory 'compressed.rsp'), $rewritten) + +$running = New-Object System.Collections.Generic.List[object] +$completed = 0 +$failed = 0 +$timer = [Diagnostics.Stopwatch]::StartNew() + +function Reap-Compressors([bool]$waitForOne) { + do { + $reaped = $false + for ($i = $running.Count - 1; $i -ge 0; --$i) { + $job = $running[$i] + if ($job.Process.HasExited) { + $stdout = $job.Process.StandardOutput.ReadToEnd().Trim() + $stderr = $job.Process.StandardError.ReadToEnd().Trim() + if ($job.Process.ExitCode -ne 0) { + ++$script:failed + Write-Error "compression failed ($($job.Process.ExitCode)): $($job.Source)`n$stderr" + } + $job.Process.Dispose() + $running.RemoveAt($i) + ++$script:completed + $reaped = $true + if (($script:completed % 100) -eq 0 -or $script:completed -eq $work.Count) { + Write-Host ("compressed {0}/{1} objects in {2:n1}s" -f $script:completed,$work.Count,$timer.Elapsed.TotalSeconds) + } + } + } + if ($waitForOne -and !$reaped) { Start-Sleep -Milliseconds 25 } + } while ($waitForOne -and !$reaped) +} + +foreach ($item in $work) { + if ([IO.File]::Exists($item.Destination)) { + ++$completed + continue + } + while ($running.Count -ge $Workers) { Reap-Compressors $true } + $psi = New-Object Diagnostics.ProcessStartInfo + $psi.FileName = $Compressor + $psi.Arguments = ('"{0}" "{1}" {2} {3} {4} {5}' -f $item.Source,$item.Destination,$SegmentKiB,$Codec,$SpaceSpeedTradeoff,$CompressionLevel) + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $process = [Diagnostics.Process]::Start($psi) + $running.Add([pscustomobject]@{ Process=$process; Source=$item.Source }) +} +while ($running.Count) { Reap-Compressors $true } + +if ($failed) { throw "$failed compression jobs failed" } +$rawBytes = ($work | ForEach-Object { [IO.FileInfo]$_.Source } | Measure-Object Length -Sum).Sum +$logicalBytes = ($work | ForEach-Object { [IO.FileInfo]$_.Destination } | Measure-Object Length -Sum).Sum +$allocatedBytes = 0L +foreach ($item in $work) { + [uint32]$high = 0 + [uint32]$low = [CObjFileSize]::GetCompressedFileSizeW($item.Destination, [ref]$high) + if ($low -eq [uint32]::MaxValue -and [Runtime.InteropServices.Marshal]::GetLastWin32Error() -ne 0) { + throw "unable to query allocated size: $($item.Destination)" + } + $allocatedBytes += ([int64]$high -shl 32) -bor $low +} +Write-Host ("done: {0} independent objects, raw {1:n2} GiB, container logical {2:n2} GiB, allocated {3:n2} GiB ({4:n1}% of raw), {5:n1}s" -f + $work.Count,($rawBytes/1GB),($logicalBytes/1GB),($allocatedBytes/1GB), + (100.0*$allocatedBytes/$rawBytes),$timer.Elapsed.TotalSeconds) diff --git a/scripts/cobj_smoke.c b/scripts/cobj_smoke.c new file mode 100644 index 000000000..77bc677e8 --- /dev/null +++ b/scripts/cobj_smoke.c @@ -0,0 +1,5 @@ +int +main(void) +{ + return 0; +} diff --git a/scripts/retarget_rsp.ps1 b/scripts/retarget_rsp.ps1 new file mode 100644 index 000000000..890b44cfd --- /dev/null +++ b/scripts/retarget_rsp.ps1 @@ -0,0 +1,45 @@ +param( + [Parameter(Mandatory=$true)][string]$ResponseFile, + [Parameter(Mandatory=$true)][string]$Destination, + [Parameter(Mandatory=$true)][string]$Output, + [Parameter(Mandatory=$true)][string]$Pdb +) + +$ErrorActionPreference = 'Stop' +$ResponseFile = [IO.Path]::GetFullPath($ResponseFile) +$Destination = [IO.Path]::GetFullPath($Destination) +$Output = [IO.Path]::GetFullPath($Output) +$Pdb = [IO.Path]::GetFullPath($Pdb) + +foreach ($directory in @([IO.Path]::GetDirectoryName($Destination), + [IO.Path]::GetDirectoryName($Output), + [IO.Path]::GetDirectoryName($Pdb))) { + [IO.Directory]::CreateDirectory($directory) | Out-Null +} + +$outputArg = '/OUT:"' + ($Output -replace '\\','/') + '"' +$pdbArg = '/PDB:"' + ($Pdb -replace '\\','/') + '"' +$foundOutput = $false +$foundPdb = $false +$rewritten = foreach ($line in [IO.File]::ReadAllLines($ResponseFile)) { + if ($line -match '(?i)^\s*/OUT:') { + $foundOutput = $true + $outputArg + } elseif ($line -match '(?i)^\s*/PDB:') { + $foundPdb = $true + $pdbArg + } else { + $line + } +} + +if (!$foundOutput) { throw "Response file has no /OUT argument: $ResponseFile" } +if (!$foundPdb) { throw "Response file has no /PDB argument: $ResponseFile" } +[IO.File]::WriteAllLines($Destination, $rewritten) + +[pscustomobject]@{ + Source = $ResponseFile + Destination = $Destination + Output = $Output + Pdb = $Pdb +} | ConvertTo-Json -Compress diff --git a/scripts/run_cobj_smoke.bat b/scripts/run_cobj_smoke.bat new file mode 100644 index 000000000..a7830e304 --- /dev/null +++ b/scripts/run_cobj_smoke.bat @@ -0,0 +1,15 @@ +@echo off +setlocal +where cl >nul 2>nul || ( + echo Run this script from an x64 Visual Studio developer command prompt + exit /b 1 +) +pushd "%~dp0..\build" +cl /nologo /c /Z7 /Od "%~dp0cobj_smoke.c" /Focobj_smoke_raw.obj || exit /b 1 +rad_obj_compress.exe cobj_smoke_raw.obj cobj_smoke_compressed.obj 512 selkie || exit /b 1 +radlink.exe /nologo /debug:ghash /pdb:cobj_smoke.pdb /entry:main /subsystem:console /out:cobj_smoke_raw.exe cobj_smoke_raw.obj || exit /b 1 +set RAD_COBJ_CACHE_MIB=16 +set RAD_COBJ_SKIP_CLEANUP=1 +radlink.exe /nologo /debug:ghash /pdb:cobj_smoke.pdb /entry:main /subsystem:console /out:cobj_smoke_compressed.exe cobj_smoke_compressed.obj || exit /b 1 +fc /b cobj_smoke_raw.exe cobj_smoke_compressed.exe || exit /b 1 +popd diff --git a/scripts/stage_raw_rsp.ps1 b/scripts/stage_raw_rsp.ps1 new file mode 100644 index 000000000..f25672650 --- /dev/null +++ b/scripts/stage_raw_rsp.ps1 @@ -0,0 +1,46 @@ +param( + [Parameter(Mandatory=$true)][string]$ResponseFile, + [Parameter(Mandatory=$true)][string]$OutputDirectory, + [int]$Workers = 16 +) + +$ErrorActionPreference = 'Stop' +$ResponseFile = [IO.Path]::GetFullPath($ResponseFile) +$OutputDirectory = [IO.Path]::GetFullPath($OutputDirectory) +[IO.Directory]::CreateDirectory($OutputDirectory) | Out-Null + +$lines = [IO.File]::ReadAllLines($ResponseFile) +$rewritten = New-Object string[] $lines.Length +$sources = [Collections.Generic.List[string]]::new() +$destinations = [Collections.Generic.List[string]]::new() +for ($lineIndex = 0; $lineIndex -lt $lines.Length; ++$lineIndex) { + $path = $lines[$lineIndex].Trim().Trim('"') + if ($path -match '(?i)\.obj$' -and [IO.File]::Exists($path)) { + $dest = Join-Path $OutputDirectory ('{0:D5}_{1}' -f $sources.Count,[IO.Path]::GetFileName($path)) + $sources.Add($path) + $destinations.Add($dest) + $rewritten[$lineIndex] = '"' + ($dest -replace '\\','/') + '"' + } else { + $rewritten[$lineIndex] = $lines[$lineIndex] + } +} + +Add-Type -TypeDefinition @' +using System; +using System.IO; +using System.Threading.Tasks; +public static class RawObjStager { + public static void Copy(string[] sources, string[] destinations, int workers) { + Parallel.For(0, sources.Length, new ParallelOptions { MaxDegreeOfParallelism = workers }, i => { + File.Copy(sources[i], destinations[i], true); + }); + } +} +'@ + +$timer = [Diagnostics.Stopwatch]::StartNew() +[RawObjStager]::Copy($sources.ToArray(), $destinations.ToArray(), $Workers) +[IO.File]::WriteAllLines((Join-Path $OutputDirectory 'raw.rsp'), $rewritten) +$bytes = ($destinations | ForEach-Object { [IO.FileInfo]$_ } | Measure-Object Length -Sum).Sum +Write-Host ("staged {0} independent raw objects, {1:n2} GiB in {2:n1}s" -f + $sources.Count,($bytes/1GB),$timer.Elapsed.TotalSeconds) diff --git a/scripts/test_link_determinism.ps1 b/scripts/test_link_determinism.ps1 new file mode 100644 index 000000000..a501ca71d --- /dev/null +++ b/scripts/test_link_determinism.ps1 @@ -0,0 +1,71 @@ +param( + [Parameter(Mandatory=$true)][string]$Rsp, + [Parameter(Mandatory=$true)][string]$Output, + [Parameter(Mandatory=$true)][string]$Pdb, + [int]$Iterations = 5, + [string]$ExtraArgs = '', + [string]$WorkingDirectory = '', + [string]$Radlink = '' +) + +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +if (!$Radlink) { $Radlink = Join-Path $root 'out_cobj\radlink.exe' } +if (!$WorkingDirectory) { $WorkingDirectory = (Get-Location).Path } +$Radlink = [IO.Path]::GetFullPath($Radlink) +$Rsp = [IO.Path]::GetFullPath($Rsp) +$Output = [IO.Path]::GetFullPath($Output) +$Pdb = [IO.Path]::GetFullPath($Pdb) + +$expectedOutputHash = $null +$expectedPdbHash = $null +$results = [Collections.Generic.List[object]]::new() + +for ($iteration = 1; $iteration -le $Iterations; ++$iteration) { + # Never let a stale artifact make a successful-but-nonproducing invocation look valid. + [IO.File]::Delete($Output) + [IO.File]::Delete($Pdb) + + $psi = [Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $Radlink + $psi.WorkingDirectory = $WorkingDirectory + $psi.Arguments = '@"' + $Rsp + '" ' + $ExtraArgs + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + + $process = [Diagnostics.Process]::new() + $process.StartInfo = $psi + $timer = [Diagnostics.Stopwatch]::StartNew() + [void]$process.Start() + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + $process.WaitForExit() + $timer.Stop() + $combined = $stdout.GetAwaiter().GetResult() + $stderr.GetAwaiter().GetResult() + if ($process.ExitCode -ne 0) { + throw "link iteration $iteration failed with exit code $($process.ExitCode):`n$combined" + } + if (!(Test-Path -LiteralPath $Output) -or !(Test-Path -LiteralPath $Pdb)) { + throw "link iteration $iteration did not produce both requested outputs" + } + + $outputHash = (Get-FileHash -LiteralPath $Output -Algorithm SHA256).Hash + $pdbHash = (Get-FileHash -LiteralPath $Pdb -Algorithm SHA256).Hash + if ($iteration -eq 1) { + $expectedOutputHash = $outputHash + $expectedPdbHash = $pdbHash + } elseif ($outputHash -ne $expectedOutputHash -or $pdbHash -ne $expectedPdbHash) { + throw "non-deterministic output at iteration $iteration (image $outputHash, PDB $pdbHash)" + } + + $results.Add([pscustomobject]@{ + Iteration = $iteration + WallSeconds = $timer.Elapsed.TotalSeconds + ImageSha256 = $outputHash + PdbSha256 = $pdbHash + }) +} + +$results | ConvertTo-Json diff --git a/scripts/warm_rsp.ps1 b/scripts/warm_rsp.ps1 new file mode 100644 index 000000000..7fed9df49 --- /dev/null +++ b/scripts/warm_rsp.ps1 @@ -0,0 +1,31 @@ +param([Parameter(Mandatory=$true)][string]$Rsp) + +$ErrorActionPreference = 'Stop' +$Rsp = [IO.Path]::GetFullPath($Rsp) +$Buffer = New-Object byte[] (16MB) +$Files = [Collections.Generic.List[string]]::new() + +foreach ($line in [IO.File]::ReadLines($Rsp)) { + $path = $line.Trim().Trim('"') + if ($path -notmatch '(?i)\.obj$' -or -not [IO.File]::Exists($path)) { continue } + $Files.Add($path) +} + +$total = 0L +$timer = [Diagnostics.Stopwatch]::StartNew() +foreach ($path in $Files) { + $fs = [IO.FileStream]::new($path, [IO.FileMode]::Open, [IO.FileAccess]::Read, + [IO.FileShare]::ReadWrite -bor [IO.FileShare]::Delete, + 16MB, [IO.FileOptions]::SequentialScan) + try { + while (($got = $fs.Read($Buffer, 0, $Buffer.Length)) -gt 0) { $total += $got } + } finally { $fs.Dispose() } +} +$timer.Stop() +[pscustomobject]@{ + Rsp = $Rsp + Files = $Files.Count + LogicalGiBRead = $total / 1GB + Seconds = $timer.Elapsed.TotalSeconds + GiBPerSecond = if ($timer.Elapsed.TotalSeconds) { ($total / 1GB) / $timer.Elapsed.TotalSeconds } else { 0 } +} | ConvertTo-Json -Compress diff --git a/src/base/base_arena.c b/src/base/base_arena.c index d4376c105..2753f603a 100644 --- a/src/base/base_arena.c +++ b/src/base/base_arena.c @@ -353,6 +353,54 @@ arena_pop_to(Arena *arena, U64 pos) } +//- rjf: arena decommit of unused (rewound/free) pages + +internal void +arena_decommit_unused(Arena *arena) +{ + // NOTE(perf): decommit committed-but-unused pages so they stop counting against + // working set, while keeping the reservation. Only touches pages strictly above + // the live `pos` high-water of each block in the active chain, and the unused + // bodies of free-list blocks. Live data (<= pos) is never touched. The push path + // re-commits on demand (arena_push grows `cmt`), so reuse is transparent. + if(arena->flags & ArenaFlag_LargePages) + { + // large pages cannot be partially decommitted safely; skip. + return; + } + U64 page_size = get_system_info()->page_size; + + // rjf: active chain -- decommit committed region above each block's live pos + for(Arena *n = arena->current; n != 0; n = n->prev) + { + U64 pos_aligned = AlignPow2(n->pos, page_size); + if(pos_aligned < n->cmt) + { + U8 *decommit_ptr = (U8 *)n + pos_aligned; + U64 decommit_size = n->cmt - pos_aligned; + AsanPoisonMemoryRegion(decommit_ptr, decommit_size); + decommit_memory(decommit_ptr, decommit_size); + n->cmt = pos_aligned; + } + } + +#if ARENA_FREE_LIST + // rjf: free chain -- decommit everything above the first (header) page + for(Arena *n = arena->free_last; n != 0; n = n->prev) + { + U64 keep = AlignPow2(ARENA_HEADER_SIZE, page_size); + if(keep < n->cmt) + { + U8 *decommit_ptr = (U8 *)n + keep; + U64 decommit_size = n->cmt - keep; + AsanPoisonMemoryRegion(decommit_ptr, decommit_size); + decommit_memory(decommit_ptr, decommit_size); + n->cmt = keep; + } + } +#endif +} + //- rjf: arena push/pop helpers internal void diff --git a/src/base/base_arena.h b/src/base/base_arena.h index b3ba37827..a61f5bd3f 100644 --- a/src/base/base_arena.h +++ b/src/base/base_arena.h @@ -76,6 +76,9 @@ internal void *arena_push(Arena *arena, U64 size, U64 align, B32 zero); internal U64 arena_pos(Arena *arena); internal void arena_pop_to(Arena *arena, U64 pos); +//- rjf: arena decommit of unused (rewound/free) pages +internal void arena_decommit_unused(Arena *arena); + //- rjf: arena push/pop helpers internal void arena_clear(Arena *arena); internal void arena_pop(Arena *arena, U64 amt); diff --git a/src/base/base_core.h b/src/base/base_core.h index 438316179..043726e3a 100644 --- a/src/base/base_core.h +++ b/src/base/base_core.h @@ -672,18 +672,8 @@ Compiler; # define Compiler_CURRENT Compiler_Null #endif -typedef enum Linker -{ - Linker_Null, - Linker_radlink, - Linker_msvc, - Linker_lld, - Linker_COUNT -} -Linker; - -//////////////////////////////// -//~ rjf: Access Flags +//////////////////////////////// +//~ rjf: Access Flags typedef U32 AccessFlags; enum diff --git a/src/base/base_files.c b/src/base/base_files.c index ca63d3900..4c24664f4 100644 --- a/src/base/base_files.c +++ b/src/base/base_files.c @@ -18,6 +18,20 @@ file_match(File a, File b) return result; } +internal B32 +file_pair_ok(FilePair f) +{ + return !file_match(f.read, file_zero()) && !file_match(f.write, file_zero()); +} + +internal void +file_pair_close(FilePair *p) +{ + file_close(p->read); + file_close(p->write); + MemoryZeroStruct(p); +} + //////////////////////////////// //~ rjf: Filesystem Helpers (Helpers, Implemented Once) diff --git a/src/base/base_files.h b/src/base/base_files.h index b06216c17..016115ec5 100644 --- a/src/base/base_files.h +++ b/src/base/base_files.h @@ -58,6 +58,8 @@ struct FileMap internal File file_zero(void); internal B32 file_match(File a, File b); +internal B32 file_pair_ok(FilePair p); +internal void file_pair_close(FilePair *p); //////////////////////////////// //~ rjf: Filesystem Helpers (Helpers, Implemented Once) diff --git a/src/base/base_thread_context.c b/src/base/base_thread_context.c index 421636101..df2ec71e5 100644 --- a/src/base/base_thread_context.c +++ b/src/base/base_thread_context.c @@ -17,15 +17,21 @@ C_LINKAGE thread_static TCTX *tctx_thread_local = 0; internal TCTX * tctx_alloc(void) { + // 2MB commit quantum for scratch arenas (vs the 64KB default): scratch takes + // heavy churn on every thread (the linker pushes tens of GB through these); + // the larger quantum cuts VirtualAlloc(MEM_COMMIT) calls -- all serialized on + // the process address-space lock -- ~32x. Slack is <= 2MB per scratch arena + // past its high-water mark (2 arenas per thread), and arena_decommit_unused + // still trims page-granular, independent of the commit quantum. #if PROFILE_TELEMETRY thread_static static char name[2][1024]; raddbg_snprintf(name[0], sizeof(name[0]), "Scratch/0[TID:%u]", tid()); raddbg_snprintf(name[1], sizeof(name[1]), "Scratch/1[TID:%u]", tid()); - Arena *arena_0 = arena_alloc(.name = name[0]); - Arena *arena_1 = arena_alloc(.name = name[1]); + Arena *arena_0 = arena_alloc(.commit_size = MB(2), .name = name[0]); + Arena *arena_1 = arena_alloc(.commit_size = MB(2), .name = name[1]); #else - Arena *arena_0 = arena_alloc(); - Arena *arena_1 = arena_alloc(); + Arena *arena_0 = arena_alloc(.commit_size = MB(2)); + Arena *arena_1 = arena_alloc(.commit_size = MB(2)); #endif TCTX *tctx = push_array(arena_0, TCTX, 1); tctx->arenas[0] = arena_0; @@ -82,6 +88,21 @@ tctx_get_scratch(Arena **conflicts, U64 count) return result; } +//- rjf: scratch decommit (release committed-but-unused scratch pages back to OS) + +internal void +tctx_scratch_decommit(void) +{ + TCTX *tctx = tctx_selected(); + for(U64 i = 0; i < ArrayCount(tctx->arenas); i += 1) + { + if(tctx->arenas[i] != 0) + { + arena_decommit_unused(tctx->arenas[i]); + } + } +} + //- rjf: lane metadata internal LaneCtx diff --git a/src/base/base_thread_context.h b/src/base/base_thread_context.h index 494d1804f..e52b36476 100644 --- a/src/base/base_thread_context.h +++ b/src/base/base_thread_context.h @@ -90,6 +90,7 @@ internal TCTX *tctx_selected(void); //- rjf: scratch arenas internal Arena *tctx_get_scratch(Arena **conflicts, U64 count); +internal void tctx_scratch_decommit(void); #define scratch_begin(conflicts, count) temp_begin(tctx_get_scratch((conflicts), (count))) #define scratch_end(scratch) temp_end(scratch) diff --git a/src/base/base_threads.h b/src/base/base_threads.h index 3aae2ea6c..b40c10f9f 100644 --- a/src/base/base_threads.h +++ b/src/base/base_threads.h @@ -128,8 +128,12 @@ internal void semaphore_release(Semaphore semaphore); internal Semaphore semaphore_open(String8 name); internal void semaphore_close(Semaphore semaphore); internal B32 semaphore_take(Semaphore semaphore, U64 endt_us); +internal B32 semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us); // blocking acquire of `count` permits (off hot path) internal void semaphore_drop(Semaphore semaphore); internal void semaphore_drop_count(Semaphore semaphore, U64 drop_count); +internal void semaphore_drop_if_room(Semaphore semaphore); // best-effort post; no-op if already at max +internal void semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall +internal B32 semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out); // release 1 permit + report the pre-release count (win32: exact, from ReleaseSemaphore; posix: best-effort sem_getvalue) //- rjf: barriers internal Barrier barrier_alloc(U64 count); diff --git a/src/codeview/codeview.h b/src/codeview/codeview.h index 196e19960..b433af228 100644 --- a/src/codeview/codeview.h +++ b/src/codeview/codeview.h @@ -2701,6 +2701,32 @@ struct CV_TypeIndexInfoList CV_TypeIndexInfo *last; }; +typedef struct CV_TiOff CV_TiOff; +struct CV_TiOff +{ + CV_TypeIndexSource source; + U32 offset; +}; + +// Flat, allocation-free view over a record's type-index sites. For any record kind at +// most one of the two parts is populated: +// - `arr`: fixed-shape kinds point at a static per-kind table (zero allocation); +// member-walk kinds (FIELDLIST/METHODLIST/inlinee lines) point at an arena- +// materialized array. +// - homogeneous run (count-stride kinds: ARGLIST, SUBSTR_LIST, BUILDINFO, VFTPATH, +// CALLERS/CALLEES/INLINEES): offset(i) = run_base + i*sizeof(CV_TypeIndex). +// Emission order (arr order, then ascending run) matches the legacy +// CV_TypeIndexInfoList push order exactly; hash streams depend on it. +typedef struct CV_TiOffsets CV_TiOffsets; +struct CV_TiOffsets +{ + const CV_TiOff *arr; + U32 arr_count; + CV_TypeIndexSource run_source; + U32 run_base; + U32 run_count; +}; + typedef struct CV_TypeIndexArray CV_TypeIndexArray; struct CV_TypeIndexArray { diff --git a/src/codeview/codeview_parse.c b/src/codeview/codeview_parse.c index 3997d4a47..bb0b84c5e 100644 --- a/src/codeview/codeview_parse.c +++ b/src/codeview/codeview_parse.c @@ -588,180 +588,269 @@ cv_symbol_type_index_info_push(Arena *arena, CV_TypeIndexInfoList *list, CV_Type SLLQueuePush(list->first, list->last, info); list->count += 1; - + return info; } -internal CV_TypeIndexInfoList -cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data) +//////////////////////////////// +//~ Type-index offset descriptors +// +// Static per-kind tables for fixed-shape records; homogeneous runs for +// count-stride records. Table entry order mirrors the legacy +// cv_symbol_type_index_info_push order exactly (incl. FUNC_ID IPI-before-TPI +// and UDT_SRC_LINE TPI-then-IPI asymmetries) -- leaf hash streams depend on it. + +#define CV_TIOFF(s, o) { CV_TypeIndexSource_##s, (U32)(o) } + +// leaves +read_only global CV_TiOff cv_tioffs_lf_modifier[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafModifier, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_pointer[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafPointer, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_pointer_ex[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafPointer, itype)), CV_TIOFF(TPI, sizeof(CV_LeafPointer) + 0) }; +read_only global CV_TiOff cv_tioffs_lf_array[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafArray, entry_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafArray, index_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_struct[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, field_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, derived_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, vshape_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_struct2[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, field_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, derived_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, vshape_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_union[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUnion, field_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_alias[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafAlias, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_func_id[] = { CV_TIOFF(IPI, OffsetOf(CV_LeafFuncId, scope_string_id)), CV_TIOFF(TPI, OffsetOf(CV_LeafFuncId, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_mfunc_id[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMFuncId, owner_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFuncId, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_string_id[] = { CV_TIOFF(IPI, OffsetOf(CV_LeafStringId, substr_list_id)) }; +read_only global CV_TiOff cv_tioffs_lf_udt_src_line[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUDTSrcLine, udt_itype)), CV_TIOFF(IPI, OffsetOf(CV_LeafUDTSrcLine, src_string_id)) }; +read_only global CV_TiOff cv_tioffs_lf_udt_mod_src_line[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUDTModSrcLine, udt_itype)), CV_TIOFF(IPI, OffsetOf(CV_LeafUDTModSrcLine, src_string_id)) }; +read_only global CV_TiOff cv_tioffs_lf_enum[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafEnum, base_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafEnum, field_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_procedure[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafProcedure, ret_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafProcedure, arg_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_mfunction[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, ret_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, class_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, this_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, arg_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vftable[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFTable, owner_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafVFTable, base_table_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_skip[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafSkip, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_method[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMethod, list_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_onemethod[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafOneMethod, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_bitfield[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafBitField, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_index[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafIndex, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_member[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMember, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vfunctab[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFuncTab, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vfuncoff[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFuncOff, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_nesttype[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafNestType, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_nesttypeex[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafNestTypeEx, itype)) }; + +// symbols +read_only global CV_TiOff cv_tioffs_s_buildinfo[] = { CV_TIOFF(IPI, OffsetOf(CV_SymBuildInfo, id)) }; +read_only global CV_TiOff cv_tioffs_s_data32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymData32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_proc32_id[] = { CV_TIOFF(IPI, OffsetOf(CV_SymProc32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_proc32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymProc32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_udt[] = { CV_TIOFF(TPI, OffsetOf(CV_SymUDT, itype)) }; +read_only global CV_TiOff cv_tioffs_s_thread32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymThread32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_filestatic[] = { CV_TIOFF(TPI, OffsetOf(CV_SymFileStatic, itype)) }; +read_only global CV_TiOff cv_tioffs_s_local[] = { CV_TIOFF(TPI, OffsetOf(CV_SymLocal, itype)) }; +read_only global CV_TiOff cv_tioffs_s_regrel32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymRegrel32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_register[] = { CV_TIOFF(TPI, OffsetOf(CV_SymRegister, itype)) }; +read_only global CV_TiOff cv_tioffs_s_constant[] = { CV_TIOFF(TPI, OffsetOf(CV_SymConstant, itype)) }; +read_only global CV_TiOff cv_tioffs_s_callsiteinfo[] = { CV_TIOFF(TPI, OffsetOf(CV_SymCallSiteInfo, itype)) }; +read_only global CV_TiOff cv_tioffs_s_inlinesite[] = { CV_TIOFF(IPI, OffsetOf(CV_SymInlineSite, inlinee)) }; +read_only global CV_TiOff cv_tioffs_s_heapallocsite[] = { CV_TIOFF(TPI, OffsetOf(CV_SymHeapAllocSite, itype)) }; + +#undef CV_TIOFF + +#define CV_TIOFFS_FIXED(result, table) do { (result).arr = (table); (result).arr_count = ArrayCount(table); } while (0) + +internal U64 +cv_ti_offsets_count(const CV_TiOffsets *offs) { - CV_TypeIndexInfoList list = {0}; + return (U64)offs->arr_count + (U64)offs->run_count; +} + +internal CV_TiOff +cv_ti_offset_at(const CV_TiOffsets *offs, U64 idx) +{ + CV_TiOff result; + if (idx < offs->arr_count) { + result = offs->arr[idx]; + } else { + result.source = offs->run_source; + result.offset = offs->run_base + (U32)(idx - offs->arr_count) * (U32)sizeof(CV_TypeIndex); + } + return result; +} + +// grow-by-doubling scratch for member-walk kinds (FIELDLIST/METHODLIST/inlinee +// lines); memory footprint stays comparable to the legacy per-node list +typedef struct CV_TiOffBuilder +{ + Arena *arena; + CV_TiOff *v; + U32 count; + U32 cap; +} CV_TiOffBuilder; + +internal void +cv_tioff_builder_put(CV_TiOffBuilder *b, CV_TypeIndexSource source, U64 offset) +{ + if (b->count == b->cap) { + U32 new_cap = b->cap ? b->cap * 2 : 16; + CV_TiOff *new_v = push_array_no_zero(b->arena, CV_TiOff, new_cap); + MemoryCopy(new_v, b->v, sizeof(CV_TiOff) * b->count); + b->v = new_v; + b->cap = new_cap; + } + b->v[b->count].source = source; + b->v[b->count].offset = (U32)offset; + b->count += 1; +} + +internal CV_TiOffsets +cv_symbol_ti_offsets(CV_SymKind kind, String8 data) +{ + CV_TiOffsets result = {0}; switch (kind) { case CV_SymKind_BUILDINFO: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymBuildInfo, id)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_buildinfo); } break; case CV_SymKind_GDATA32: case CV_SymKind_LDATA32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymData32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_data32); } break; case CV_SymKind_LPROC32_ID: - case CV_SymKind_GPROC32_ID: + case CV_SymKind_GPROC32_ID: case CV_SymKind_LPROC32_DPC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymProc32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_proc32_id); } break; case CV_SymKind_GPROC32: - case CV_SymKind_LPROC32: + case CV_SymKind_LPROC32: case CV_SymKind_LPROC32_DPC: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymProc32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_proc32); } break; case CV_SymKind_UDT: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymUDT, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_udt); } break; case CV_SymKind_GTHREAD32: case CV_SymKind_LTHREAD32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymThread32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_thread32); } break; case CV_SymKind_FILESTATIC: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymFileStatic, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_filestatic); } break; case CV_SymKind_LOCAL: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymLocal, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_local); } break; - case CV_SymKind_REGREL32: + case CV_SymKind_REGREL32: case CV_SymKind_BPREL32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymRegrel32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_regrel32); } break; case CV_SymKind_REGISTER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymRegister, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_register); } break; case CV_SymKind_CONSTANT: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymConstant, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_constant); } break; case CV_SymKind_CALLSITEINFO: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymCallSiteInfo, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_callsiteinfo); } break; case CV_SymKind_CALLERS: case CV_SymKind_CALLEES: case CV_SymKind_INLINEES: { Assert(data.size >= sizeof(CV_SymFunctionList)); CV_SymFunctionList *func_list = (CV_SymFunctionList*)data.str; - for (U64 i = 0; i < func_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_SymFunctionList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_SymFunctionList); + result.run_count = func_list->count; } break; case CV_SymKind_INLINESITE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymInlineSite, inlinee)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_inlinesite); } break; case CV_SymKind_HEAPALLOCSITE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymHeapAllocSite, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_heapallocsite); } break; } - return list; + return result; } -internal CV_TypeIndexInfoList -cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) +internal CV_TiOffsets +cv_leaf_ti_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) { - CV_TypeIndexInfoList list = {0}; + CV_TiOffsets result = {0}; switch (leaf_kind) { case CV_LeafKind_NOTYPE: case CV_LeafKind_VTSHAPE: case CV_LeafKind_LABEL: - case CV_LeafKind_NULL: + case CV_LeafKind_NULL: case CV_LeafKind_NOTTRAN: { // no type indices } break; case CV_LeafKind_MODIFIER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafModifier, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_modifier); } break; case CV_LeafKind_POINTER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafPointer, itype)); - CV_LeafPointer *ptr = (CV_LeafPointer *)data.str; - CV_PointerKind ptr_kind = CV_PointerAttribs_Extract_Kind(ptr->attribs); + CV_LeafPointer *ptr = (CV_LeafPointer *)data.str; + CV_PointerKind ptr_kind = CV_PointerAttribs_Extract_Kind(ptr->attribs); if (ptr_kind == CV_PointerKind_BaseType) { // TODO: add CV_LeafPointerBaseType - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafPointer) + 0); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer_ex); } else { CV_PointerMode ptr_mode = CV_PointerAttribs_Extract_Mode(ptr->attribs); if (ptr_mode == CV_PointerMode_PtrMem || ptr_mode == CV_PointerMode_PtrMethod) { // TODO: add type for the CvLeafPointerMember to syms_cv.mc - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafPointer) + 0); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer_ex); + } else { + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer); } } } break; case CV_LeafKind_ARRAY: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafArray, entry_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafArray, index_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_array); } break; - case CV_LeafKind_CLASS: + case CV_LeafKind_CLASS: case CV_LeafKind_STRUCTURE: case CV_LeafKind_INTERFACE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, field_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, derived_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, vshape_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_struct); } break; case CV_LeafKind_CLASS2: case CV_LeafKind_STRUCT2: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, field_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, derived_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, vshape_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_struct2); } break; case CV_LeafKind_UNION: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUnion, field_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_union); } break; case CV_LeafKind_ALIAS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafAlias, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_alias); } break; case CV_LeafKind_FUNC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafFuncId, scope_string_id)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafFuncId, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_func_id); } break; case CV_LeafKind_MFUNC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFuncId, owner_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFuncId, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_mfunc_id); } break; case CV_LeafKind_STRING_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafStringId, substr_list_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_string_id); } break; case CV_LeafKind_UDT_SRC_LINE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUDTSrcLine, udt_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafUDTSrcLine, src_string_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_udt_src_line); } break; case CV_LeafKind_UDT_MOD_SRC_LINE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUDTModSrcLine, udt_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafUDTModSrcLine, src_string_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_udt_mod_src_line); } break; case CV_LeafKind_BUILDINFO: { Assert(data.size >= sizeof(CV_LeafBuildInfo)); CV_LeafBuildInfo *build_info = (CV_LeafBuildInfo *)data.str; - for (U16 i = 0; i < build_info->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_LeafBuildInfo) + i * sizeof(CV_ItemId)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_LeafBuildInfo); + result.run_count = build_info->count; } break; case CV_LeafKind_ENUM: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafEnum, base_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafEnum, field_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_enum); } break; case CV_LeafKind_PROCEDURE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafProcedure, ret_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafProcedure, arg_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_procedure); } break; case CV_LeafKind_MFUNCTION: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, ret_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, class_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, this_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, arg_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_mfunction); } break; case CV_LeafKind_VFTABLE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFTable, owner_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFTable, base_table_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vftable); } break; case CV_LeafKind_VFTPATH: { Assert(sizeof(CV_LeafVFPath) <= data.size); CV_LeafVFPath *vfpath = (CV_LeafVFPath *)data.str; - for (U32 i = 0; i < vfpath->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafVFPath) + i * sizeof(CV_TypeId)); - } + result.run_source = CV_TypeIndexSource_TPI; + result.run_base = sizeof(CV_LeafVFPath); + result.run_count = vfpath->count; } break; case CV_LeafKind_TYPESERVER: case CV_LeafKind_TYPESERVER2: @@ -769,76 +858,77 @@ cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data // no type indices } break; case CV_LeafKind_SKIP: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafSkip, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_skip); } break; case CV_LeafKind_SUBSTR_LIST: { Assert(sizeof(CV_LeafArgList) <= data.size); CV_LeafArgList *arg_list = (CV_LeafArgList*)data.str; - for (U32 i = 0; i < arg_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_LeafArgList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_LeafArgList); + result.run_count = arg_list->count; } break; case CV_LeafKind_ARGLIST: { Assert(sizeof(CV_LeafArgList) <= data.size); CV_LeafArgList *arg_list = (CV_LeafArgList*)data.str; - for (U32 i = 0; i < arg_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafArgList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_TPI; + result.run_base = sizeof(CV_LeafArgList); + result.run_count = arg_list->count; } break; - case CV_LeafKind_LIST: + case CV_LeafKind_LIST: case CV_LeafKind_FIELDLIST: { + CV_TiOffBuilder b = { arena }; for (U64 cursor = 0; cursor < data.size; ) { CV_LeafKind list_member_kind = 0; U64 read_size = str8_deserial_read_struct(data, cursor, &list_member_kind); - + if(read_size != sizeof(list_member_kind)) { Assert(!"malformed LF_FIELDLIST"); break; } cursor += read_size; - + switch (list_member_kind) { default: Assert(!"TODO: handle malformed field member"); break; case CV_LeafKind_INDEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafIndex, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafIndex, itype)); cursor += sizeof(CV_LeafIndex); } break; case CV_LeafKind_MEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMember, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMember, itype)); cursor += sizeof(CV_LeafMember); - + CV_NumericParsed size; cursor += cv_read_numeric(data, cursor, &size); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_STMEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafStMember, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafStMember, itype)); cursor += sizeof(CV_LeafStMember); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_METHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethod, list_itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethod, list_itype)); cursor += sizeof(CV_LeafMethod); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_ONEMETHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafOneMethod, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafOneMethod, itype)); + CV_LeafOneMethod onemethod; cursor += str8_deserial_read_struct(data, cursor, &onemethod); - + CV_MethodProp prop = CV_FieldAttribs_Extract_MethodProp(onemethod.attribs); if(prop == CV_MethodProp_PureIntro || prop == CV_MethodProp_Intro) { cursor += sizeof(U32); // virtoff } - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; @@ -851,134 +941,140 @@ cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_NESTTYPE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestType, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestType, itype)); cursor += sizeof(CV_LeafNestType); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_NESTTYPEEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestTypeEx, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestTypeEx, itype)); + cursor += sizeof(CV_LeafNestTypeEx); String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_BCLASS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafBClass, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafBClass, itype)); + cursor += sizeof(CV_LeafBClass); CV_NumericParsed offset; cursor += cv_read_numeric(data, cursor, &offset); } break; case CV_LeafKind_VBCLASS: case CV_LeafKind_IVBCLASS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVBClass, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVBClass, itype)); cursor += sizeof(CV_LeafVBClass); - + CV_NumericParsed virtual_base_pointer; cursor += cv_read_numeric(data, cursor, &virtual_base_pointer); - + CV_NumericParsed virtual_base_offset; cursor += cv_read_numeric(data, cursor, &virtual_base_offset); } break; case CV_LeafKind_VFUNCTAB: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncTab, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncTab, itype)); cursor += sizeof(CV_LeafVFuncTab); } break; case CV_LeafKind_VFUNCOFF: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncOff, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncOff, itype)); cursor += sizeof(CV_LeafVFuncOff); } break; } cursor = AlignPow2(cursor, 4); } + result.arr = b.v; + result.arr_count = b.count; } break; case CV_LeafKind_METHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMethod, list_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_method); } break; case CV_LeafKind_METHODLIST: { + CV_TiOffBuilder b = { arena }; for (U64 cursor = 0; cursor < data.size; ) { // read method CV_LeafMethodListMember method; U64 read_size = str8_deserial_read_struct(data, cursor, &method); - + // error check read if (read_size != sizeof(method)) { Assert(!"malformed LF_METHODLIST"); break; } - + // push type index offset - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethodListMember, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethodListMember, itype)); + // take into account intro virtual offset CV_MethodProp mprop = CV_FieldAttribs_Extract_MethodProp(method.attribs); if (mprop == CV_MethodProp_Intro || mprop == CV_MethodProp_PureIntro) { read_size += sizeof(U32); } - + // advance cursor += read_size; } + result.arr = b.v; + result.arr_count = b.count; } break; case CV_LeafKind_ONEMETHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafOneMethod, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_onemethod); } break; case CV_LeafKind_BITFIELD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafBitField, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_bitfield); } break; case CV_LeafKind_PRECOMP: case CV_LeafKind_REFSYM: { // no type indices } break; case CV_LeafKind_INDEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafIndex, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_index); } break; case CV_LeafKind_MEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMember, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_member); } break; case CV_LeafKind_VFUNCTAB: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFuncTab, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vfunctab); } break; case CV_LeafKind_VFUNCOFF: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFuncOff, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vfuncoff); } break; case CV_LeafKind_NESTTYPE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafNestType, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_nesttype); } break; case CV_LeafKind_NESTTYPEEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafNestTypeEx, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_nesttypeex); } break; default: { NotImplemented; } break; } - return list; + return result; } -internal CV_TypeIndexInfoList -cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) +internal CV_TiOffsets +cv_inlinee_ti_offsets(Arena *arena, String8 raw_data) { - CV_TypeIndexInfoList list = {0}; - + CV_TiOffsets result = {0}; + CV_TiOffBuilder b = { arena }; + U64 cursor = 0; - + // first four bytes are always signature CV_C13InlineeLinesSig sig = max_U32; cursor += str8_deserial_read_struct(raw_data, cursor, &sig); - + while(cursor < raw_data.size) { // read header CV_C13InlineeSourceLineHeader *header = (CV_C13InlineeSourceLineHeader *) str8_deserial_get_raw_ptr(raw_data, cursor, sizeof(CV_C13InlineeSourceLineHeader)); - + // store type index offset - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, cursor + OffsetOf(CV_C13InlineeSourceLineHeader, inlinee)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_IPI, cursor + OffsetOf(CV_C13InlineeSourceLineHeader, inlinee)); + // advance past header cursor += sizeof(*header); - + // skip extra files B32 has_extra_files = (sig == CV_C13InlineeLinesSig_EXTRA_FILES); if (has_extra_files) @@ -988,10 +1084,41 @@ cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) cursor += /* file id: */ sizeof(U32) * file_count; } } - + + result.arr = b.v; + result.arr_count = b.count; + return result; +} + +internal CV_TypeIndexInfoList +cv_ti_offsets_to_list(Arena *arena, CV_TiOffsets offs) +{ + CV_TypeIndexInfoList list = {0}; + for (U64 i = 0, count = cv_ti_offsets_count(&offs); i < count; i += 1) { + CV_TiOff ti = cv_ti_offset_at(&offs, i); + cv_symbol_type_index_info_push(arena, &list, ti.source, ti.offset); + } return list; } +internal CV_TypeIndexInfoList +cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data) +{ + return cv_ti_offsets_to_list(arena, cv_symbol_ti_offsets(kind, data)); +} + +internal CV_TypeIndexInfoList +cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) +{ + return cv_ti_offsets_to_list(arena, cv_leaf_ti_offsets(arena, leaf_kind, data)); +} + +internal CV_TypeIndexInfoList +cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) +{ + return cv_ti_offsets_to_list(arena, cv_inlinee_ti_offsets(arena, raw_data)); +} + internal String8Array cv_get_data_around_type_indices(Arena *arena, CV_TypeIndexInfoList ti_list, String8 data) { diff --git a/src/codeview/codeview_parse.h b/src/codeview/codeview_parse.h index c33377988..b922f7eaa 100644 --- a/src/codeview/codeview_parse.h +++ b/src/codeview/codeview_parse.h @@ -291,6 +291,12 @@ internal B32 cv_is_leaf_type_server(CV_LeafKind kind); internal B32 cv_is_leaf_pch(CV_LeafKind kind); internal CV_TypeIndexSource cv_type_index_source_from_leaf_kind(CV_LeafKind leaf_kind); +internal CV_TiOffsets cv_symbol_ti_offsets(CV_SymKind kind, String8 data); // never allocates +internal CV_TiOffsets cv_leaf_ti_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data); // allocates only for FIELDLIST/METHODLIST +internal CV_TiOffsets cv_inlinee_ti_offsets(Arena *arena, String8 raw_data); +internal U64 cv_ti_offsets_count(const CV_TiOffsets *offs); +internal CV_TiOff cv_ti_offset_at(const CV_TiOffsets *offs, U64 idx); + internal CV_TypeIndexInfoList cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data); internal CV_TypeIndexInfoList cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data); internal CV_TypeIndexInfoList cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data); diff --git a/src/coff/coff.c b/src/coff/coff.c index 437b39de1..e7da807b7 100644 --- a/src/coff/coff.c +++ b/src/coff/coff.c @@ -293,6 +293,26 @@ coff_make_import_lookup(Arena *arena, U16 hint, String8 name) return result; } +internal String8 +coff_import_lookup_name_from_import_by(String8 name, COFF_ImportByType import_by) +{ + // IMPORT_OBJECT_NAME_NO_PREFIX: the public symbol name without a leading ?, @, or _. + // IMPORT_OBJECT_NAME_UNDECORATE: prefix skipped as above, then truncated at the first @. + String8 result = name; + if (import_by == COFF_ImportBy_NameNoPrefix || import_by == COFF_ImportBy_Undecorate) { + if (result.size > 0 && (result.str[0] == '?' || result.str[0] == '@' || result.str[0] == '_')) { + result = str8_skip(result, 1); + } + if (import_by == COFF_ImportBy_Undecorate) { + U64 at_pos = str8_find_needle(result, 0, str8_lit("@"), 0); + if (at_pos < result.size) { + result = str8_prefix(result, at_pos); + } + } + } + return result; +} + internal U32 coff_make_ordinal32(U16 hint) { diff --git a/src/coff/coff.h b/src/coff/coff.h index 5f379ee55..94b91f511 100644 --- a/src/coff/coff.h +++ b/src/coff/coff.h @@ -621,6 +621,7 @@ internal String8 coff_ordinal_data_from_hint(Arena *arena, COFF_MachineType mach internal String8 coff_make_lib_member_header(Arena *arena, String8 name, COFF_TimeStamp time_stamp, U16 user_id, U16 group_id, U16 mode, U32 size); internal String8 coff_make_import_lookup(Arena *arena, U16 hint, String8 name); +internal String8 coff_import_lookup_name_from_import_by(String8 name, COFF_ImportByType import_by); internal String8 coff_make_import_header(Arena *arena, COFF_MachineType machine, COFF_TimeStamp time_stamp, String8 dll_name, COFF_ImportByType import_by, String8 name, U16 hint_or_ordinal, COFF_ImportType type); //////////////////////////////// diff --git a/src/coff/coff_obj_writer.c b/src/coff/coff_obj_writer.c index 01b9cbfbd..bceaf9646 100644 --- a/src/coff/coff_obj_writer.c +++ b/src/coff/coff_obj_writer.c @@ -32,12 +32,13 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) str8_list_push(scratch.arena, &string_table, str8_struct(string_table_size)); // - // assing section numbers + // assign section numbers // U64 obj_sections_count; COFF_ObjSection **obj_sections; { obj_sections_count = obj_writer->sect_count; + AssertAlways(obj_sections_count < COFF_Symbol_DebugSection32); obj_sections = push_array(scratch.arena, COFF_ObjSection *, obj_writer->sect_count); U64 sect_idx = 0; for (COFF_ObjSectionNode *sect_n = obj_writer->sect_first; sect_n != 0; sect_n = sect_n->next, sect_idx += 1) { @@ -47,7 +48,10 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) } } - AssertAlways(obj_sections_count <= max_U16); + // Standard COFF reserves section numbers 0xff00 and above. BigObj also + // widens every symbol/auxiliary record from 18 to 20 bytes. + B32 is_big_obj = obj_sections_count > 0xfeff; + U64 symbol_size = is_big_obj ? sizeof(COFF_Symbol32) : sizeof(COFF_Symbol16); // // serialize symbol table @@ -69,9 +73,6 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) for (COFF_ObjSymbolNode *symbol_n = obj_writer->symbol_first; symbol_n != 0; symbol_n = symbol_n->next) { COFF_ObjSymbol *s = &symbol_n->v; - COFF_Symbol16 *d = push_array(scratch.arena, COFF_Symbol16, 1); - str8_list_push(scratch.arena, &symbol_table, str8_struct(d)); - COFF_SymbolName name = {0}; // long name if (s->name.size > sizeof(name.short_name)) { @@ -90,28 +91,43 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) // symbol header AssertAlways(s->aux_symbols.node_count <= max_U8); - d->name = name; - d->value = s->value; + U32 section_number = 0; switch (s->loc.type) { case COFF_SymbolLocation_Null: break; - case COFF_SymbolLocation_Section: d->section_number = safe_cast_u16(s->loc.u.section->section_number); break; - case COFF_SymbolLocation_Abs: d->section_number = COFF_Symbol_AbsSection16; break; - case COFF_SymbolLocation_Undef: d->section_number = COFF_Symbol_UndefinedSection; break; - case COFF_SymbolLocation_Common: d->section_number = COFF_Symbol_UndefinedSection; break; + case COFF_SymbolLocation_Section: section_number = s->loc.u.section->section_number; break; + case COFF_SymbolLocation_Abs: section_number = is_big_obj ? COFF_Symbol_AbsSection32 : COFF_Symbol_AbsSection16; break; + case COFF_SymbolLocation_Undef: section_number = COFF_Symbol_UndefinedSection; break; + case COFF_SymbolLocation_Common: section_number = COFF_Symbol_UndefinedSection; break; + } + if (is_big_obj) { + COFF_Symbol32 *d = push_array(scratch.arena, COFF_Symbol32, 1); + d->name = name; + d->value = s->value; + d->section_number = section_number; + d->type = s->type; + d->storage_class = s->storage_class; + d->aux_symbol_count = (U8)s->aux_symbols.node_count; + str8_list_push(scratch.arena, &symbol_table, str8_struct(d)); + } else { + COFF_Symbol16 *d = push_array(scratch.arena, COFF_Symbol16, 1); + d->name = name; + d->value = s->value; + d->section_number = safe_cast_u16(section_number); + d->type = s->type; + d->storage_class = s->storage_class; + d->aux_symbol_count = (U8)s->aux_symbols.node_count; + str8_list_push(scratch.arena, &symbol_table, str8_struct(d)); } - d->type = s->type; - d->storage_class = s->storage_class; - d->aux_symbol_count = 0; U64 start_symbol_idx = symbol_idx; if (s->storage_class == COFF_SymStorageClass_WeakExternal) { if (s->aux_symbols.node_count > 0) { COFF_ObjSymbolWeak *s_weak = (COFF_ObjSymbolWeak *)s->aux_symbols.first->string.str; - COFF_SymbolWeakExt *d_weak = push_array(scratch.arena, COFF_SymbolWeakExt, 1); + COFF_SymbolWeakExt *d_weak = (COFF_SymbolWeakExt *)push_array(scratch.arena, U8, symbol_size); d_weak->tag_index = s_weak->tag ? s_weak->tag->idx : max_U32; d_weak->characteristics = s_weak->characteristics; - str8_list_push(scratch.arena, &symbol_table, str8_struct(d_weak)); + str8_list_push(scratch.arena, &symbol_table, str8((U8 *)d_weak, symbol_size)); symbol_idx += 1; } } else if (s->storage_class == COFF_SymStorageClass_Static) { @@ -120,15 +136,17 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) COFF_ObjSection *sect = s->loc.u.section; COFF_ObjSymbolSecDef *s_sd = (COFF_ObjSymbolSecDef *)s->aux_symbols.first->string.str; - COFF_SymbolSecDef *d_sd = push_array(scratch.arena, COFF_SymbolSecDef, 1); + COFF_SymbolSecDef *d_sd = (COFF_SymbolSecDef *)push_array(scratch.arena, U8, symbol_size); d_sd->length = safe_cast_u32(sect->data.total_size); d_sd->number_of_relocations = (U16)sect->reloc_count; d_sd->check_sum = 0; - d_sd->number_lo = s_sd->selection == COFF_ComdatSelect_Associative ? safe_cast_u16(s_sd->associate->section_number) : 0; + U32 associate_number = s_sd->selection == COFF_ComdatSelect_Associative ? s_sd->associate->section_number : 0; + d_sd->number_lo = (U16)associate_number; + d_sd->number_hi = (U16)(associate_number >> 16); d_sd->selection = s_sd->selection; - str8_list_push(scratch.arena, &symbol_table, str8_struct(d_sd)); + str8_list_push(scratch.arena, &symbol_table, str8((U8 *)d_sd, symbol_size)); symbol_idx += 1; } } @@ -136,26 +154,37 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) U8 processed_aux_symbol_count = (U8)(symbol_idx - start_symbol_idx); for (U64 aux_idx = processed_aux_symbol_count; aux_idx < s->aux_symbols.node_count; aux_idx += 1) { - COFF_Symbol16 *a = push_array(scratch.arena, COFF_Symbol16, 1); - str8_list_push(scratch.arena, &symbol_table, str8_struct(a)); + U8 *a = push_array(scratch.arena, U8, symbol_size); + str8_list_push(scratch.arena, &symbol_table, str8(a, symbol_size)); } - - d->aux_symbol_count = (U8)s->aux_symbols.node_count; } } // // file header // - COFF_FileHeader *file_header = push_array(scratch.arena, COFF_FileHeader, 1); - file_header->machine = obj_writer->machine; - file_header->section_count = obj_sections_count; - file_header->time_stamp = obj_writer->time_stamp; - file_header->symbol_table_foff = 0; - file_header->symbol_count = safe_cast_u32(symbol_table.node_count); - file_header->optional_header_size = 0; - file_header->flags = 0; - str8_list_push(scratch.arena, &srl, str8_struct(file_header)); + U32 *symbol_table_foff; + if (is_big_obj) { + COFF_BigObjHeader *header = push_array(scratch.arena, COFF_BigObjHeader, 1); + header->sig1 = COFF_MachineType_Unknown; + header->sig2 = max_U16; + header->version = 2; + header->machine = obj_writer->machine; + header->time_stamp = obj_writer->time_stamp; + MemoryCopy(header->magic, g_coff_big_header_magic, sizeof(header->magic)); + header->section_count = safe_cast_u32(obj_sections_count); + header->symbol_count = safe_cast_u32(symbol_table.node_count); + symbol_table_foff = &header->symbol_table_foff; + str8_list_push(scratch.arena, &srl, str8_struct(header)); + } else { + COFF_FileHeader *header = push_array(scratch.arena, COFF_FileHeader, 1); + header->machine = obj_writer->machine; + header->section_count = safe_cast_u16(obj_sections_count); + header->time_stamp = obj_writer->time_stamp; + header->symbol_count = safe_cast_u32(symbol_table.node_count); + symbol_table_foff = &header->symbol_table_foff; + str8_list_push(scratch.arena, &srl, str8_struct(header)); + } // // section table @@ -230,7 +259,7 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) // symbol table // if (symbol_table.total_size || string_table.total_size > sizeof(*string_table_size)) { - file_header->symbol_table_foff = srl.total_size; + *symbol_table_foff = safe_cast_u32(srl.total_size); str8_list_concat_in_place(&srl, &symbol_table); } diff --git a/src/linker/base_ext/base_blake3.c b/src/linker/base_ext/base_blake3.c index 59f063c42..df9324555 100644 --- a/src/linker/base_ext/base_blake3.c +++ b/src/linker/base_ext/base_blake3.c @@ -1,103 +1,16 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wmacro-redefined" -#pragma clang diagnostic ignored "-Wsign-conversion" -#elif defined(_MSC_VER) -#pragma warning (push, 0) -#endif - -#include "third_party/blake3/c/blake3_portable.c" - -#if defined(_M_AMD64) || defined(__x86_64__) - -#define round_fn sse2_round_fn -#define compress_pre sse2_compress_pre - -#include "third_party/blake3/c/blake3_sse2.c" -#define loadu sse41_loadu -#define storeu sse41_storeu -#define addv sse41_addv -#define xorv sse41_xorv -#define set1 sse41_set1 -#define set4 sse41_set4 -#define rot16 sse41_rot16 -#define rot12 sse41_rot12 -#define rot8 sse41_rot8 -#define rot7 sse41_rot7 -#define g1 sse41_g1 -#define g2 sse41_g2 -#define diagonalize sse41_diagonalize -#define undiagonalize sse41_undiagonalize -#define compress_pre sse41_compress_pre -#define round_fn sse41_round_fn -#define transpose_vecs sse41_transpose_vecs -#define transpose_msg_vecs sse41_transpose_msg_vecs -#define load_counters sse41_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("sse4.1"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_sse41.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#define loadu avx2_loadu -#define storeu avx2_storeu -#define addv avx2_addv -#define xorv avx2_xorv -#define set1 avx2_set1 -#define rot7 avx2_rot7 -#define rot8 avx2_rot8 -#define rot12 avx2_rot12 -#define rot16 avx2_rot16 -#define round_fn avx2_round_fn -#define transpose_vecs avx2_transpose_vecs -#define transpose_msg_vecs avx2_transpose_msg_vecs -#define load_counters avx2_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("avx2"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_avx2.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#define set4 avx512_set4 -#define g1 avx512_g1 -#define g2 avx512_g2 -#define diagonalize avx512_diagonalize -#define undiagonalize avx512_undiagonalize -#define compress_pre avx512_compress_pre -#define transpose_vecs avx512_transpose_vecs -#define transpose_msg_vecs avx512_transpose_msg_vecs -#define load_counters avx512_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("avx512f,avx512vl"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_avx512.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#endif +#include "third_party/blake3/blake3_portable.c" #if defined(__aarch64__) || defined(_M_ARM64) -#include "third_party/blake3/c/blake3_neon.c" +#include "third_party/blake3/blake3_neon.c" #endif -#include "third_party/blake3/c/blake3_dispatch.c" -#include "third_party/blake3/c/blake3.c" +#include "third_party/blake3/blake3_dispatch.c" +#include "third_party/blake3/blake3.c" -#if defined(__clang__) -#pragma clang diagnostic pop -#elif defined(_MSC_VER) -#pragma warning (pop, 0) +#if defined(_MSC_VER) && defined(_M_AMD64) +#pragma comment (lib, "blake3") #endif - diff --git a/src/linker/base_ext/base_blake3.h b/src/linker/base_ext/base_blake3.h index e80c1e35a..90e7cdfdf 100644 --- a/src/linker/base_ext/base_blake3.h +++ b/src/linker/base_ext/base_blake3.h @@ -6,7 +6,7 @@ #define BLAKE3_API static #define BLAKE3_PRIVATE static -#include "third_party/blake3/c/blake3.h" +#include "third_party/blake3/blake3.h" static void blake3(void* out, size_t outlen, void* in, size_t inlen) diff --git a/src/linker/base_ext/base_radix_sort.c b/src/linker/base_ext/base_radix_sort.c index f4c07fab7..e5739ff85 100644 --- a/src/linker/base_ext/base_radix_sort.c +++ b/src/linker/base_ext/base_radix_sort.c @@ -67,16 +67,22 @@ u64_array_sort_radix_parallel(TP_Context *tp, U64 count, U64 *v) tp_for_parallel(tp, 0, task_count, u64_radix_hist_task, &task); U64 cursor = 0; + U64 populated_digit_count = 0; for EachIndex(digit, U64_RADIX_SIZE) { + U64 digit_begin = cursor; for EachIndex(task_idx, task_count) { U64 *slot = &offsets[task_idx * U64_RADIX_SIZE + digit]; U64 slot_count = *slot; *slot = cursor; cursor += slot_count; } + populated_digit_count += (cursor != digit_begin); } Assert(cursor == count); + // A stable radix pass with a single populated digit is the identity transform. + if (populated_digit_count == 1) { continue; } + tp_for_parallel(tp, 0, task_count, u64_radix_scatter_task, &task); Swap(U64 *, task.src, task.dst); } diff --git a/src/linker/codeview_ext/codeview.c b/src/linker/codeview_ext/codeview.c index 11592f7ac..c981210d6 100644 --- a/src/linker/codeview_ext/codeview.c +++ b/src/linker/codeview_ext/codeview.c @@ -559,7 +559,46 @@ cv_obj_info_from_symbol(CV_Symbol symbol) //~ .debug$S helpers internal void -cv_debug_s_from_data_c13_(Arena *arena, CV_DebugS *debug_s, String8 raw_debug_s) +cv_debug_s_prov_list_push_node(CV_DebugSProvList *list, CV_DebugSProvNode *node) +{ + SLLQueuePush(list->first, list->last, node); + list->count += 1; +} + +internal void +cv_debug_s_prov_list_push(Arena *arena, CV_DebugSProvList *list, U64 off, U64 size, U32 sect_idx, B32 is_synthetic) +{ + CV_DebugSProvNode *node = push_array_no_zero(arena, CV_DebugSProvNode, 1); + node->next = 0; + node->off = off; + node->size = size; + node->sect_idx = sect_idx; + node->is_synthetic = is_synthetic; + node->module_symbol_size = 0; + node->gsi_candidate_count = 0; + node->proc_ref_count = 0; + node->symbol_summary_valid = 0; + cv_debug_s_prov_list_push_node(list, node); +} + +internal void +cv_debug_s_prov_list_concat_in_place(CV_DebugSProvList *dst, CV_DebugSProvList *src) +{ + if (src->first != 0) { + if (dst->last != 0) { + dst->last->next = src->first; + dst->last = src->last; + } else { + dst->first = src->first; + dst->last = src->last; + } + dst->count += src->count; + MemoryZeroStruct(src); + } +} + +internal void +cv_debug_s_from_data_c13_(Arena *arena, CV_DebugS *debug_s, String8 raw_debug_s, U64 prov_base_off) { for (U64 cursor = 0; cursor + sizeof(CV_C13SubSectionHeader) <= raw_debug_s.size; ) { // read header @@ -575,6 +614,12 @@ cv_debug_s_from_data_c13_(Arena *arena, CV_DebugS *debug_s, String8 raw_debug_s) Rng1U64 sub_sect_range = r1u64(cursor, cursor + header.size); String8 sub_sect_data = str8_substr(raw_debug_s, sub_sect_range); str8_list_push(arena, sub_sect_list, sub_sect_data); + + // record provenance: slice position within the raw input (sub_sect_data.size, not + // header.size -- str8_substr clamps a header that overruns the section, and provenance + // must describe the bytes the node actually holds) + cv_debug_s_prov_list_push(arena, &debug_s->prov_list[sub_sect_idx], prov_base_off + cursor, sub_sect_data.size, CV_DebugSProvSect_Nil, 0); + Assert(debug_s->prov_list[sub_sect_idx].last->size == sub_sect_data.size); } // advance @@ -595,7 +640,7 @@ internal CV_DebugS cv_debug_s_from_data_c13(Arena *arena, String8 raw_debug_s) { CV_DebugS debug_s = {0}; - cv_debug_s_from_data_c13_(arena, &debug_s, raw_debug_s); + cv_debug_s_from_data_c13_(arena, &debug_s, raw_debug_s, 0); return debug_s; } @@ -608,7 +653,9 @@ cv_debug_s_from_data(Arena *arena, String8 raw_debug_s) switch (sig) { case CV_Signature_C13: { String8 raw_debug_s_past_sig = str8_substr(raw_debug_s, r1u64(sizeof(sig), raw_debug_s.size)); - result = cv_debug_s_from_data_c13(arena, raw_debug_s_past_sig); + // prov base = sizeof(sig): provenance offsets are relative to the raw input handed to + // THIS function, which still carries the signature + cv_debug_s_from_data_c13_(arena, &result, raw_debug_s_past_sig, sizeof(sig)); } break; case CV_Signature_C6: { Assert(!"TODO: handle C6"); @@ -630,6 +677,7 @@ cv_debug_s_concat_in_place(CV_DebugS *dst, CV_DebugS *src) { for (U64 sub_sect_idx = 0; sub_sect_idx < ArrayCount(dst->data_list); sub_sect_idx += 1) { str8_list_concat_in_place(&dst->data_list[sub_sect_idx], &src->data_list[sub_sect_idx]); + cv_debug_s_prov_list_concat_in_place(&dst->prov_list[sub_sect_idx], &src->prov_list[sub_sect_idx]); } } @@ -749,6 +797,67 @@ cv_file_chksms_from_debug_s(CV_DebugS debug_s) return file_chksms; } +internal U64 +cv_total_sub_section_size_from_debug_s(CV_DebugS *debug_s) +{ + U64 total = 0; + for EachElement(i, debug_s->data_list) { total += debug_s->data_list[i].total_size; } + return total; +} + +internal CV_DebugSProvList * +cv_sub_section_prov_ptr_from_debug_s(CV_DebugS *debug_s, CV_C13SubSectionKind kind) +{ + CV_C13SubSectionIdxKind idx = cv_c13_sub_section_idx_from_kind(kind); + return &debug_s->prov_list[idx]; +} + +internal void +cv_debug_s_push_synthetic_sub_section(Arena *arena, CV_DebugS *debug_s, CV_C13SubSectionKind kind, String8 data) +{ + String8List *data_list = cv_sub_section_ptr_from_debug_s(debug_s, kind); + CV_DebugSProvList *prov_list = cv_sub_section_prov_ptr_from_debug_s(debug_s, kind); + Assert(prov_list->count == data_list->node_count); // keep tracked lists tracked + str8_list_push(arena, data_list, data); + cv_debug_s_prov_list_push(arena, prov_list, 0, data.size, CV_DebugSProvSect_Nil, 1); +} + +internal void +cv_debug_s_concat_sub_section_in_place(CV_DebugS *dst, CV_DebugS *src, CV_C13SubSectionKind kind) +{ + CV_C13SubSectionIdxKind idx = cv_c13_sub_section_idx_from_kind(kind); + str8_list_concat_in_place(&dst->data_list[idx], &src->data_list[idx]); + cv_debug_s_prov_list_concat_in_place(&dst->prov_list[idx], &src->prov_list[idx]); +} + +internal void +cv_debug_s_tag_prov_sect(CV_DebugS *debug_s, U32 sect_idx) +{ + for EachElement(i, debug_s->prov_list) { + for EachNode(n, CV_DebugSProvNode, debug_s->prov_list[i].first) { n->sect_idx = sect_idx; } + } +} + +// debug-only invariant walk: per subsection, provenance is either absent (untracked +// synthetic construction) or one node per data node with matching sizes +internal void +cv_debug_s_validate_prov(CV_DebugS *debug_s) +{ +#if BUILD_DEBUG + for EachElement(i, debug_s->data_list) { + if (debug_s->prov_list[i].count == 0) { Assert(debug_s->prov_list[i].first == 0); continue; } + Assert(debug_s->prov_list[i].count == debug_s->data_list[i].node_count); + CV_DebugSProvNode *prov_n = debug_s->prov_list[i].first; + for EachNode(data_n, String8Node, debug_s->data_list[i].first) { + Assert(prov_n != 0); + if (!prov_n->is_synthetic) { Assert(prov_n->size == data_n->string.size); } + prov_n = prov_n->next; + } + Assert(prov_n == 0); + } +#endif +} + //////////////////////////////// //~ String Table Deduper @@ -1179,13 +1288,32 @@ cv_debug_t_from_data(Arena *arena, String8 data, U64 align) return debug_t; } +internal U64 +cv_debug_t_get_leaf_offset(CV_DebugT *debug_t, U64 leaf_idx) +{ + Assert(leaf_idx < debug_t->count); + U64 actual_idx = leaf_idx + debug_t->sidecar_leaf_bias; + if (debug_t->sidecar_packed) { + U64 group_idx = actual_idx >> debug_t->sidecar_offset_checkpoint_shift; + U64 within = actual_idx & (((U64)1 << debug_t->sidecar_offset_checkpoint_shift) - 1); + U32 base = debug_t->sidecar_packed_v2_offset_groups[group_idx*2 + 0]; + U32 descriptor = debug_t->sidecar_packed_v2_offset_groups[group_idx*2 + 1]; + U32 width = (descriptor & 1) ? 3 : 2; + U8 *p = debug_t->sidecar_packed_v2_offset_payload + (descriptor & ~(U32)1) + within * width; + U32 delta = (U32)p[0] | ((U32)p[1] << 8); + if (width == 3) { delta |= (U32)p[2] << 16; } + return (U64)base + delta; + } + return debug_t->offsets[actual_idx]; +} + internal CV_Leaf cv_debug_t_get_leaf(CV_DebugT *debug_t, U64 leaf_idx) { CV_Leaf leaf = {0}; if (debug_t->count > 0) { Assert(leaf_idx < debug_t->count); - cv_read_leaf(debug_t->data, debug_t->offsets[leaf_idx], 1, &leaf); + cv_read_leaf(debug_t->data, cv_debug_t_get_leaf_offset(debug_t, leaf_idx), 1, &leaf); Assert(cv_header_struct_size_from_leaf_kind(leaf.kind) <= leaf.data.size); } return leaf; @@ -1234,9 +1362,9 @@ internal String8 cv_debug_t_get_raw_leaf(CV_DebugT *debug_t, U64 leaf_idx) { Assert(leaf_idx < debug_t->count); - U8 *leaf_ptr = debug_t->data.str + debug_t->offsets[leaf_idx]; - CV_LeafSize leaf_size = memory_read16(leaf_ptr); - return str8(leaf_ptr, leaf_size + sizeof(leaf_size)); + U8 *leaf_ptr = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + U64 leaf_size = cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + return str8(leaf_ptr, leaf_size); } internal CV_LeafHeader * @@ -1244,11 +1372,40 @@ cv_debug_t_get_leaf_header(CV_DebugT *debug_t, U64 leaf_idx) { CV_LeafHeader *header = 0; if (leaf_idx < debug_t->count) { - header = (CV_LeafHeader *)(debug_t->data.str + debug_t->offsets[leaf_idx]); + header = (CV_LeafHeader *)(debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx)); } return header; } +internal CV_LeafKind +cv_debug_t_get_leaf_kind(CV_DebugT *debug_t, U64 leaf_idx) +{ + Assert(leaf_idx < debug_t->count); + U64 actual_idx = leaf_idx + debug_t->sidecar_leaf_bias; + if (debug_t->sidecar_packed_kind_codes) { + return debug_t->sidecar_packed_kind_dictionary[debug_t->sidecar_packed_kind_codes[actual_idx]]; + } + if (debug_t->sidecar_kinds) { return debug_t->sidecar_kinds[actual_idx]; } + return cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind; +} + +internal U64 +cv_debug_t_get_raw_leaf_size(CV_DebugT *debug_t, U64 leaf_idx) +{ + Assert(leaf_idx < debug_t->count); + U64 actual_idx = leaf_idx + debug_t->sidecar_leaf_bias; + if (debug_t->sidecar_packed) { + U64 offset = cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + U64 total_count = debug_t->count + debug_t->sidecar_leaf_bias; + U64 next_offset = actual_idx + 1 < total_count ? cv_debug_t_get_leaf_offset(debug_t, leaf_idx + 1) : debug_t->data.size; + Assert(next_offset >= offset); + return next_offset - offset; + } + if (debug_t->sidecar_sizes) { return (U64)debug_t->sidecar_sizes[actual_idx] + sizeof(CV_LeafSize); } + U8 *leaf_ptr = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + return (U64)memory_read16(leaf_ptr) + sizeof(CV_LeafSize); +} + internal CV_TypeIndex cv_debug_t_get_type_index(CV_DebugT *debug_t, CV_TypeIndexSource ti_source, U64 leaf_idx) { @@ -1268,13 +1425,13 @@ cv_debug_t_get_leaf_index(CV_DebugT *debug_t, CV_TypeIndexSource ti_source, CV_T internal B32 cv_debug_t_is_pch(CV_DebugT *debug_t) { - return cv_is_leaf_pch(cv_debug_t_get_leaf(debug_t, 0).kind); + return debug_t->count && cv_is_leaf_pch(cv_debug_t_get_leaf_kind(debug_t, 0)); } internal B32 cv_debug_t_is_type_server_ref(CV_DebugT *debug_t) { - return cv_is_leaf_type_server(cv_debug_t_get_leaf(debug_t, 0).kind); + return debug_t->count && cv_is_leaf_type_server(cv_debug_t_get_leaf_kind(debug_t, 0)); } // $$Symbols diff --git a/src/linker/codeview_ext/codeview.h b/src/linker/codeview_ext/codeview.h index f3e5ae9fd..22a42c830 100644 --- a/src/linker/codeview_ext/codeview.h +++ b/src/linker/codeview_ext/codeview.h @@ -216,9 +216,43 @@ typedef struct CV_C13SubSectionList //////////////////////////////// +// Provenance for parsed .debug$S subsection nodes (streaming-ring P1 groundwork). +// One node per data_list node, in lockstep order; records where the slice came from so a +// later representation swap (String8 slices -> offset records) has the origin on hand. +// DORMANT: nothing reads this yet; it must merely stay correct. +// - off/size: slice position within the raw input String8 handed to cv_debug_s_from_data +// (offsets include the leading CV_Signature when the input carried one) +// - sect_idx: 0-based COFF section index the raw input was sliced from; +// CV_DebugSProvSect_Nil until the caller tags it (cv_debug_s_tag_prov_sect) +// - is_synthetic: bytes were synthesized post-parse (not sliced from any input section); +// off/size/sect_idx are meaningless +// A CV_DebugS built without the parser (wholesale list assignment) may leave prov_list +// empty: prov_list[i].count is either 0 (untracked) or data_list[i].node_count (tracked). +#define CV_DebugSProvSect_Nil max_U32 +typedef struct CV_DebugSProvNode +{ + struct CV_DebugSProvNode *next; + U64 off; + U64 size; + U32 sect_idx; + B32 is_synthetic; + U32 module_symbol_size; + U32 gsi_candidate_count; + U32 proc_ref_count; + B32 symbol_summary_valid; +} CV_DebugSProvNode; + +typedef struct CV_DebugSProvList +{ + CV_DebugSProvNode *first; + CV_DebugSProvNode *last; + U64 count; +} CV_DebugSProvList; + typedef struct CV_DebugS { - String8List data_list[CV_C13SubSectionIdxKind_COUNT]; + String8List data_list[CV_C13SubSectionIdxKind_COUNT]; + CV_DebugSProvList prov_list[CV_C13SubSectionIdxKind_COUNT]; // parallel to data_list (see above) } CV_DebugS; typedef struct CV_DebugH @@ -232,6 +266,20 @@ typedef struct CV_DebugT String8 data; U64 count; U32 *offsets; + // Optional container sidecar. Keeping these as mapped arrays avoids both + // scanning compressed leaf bodies and privately allocating the index. + U16 *sidecar_sizes; + U16 *sidecar_kinds; + U32 *sidecar_packed_v2_offset_groups; + U8 *sidecar_packed_v2_offset_payload; + U16 *sidecar_packed_kind_dictionary; + U8 *sidecar_packed_kind_codes; + B8 sidecar_packed; + U8 sidecar_offset_checkpoint_shift; + U32 sidecar_leaf_bias; + U64 sidecar_raw_base; + U64 *sidecar_complete_udt_hashes; + U64 sidecar_complete_udt_hash_count; // type server U64 source_counts [CV_TypeIndexSource_COUNT]; @@ -390,6 +438,13 @@ internal String8List * cv_sub_section_ptr_from_debug_s(CV_DebugS *debug_s, CV_C1 internal String8List cv_sub_section_from_debug_s(CV_DebugS debug_s, CV_C13SubSectionKind kind); internal String8 cv_string_table_from_debug_s(CV_DebugS debug_s); internal String8 cv_file_chksms_from_debug_s(CV_DebugS debug_s); +internal U64 cv_total_sub_section_size_from_debug_s(CV_DebugS *debug_s); + +internal CV_DebugSProvList * cv_sub_section_prov_ptr_from_debug_s(CV_DebugS *debug_s, CV_C13SubSectionKind kind); +internal void cv_debug_s_push_synthetic_sub_section(Arena *arena, CV_DebugS *debug_s, CV_C13SubSectionKind kind, String8 data); +internal void cv_debug_s_concat_sub_section_in_place(CV_DebugS *dst, CV_DebugS *src, CV_C13SubSectionKind kind); +internal void cv_debug_s_tag_prov_sect(CV_DebugS *debug_s, U32 sect_idx); +internal void cv_debug_s_validate_prov(CV_DebugS *debug_s); //////////////////////////////// //~ .debug$T helpers @@ -398,9 +453,12 @@ internal CV_DebugT cv_debug_t_from_data (Arena *arena, String8 dat internal U64 cv_leaf_idx_from_ti (CV_DebugT *debug_t, CV_TypeIndexSource source, CV_TypeIndex ti); internal CV_TypeIndex cv_ti_from_leaf_idx (CV_DebugT *debug_t, CV_TypeIndexSource source, U64 leaf_idx); internal CV_Leaf cv_debug_t_get_leaf (CV_DebugT *debug_t, U64 leaf_idx); +internal U64 cv_debug_t_get_leaf_offset (CV_DebugT *debug_t, U64 leaf_idx); internal CV_Leaf cv_debug_t_get_leaf_from_ti (CV_DebugT *debug_t, CV_TypeIndexSource source, CV_TypeIndex ti); internal String8 cv_debug_t_get_raw_leaf (CV_DebugT *debug_t, U64 leaf_idx); internal CV_LeafHeader * cv_debug_t_get_leaf_header (CV_DebugT *debug_t, U64 leaf_idx); +internal CV_LeafKind cv_debug_t_get_leaf_kind (CV_DebugT *debug_t, U64 leaf_idx); +internal U64 cv_debug_t_get_raw_leaf_size (CV_DebugT *debug_t, U64 leaf_idx); internal CV_TypeIndex cv_debug_t_get_type_index (CV_DebugT *debug_t, CV_TypeIndexSource ti_source, U64 leaf_idx); internal U64 cv_debug_t_get_leaf_index (CV_DebugT *debug_t, CV_TypeIndexSource ti_source, CV_TypeIndex ti); internal B32 cv_debug_t_is_pch (CV_DebugT *debug_t); diff --git a/src/linker/codeview_ext/ifc.c b/src/linker/codeview_ext/ifc.c new file mode 100644 index 000000000..b18cc4b5b --- /dev/null +++ b/src/linker/codeview_ext/ifc.c @@ -0,0 +1,112 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +read_only global U8 g_ifc_signature[4] = { 0x54, 0x51, 0x45, 0x1A }; +read_only global U8 g_uba_signature[4] = { 0x55, 0x42, 0x41, 0x01 }; // "UBA\x01" + +internal IFC_File +ifc_file_read(Arena *arena, String8 path, String8 *error_out) +{ + IFC_File ifc = {0}; + ifc.path = push_str8_copy(arena, path); + + B8 was_read = 0; + String8 data = lnk_read_data_from_file_path(arena, 0, path, &was_read); + ifc.data = data; + if ( ! was_read) { + *error_out = push_str8f(arena, "unable to read IFC '%S'", path); + return ifc; + } + if (data.size < 4) { + *error_out = push_str8f(arena, "IFC '%S' is too small (%llu bytes)", path, data.size); + return ifc; + } + + // detect UBA-compressed input (out of scope) + if (MemoryMatch(data.str, g_uba_signature, sizeof(g_uba_signature))) { + *error_out = push_str8f(arena, "IFC '%S' is UBA-compressed (magic 'UBA\\x01'); materialize a raw .ifc (UBA decompress unsupported)", path); + return ifc; + } + + // validate signature + if ( ! MemoryMatch(data.str, g_ifc_signature, sizeof(g_ifc_signature))) { + *error_out = push_str8f(arena, "IFC '%S' has bad signature (expected 54 51 45 1A)", path); + return ifc; + } + + // --- parse header --- + U64 off = 4; + if (off + 32 > data.size) { goto truncated; } + MemoryCopy(ifc.content_hash, data.str + off, 32); + off += 32; + + if (off + 4 > data.size) { goto truncated; } + U8 major = data.str[off+0]; + U8 minor = data.str[off+1]; + U8 abi = data.str[off+2]; (void)abi; + U8 arch = data.str[off+3]; + off += 4; + + // assert version 0.44 + x64; error otherwise (encoding proven only for these) + if ( ! (major == 0 && minor == 44)) { + *error_out = push_str8f(arena, "IFC '%S' unsupported version %u.%u (expected 0.44)", path, major, minor); + return ifc; + } + if (arch != 2) { + *error_out = push_str8f(arena, "IFC '%S' unsupported architecture %u (expected 2 == x64)", path, arch); + return ifc; + } + + U32 cplusplus; off += str8_deserial_read_struct(data, off, &cplusplus); (void)cplusplus; + U32 string_table_bytes; off += str8_deserial_read_struct(data, off, &string_table_bytes); + U32 string_table_size; off += str8_deserial_read_struct(data, off, &string_table_size); + U32 unit; off += str8_deserial_read_struct(data, off, &unit); (void)unit; + U32 src_path; off += str8_deserial_read_struct(data, off, &src_path); (void)src_path; + U32 global_scope; off += str8_deserial_read_struct(data, off, &global_scope); (void)global_scope; + U32 toc; off += str8_deserial_read_struct(data, off, &toc); + U32 partition_count; off += str8_deserial_read_struct(data, off, &partition_count); + if (off > data.size) { goto truncated; } + + // string table + if ((U64)string_table_bytes + string_table_size > data.size) { + *error_out = push_str8f(arena, "IFC '%S' string table out of bounds", path); + return ifc; + } + String8 string_table = str8(data.str + string_table_bytes, string_table_size); + + // --- partition summary table --- + String8 needle = str8_lit(".msvc.trait.debug-records"); + U64 po = toc; + for (U32 i = 0; i < partition_count; ++i, po += 16) { + if (po + 16 > data.size) { goto truncated; } + U32 name_off, p_off, count, entity_size; + str8_deserial_read_struct(data, po + 0, &name_off); + str8_deserial_read_struct(data, po + 4, &p_off); + str8_deserial_read_struct(data, po + 8, &count); + str8_deserial_read_struct(data, po + 12, &entity_size); + + if (name_off >= string_table.size) { continue; } + String8 name = str8_cstring((char *)string_table.str + name_off); + if (str8_match(name, needle, 0)) { + // entity_size == 1 -> count is a byte length + if ((U64)p_off + count > data.size) { + *error_out = push_str8f(arena, "IFC '%S' debug-records partition out of bounds", path); + return ifc; + } + ifc.debug_records = str8(data.str + p_off, count); + break; + } + } + + if (ifc.debug_records.size == 0) { + *error_out = push_str8f(arena, "IFC '%S' has no '.msvc.trait.debug-records' partition", path); + return ifc; + } + + ifc.is_valid = 1; + return ifc; + +truncated: + *error_out = push_str8f(arena, "IFC '%S' is truncated", path); + return ifc; +} diff --git a/src/linker/codeview_ext/ifc.h b/src/linker/codeview_ext/ifc.h new file mode 100644 index 000000000..fb3169b75 --- /dev/null +++ b/src/linker/codeview_ext/ifc.h @@ -0,0 +1,43 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +//////////////////////////////// +// MSVC IFC (header-unit module interface) reader +// +// radlink consumes the `.msvc.trait.debug-records` partition embedded in an +// MSVC `.ifc` (C++20 module / header-unit interface) file. That partition is a +// raw CodeView type-leaf stream (entity_size == 1, NO u32 signature, first leaf +// at offset 0, TI base 0x1000). A consuming `.obj` references this stream via +// LF_IFC_RECORD (0x1522) leaves -- see lnk_debug_info.c. +// +// File layout (microsoft/ifc-spec, FileHeader): +// u8[4] signature = { 0x54,0x51,0x45,0x1A } ("TQE\x1a") +// u8[32] content_hash (sha256) -- record.GUID(16)++record.hash(16) == first 32 bytes here +// u8 major, minor -- assert 0.44 +// u8 abi +// u8 arch -- 2 == x64 +// u32 cplusplus +// u32 string_table_bytes (off), u32 string_table_size +// u32 unit +// u32 src_path (textoffset) +// u32 global_scope +// u32 toc -- offset to partition summary table +// u32 partition_count +// u8 internal_partition +// Partition summary entry (16 bytes): { u32 name(textoffset); u32 offset; u32 count; u32 entity_size } + +typedef struct IFC_File +{ + String8 data; // whole .ifc bytes (owning view into arena) + String8 path; // .ifc path (copied) + U8 content_hash[32]; + String8 debug_records; // .msvc.trait.debug-records blob {ptr,size}; size 0 if absent + B32 is_valid; +} IFC_File; + +// Reads `path`, validates magic/version/arch, locates `.msvc.trait.debug-records`. +// On error fills *error_out and returns is_valid=0. Detects UBA-compressed inputs +// (magic "UBA\x01") and reports them (decompression is out of scope). +internal IFC_File ifc_file_read(Arena *arena, String8 path, String8 *error_out); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index dbf4f577e..ead725ee4 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -54,6 +54,10 @@ #include "llvm/llvm.c" #include "dwarf/x64/dwarf_x64.c" +#if OS_WINDOWS +# include // GetProcessMemoryInfo for the end-of-link summary line +#endif + // --- Third Party ------------------------------------------------------------- #include "base_ext/base_blake3.h" @@ -114,13 +118,16 @@ #include "lnk_debug_helper.h" #include "lnk_obj.h" #include "lnk_lib.h" +#include "codeview_ext/ifc.h" #include "lnk_debug_info.h" +#include "lnk_compressed_obj.h" #include "lnk.h" #include "lnk_log.c" #include "lnk_timer.c" #include "lnk_hasher.c" #include "lnk_io.c" +#include "lnk_compressed_obj.c" #include "lnk_cmd_line.c" #include "lnk_config.c" #include "lnk_symbol_table.c" @@ -128,6 +135,7 @@ #include "lnk_obj.c" #include "lnk_debug_helper.c" #include "lnk_lib.c" +#include "codeview_ext/ifc.c" #include "lnk_debug_info.c" // ----------------------------------------------------------------------------- @@ -984,6 +992,49 @@ lnk_inputer_has_items(LNK_Inputer *inputer) return 0; } +typedef struct LNK_InputOpenTask +{ + LNK_Input **inputs; + String8 *datas; +} LNK_InputOpenTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_input_open_task) +{ + LNK_InputOpenTask *task = raw_task; + LNK_Input *input = task->inputs[task_id]; + String8 data = task->datas[task_id]; + if (!input->has_disk_read_failed) { + lnk_compressed_obj_open(input, data); + } +} + +typedef struct LNK_InputClassifyTask +{ + String8 *datas; + U8 *tags; +} LNK_InputClassifyTask; + +enum +{ + LNK_InputClassify_PortableCompressed = (1 << 0), +}; + +internal +THREAD_POOL_TASK_FUNC(lnk_input_classify_task) +{ + LNK_InputClassifyTask *task = raw_task; + String8 data = task->datas[task_id]; + + U8 tag = 0; + if (data.size >= sizeof(LNK_CObjHeader) && ((LNK_CObjHeader *)data.str)->magic == LNK_COBJ_MAGIC) { + if (((LNK_CObjHeader *)data.str)->flags & LNK_COBJ_FLAG_PORTABLE_RAW_MAP) { + tag |= LNK_InputClassify_PortableCompressed; + } + } + task->tags[task_id] = tag; +} + internal LNK_InputPtrArray lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Flags io_flags, LNK_InputList *all_inputs, LNK_InputList *new_inputs) { @@ -1014,7 +1065,6 @@ lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Fla ProfEnd(); ProfBegin("Load Inputs From Disk"); - B8 *thin_input_was_read = push_array(scratch.arena, B8, thin_input_paths.count); String8Array thin_input_datas = lnk_read_data_from_file_path_parallel(tp, inputer->arena, io_flags, thin_input_paths, thin_input_was_read); B32 is_mapped = !!(io_flags & (LNK_IO_Flags_MemoryMapFilesReadWrite|LNK_IO_Flags_MemoryMapFilesReadOnly)); @@ -1025,6 +1075,30 @@ lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Fla thin_inputs[thin_input_idx]->data = thin_input_datas.v[thin_input_idx]; } + // Keep the ordinary-object path as direct mmap followed by cheap first-page classification. + // Portable compressed objects have independent metadata and can be opened in parallel. + B32 has_portable_cobj = 0; + U8 *classify_tags = push_array_no_zero(scratch.arena, U8, thin_inputs_count); + LNK_InputClassifyTask classify_task = {thin_input_datas.v, classify_tags}; + if (thin_inputs_count >= 64) { + lnk_tp_for_parallel_capped(tp, 0, 32, thin_inputs_count, lnk_input_classify_task, &classify_task); + } else { + for EachIndex(i, thin_inputs_count) { lnk_input_classify_task(0, 0, i, &classify_task, tp); } + } + for EachIndex(thin_input_idx, thin_inputs_count) { + has_portable_cobj |= !!(classify_tags[thin_input_idx] & LNK_InputClassify_PortableCompressed); + } + if (has_portable_cobj) { + lnk_compressed_obj_prepare_cache(thin_input_datas.v, thin_input_datas.count); + LNK_InputOpenTask open_task = {thin_inputs, thin_input_datas.v}; + // Reserving/splitting thousands of logical OBJ views contends on the Windows process VAD + // lock. Four lanes measured substantially faster than 8-64 on the UEFN corpus. + lnk_tp_for_parallel_capped(tp, 0, 4, thin_inputs_count, lnk_input_open_task, &open_task); + } + // Compressed inputs register their reserved address regions independently while the open tasks + // run. Sort once after all tasks have joined; sorting after every insertion serialized the + // parallel open and performed thousands of increasingly large qsorts. + lnk_compressed_obj_finalize_open(); ProfEnd(); ProfBegin("Disk Read Check"); @@ -1049,13 +1123,17 @@ THREAD_POOL_TASK_FUNC(lnk_release_file_map_task) { LNK_Input **mapped_inputs = raw_task; LNK_Input *input = mapped_inputs[task_id]; - file_map_view_close((FileMap){0}, input->data.str, r1u64(0, input->data.size)); - input->data = str8_zero(); - input->owns_file_map = 0; + if (input->compressed_obj) { + lnk_compressed_obj_close(input); + } else { + file_map_view_close((FileMap){0}, input->data.str, r1u64(0, input->data.size)); + input->data = str8_zero(); + input->owns_file_map = 0; + } } internal void -lnk_inputer_release_file_maps(TP_Context *tp, LNK_Inputer *inputer) +lnk_inputer_release_file_maps(TP_Context *tp, U64 worker_cap, LNK_Inputer *inputer) { Temp scratch = scratch_begin(0, 0); @@ -1081,7 +1159,12 @@ lnk_inputer_release_file_maps(TP_Context *tp, LNK_Inputer *inputer) } Assert(mapped_input_idx == mapped_input_count); - tp_for_parallel(tp, 0, mapped_input_count, lnk_release_file_map_task, mapped_inputs); + // UnmapViewOfFile serializes on the process address-space lock: with the + // full pool every worker camps in the kernel for the whole pass (measured + // ~257s of kernel CPU for a ~4.1s wall pass at 64 workers on the FN editor + // DLL). Cap the lanes like the other fault/VAD-bound stages -- same wall, + // a fraction of the spin. + lnk_tp_for_parallel_capped(tp, 0, worker_cap, mapped_input_count, lnk_release_file_map_task, mapped_inputs); scratch_end(scratch); } @@ -1222,6 +1305,7 @@ internal void lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer *inputer, LNK_SymbolTable *symtab, LNK_Link *link) { ProfBeginFunction(); + lnk_summary_phase_begin(LNK_SummaryPhase_Input); Temp scratch = scratch_begin(arena->v, arena->count); U64 obj_id_base = link->objs.count; @@ -1310,11 +1394,11 @@ lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer { B32 has_function_overrides = 0; - for EachNode(alt_name_n, LNK_AltNameNode, config->alt_name_list.first) { + // Ordinary alternate names can number in the millions. Classify overrides + // once when parsing directives, but revisit the override subset here: a + // later archive member may introduce a reference to an earlier directive. + for EachNode(alt_name_n, LNK_AltNameNode, config->function_override_list.first) { LNK_AltName alt_name = alt_name_n->v; - if ( ! str8_ends_with(alt_name.from, str8_lit("$fo$"), 0)) { - continue; - } LNK_Symbol *symbol = lnk_symbol_table_search(symtab, alt_name.from); if (symbol && lnk_interp_from_symbol(symbol) == COFF_SymbolValueInterp_Undefined) { @@ -1443,6 +1527,7 @@ lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer } scratch_end(scratch); + lnk_summary_phase_end(LNK_SummaryPhase_Input); ProfEnd(); } @@ -1532,15 +1617,8 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) LNK_LibMemberInfo *lib_member_infos = task->lib_member_infos; LNK_LibMemberRefList *member_ref_list = &task->member_ref_lists[task_id]; - LNK_SymbolHashTrieChunk *start_chunk = task->reset_search_cursor ? 0 : lib->search_cursor_chunks[task_id]; - U64 start_idx = task->reset_search_cursor ? 0 : lib->search_cursor_indices[task_id]; - LNK_SymbolHashTrieChunk *end_chunk = symtab->search_chunks[task_id].last; - U64 end_count = end_chunk ? end_chunk->count : 0; - - for EachNode(c, LNK_SymbolHashTrieChunk, start_chunk ? start_chunk : symtab->search_chunks[task_id].first) { - U64 i_begin = (c == start_chunk) ? start_idx : 0; - U64 i_end = (c == end_chunk) ? end_count : c->count; - for (U64 i = i_begin; i < i_end; i += 1) { + for EachNode(c, LNK_SymbolHashTrieChunk, symtab->search_chunks[task_id].first) { + for EachIndex(i, c->count) { LNK_Symbol *symbol = c->v[i].symbol; LNK_SymbolSearchType search_type = lnk_search_type_from_symbol(symbol); @@ -1565,27 +1643,15 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) } } } - - // cache search cursors - lib->search_cursor_chunks[task_id] = end_chunk; - lib->search_cursor_indices[task_id] = end_count; } internal U64 lnk_search_lib_task_work_count(LNK_SearchLibTask *task, U64 task_id) { - LNK_Lib *lib = task->lib; LNK_SymbolTable *symtab = task->symtab; - LNK_SymbolHashTrieChunk *start_chunk = task->reset_search_cursor ? 0 : lib->search_cursor_chunks[task_id]; - U64 start_idx = task->reset_search_cursor ? 0 : lib->search_cursor_indices[task_id]; - LNK_SymbolHashTrieChunk *end_chunk = symtab->search_chunks[task_id].last; - U64 end_count = end_chunk ? end_chunk->count : 0; - U64 work_count = 0; - for EachNode(c, LNK_SymbolHashTrieChunk, start_chunk ? start_chunk : symtab->search_chunks[task_id].first) { - U64 i_begin = (c == start_chunk) ? start_idx : 0; - U64 i_end = (c == end_chunk) ? end_count : c->count; - work_count += i_end - i_begin; + for EachNode(c, LNK_SymbolHashTrieChunk, symtab->search_chunks[task_id].first) { + work_count += c->count; } return work_count; } @@ -1635,6 +1701,14 @@ lnk_link_inputs(TP_Context *tp, ProfBeginFunction(); Temp scratch = scratch_begin(arena->v, arena->count); + // summary: this function is Input (lnk_load_inputs rounds, accumulated on its + // own bucket) + Resolve (lib search / member resolution / directives, i.e. + // everything else); attribute the remainder to Resolve at the bottom. The + // same subtraction works for every counter: they are monotonic and the Input + // brackets nest strictly inside this window + LNK_SummaryCounters summary_begin = lnk_summary_counters_now(); + LNK_SummaryCounters summary_input_at = g_summary_phase[LNK_SummaryPhase_Input]; + HashMap imports_hm = {0}; LNK_LibMemberRefList *member_ref_lists = push_array(scratch.arena, LNK_LibMemberRefList, tp->worker_count); @@ -1728,18 +1802,8 @@ lnk_link_inputs(TP_Context *tp, } else { // search symbols in lib MemoryZeroTyped(member_ref_lists, tp->worker_count); - // anti-dep mode changes which weak symbols can resolve from this lib - B32 reset_search_cursor = lib->search_cursor_chunks != 0 && lib->searched_anti_deps != search_anti_deps; - - // lazy alloc cursors for tracking searched symbols - if (lib->search_cursor_chunks == 0) { - lib->search_cursor_chunks = push_array(link->arena, LNK_SymbolHashTrieChunk *, tp->worker_count); - lib->search_cursor_indices = push_array(link->arena, U64, tp->worker_count); - } - LNK_SearchLibTask search_task = { .search_anti_deps = search_anti_deps, - .reset_search_cursor = reset_search_cursor, .link = link, .imports_hm = &imports_hm, .lib = lib, @@ -1764,9 +1828,6 @@ lnk_link_inputs(TP_Context *tp, } else { tp_for_parallel(tp, arena, tp->worker_count, lnk_search_lib_task, &search_task); } - - // cache last search mode, if the mode changes then skipped weak anti-dependency must be searched again - lib->searched_anti_deps = search_anti_deps; } LNK_LibMemberRefList queued_members = {0}; @@ -1917,6 +1978,17 @@ lnk_link_inputs(TP_Context *tp, if (resolved_members_count == 0) { break; } } + { + LNK_SummaryCounters now = lnk_summary_counters_now(); + LNK_SummaryCounters window = lnk_summary_counters_sub_sat(now, summary_begin); + LNK_SummaryCounters input_delta = lnk_summary_counters_sub_sat(g_summary_phase[LNK_SummaryPhase_Input], summary_input_at); + LNK_SummaryCounters resolve = lnk_summary_counters_sub_sat(window, input_delta); + g_summary_phase[LNK_SummaryPhase_Resolve].wall_us += resolve.wall_us; + g_summary_phase[LNK_SummaryPhase_Resolve].user_us += resolve.user_us; + g_summary_phase[LNK_SummaryPhase_Resolve].kern_us += resolve.kern_us; + g_summary_phase[LNK_SummaryPhase_Resolve].faults += resolve.faults; + } + scratch_end(scratch); ProfEnd(); } @@ -2238,6 +2310,15 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer for EachIndex(i, entry_points.count) { str8_list_push(scratch.arena, &list, entry_points.v[i]); } String8 default_entries = str8_list_join(scratch.arena, &list, &(StringJoin){.sep = str8_lit(", ")}); + StringJoin comma_join = {.sep = str8_lit(", ")}; + String8 obj_default_libs = str8_list_join(scratch.arena, &config->input_obj_lib_list, &comma_join); + String8 cmd_default_libs = str8_list_join(scratch.arena, &config->input_default_lib_list, &comma_join); + String8List loaded_lib_list = {0}; + for EachNode(lib_n, LNK_LibNode, link->libs.first) { + str8_list_push(scratch.arena, &loaded_lib_list, str8_skip_last_slash(lib_n->data.path)); + } + String8 loaded_libs = str8_list_join(scratch.arena, &loaded_lib_list, &comma_join); + lnk_error(LNK_Error_EntryPoint, "failed to infer entry point symbol from the inputs\n" " Machine: %S\n" @@ -2248,7 +2329,10 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer " Default Entries: %S\n" " User Entry: \"%S\"\n" " Input Obj Count: %S\n" - " Input Lib Count: %S", + " Input Lib Count: %S\n" + " Obj Default Libs: %S\n" + " Cmd Default Libs: %S\n" + " Loaded Libs: %S", machine_str.size ? machine_str : str8_lit("Unknown"), // Machine subsystem_str.size ? subsystem_str : str8_lit("Unknown"), // Version config->subsystem_ver.major, config->subsystem_ver.minor, // Subsystem @@ -2257,7 +2341,10 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer default_entries.size ? default_entries : str8_lit("None"), // Default Entry Points config->entry_point_name, // /ENTRY str8_from_count(scratch.arena, config->input_list[LNK_Input_Obj].node_count), // Input Objects Count - str8_from_count(scratch.arena, config->input_list[LNK_Input_Lib].node_count) // Input Libs Count + str8_from_count(scratch.arena, config->input_list[LNK_Input_Lib].node_count), // Input Libs Count + obj_default_libs.size ? obj_default_libs : str8_lit("None"), + cmd_default_libs.size ? cmd_default_libs : str8_lit("None"), + loaded_libs.size ? loaded_libs : str8_lit("None") ); } @@ -2420,6 +2507,10 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer next_undefined_symbol:; lnk_error(LNK_Error_UnresolvedSymbol, "unresolved symbol '%S'", symbol->name); + if (str8_match(str8_prefix(symbol->name, 6), str8_lit("__imp_"), 0)) { + lnk_supplement_error("this is a DLL import, but no linked object or import library defines it"); + lnk_supplement_error("verify that the response file includes the import library for this module dependency"); + } lnk_supplement_error_list(ref_messages); } @@ -2441,7 +2532,9 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer // if (config->opt_ref == LNK_SwitchState_Yes) { if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_summary_phase_begin(LNK_SummaryPhase_Ref); lnk_opt_ref(tp, symtab, config, objs, link->objs.count); + lnk_summary_phase_end(LNK_SummaryPhase_Ref); } // @@ -2449,7 +2542,17 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer // if (config->opt_icf == LNK_SwitchState_Yes) { if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } - lnk_opt_icf(tp, symtab, config, objs, link->objs.count); + lnk_summary_phase_begin(LNK_SummaryPhase_Icf); + lnk_opt_icf(tp, arena->v[0], symtab, config, objs, link->objs.count); + lnk_summary_phase_end(LNK_SummaryPhase_Icf); + } + + // + // keep line info for ICF-folded functions (bound to the leader RVA) -- see task comment + // + if (config->opt_icf == LNK_SwitchState_Yes && config->opt_ref == LNK_SwitchState_Yes) { + if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_icf_mark_folded_lines(tp, arena, objs, link->objs.count); } } @@ -2647,20 +2750,29 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) LNK_Obj **objs = task->objs; U64 objs_count = task->objs_count; - U8 **is_live = 0; - U64 *active_thread_count = 0; + // "Remove Unreachable Sections" per-task stat accumulators (reduced on task 0 for the log) + typedef struct { U64 vsize; U64 fsize; U64 section_count; } LNK_OptRefStat; + enum { LNK_OptRefStat_Null, LNK_OptRefStat_Code, LNK_OptRefStat_Data, LNK_OptRefStat_Debug, LNK_OptRefStat_Count }; + + U8 **is_live = 0; + LNK_Obj **objs_by_idx = 0; // input_idx -> obj, for the strided removal pass + U64 *active_thread_count = 0; LNK_RelocRefsBatchList *global_batch_list = 0; + LNK_OptRefStat *remove_stats = 0; if (task_id == 0) { - active_thread_count = push_array(scratch.arena, U64, 1); + remove_stats = push_array(scratch.arena, LNK_OptRefStat, LNK_OptRefStat_Count * tp->worker_count); + active_thread_count = push_array(scratch.arena, U64, 1); global_batch_list = push_array(scratch.arena, LNK_RelocRefsBatchList, 1); // alloc live flags and set live status on every non-COMDAT section - is_live = push_array_no_zero(scratch.arena, U8 *, objs_count); + is_live = push_array_no_zero(scratch.arena, U8 *, objs_count); + objs_by_idx = push_array_no_zero(scratch.arena, LNK_Obj *, objs_count ? objs_count : 1); { for EachIndex(obj_idx, objs_count) { LNK_Obj *obj = objs[obj_idx]; - is_live[obj_idx] = push_array(scratch.arena, U8, obj->coff.sections.count_no_null + 1); + is_live[obj_idx] = push_array(scratch.arena, U8, obj->coff.sections.count_no_null + 1); + objs_by_idx[obj_idx] = obj; for LNK_EachCoffSection(it, obj) { is_live[obj_idx][it.v.section_number] = !(*it.v.flags & COFF_SectionFlag_LnkCOMDAT); @@ -2720,8 +2832,25 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) } } tp_broadcast(&is_live); + tp_broadcast(&objs_by_idx); tp_broadcast(&global_batch_list); tp_broadcast(&active_thread_count); + tp_broadcast(&remove_stats); + + // Per-worker memo: (obj input idx << 32 | coff symbol idx) -> final ref of the reloc-symbol + // resolve chain below. A symbol is referenced by one reloc per call site, so the chain + // (interp parse + symbol-table trie search per hop) otherwise repeats for identical inputs + // millions of times. Open-addressing and lossy (a collision past the probe window evicts); + // a miss only costs the recompute -- the cached value is a pure function of the key because + // the symbol table and parsed_symbols are read-only during /OPT:REF. + // Keep the table small enough that initializing every worker does not become a page-fault + // amplifier. Pack the resolved object input index + symbol index into one U64 and recover the + // object through objs_by_idx: 32K slots is 512KiB per worker (versus 768KiB with a pointer and + // widened symbol index, or 24MiB at the old 1M-slot size). + typedef struct { U64 key; U64 ref; } LNK_RefResolveSlot; + U64 resolve_cache_mask = (1ull << 15) - 1; + LNK_RefResolveSlot *resolve_cache = push_array_no_zero(scratch.arena, LNK_RefResolveSlot, resolve_cache_mask + 1); + MemorySet(resolve_cache, 0xff, sizeof(resolve_cache[0]) * (resolve_cache_mask + 1)); LNK_RelocRefsBatchList free_list = {0}; for (;;) { @@ -2741,30 +2870,153 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) // reloc -> symbol LNK_ObjSymbolRef ref_symbol = (LNK_ObjSymbolRef){ .obj = batch->v[i].obj, .symbol_idx = reloc->isymbol }; - lnk_resolve_reloc_target_symbol(scratch2.arena, symtab, ref_symbol, str8_lit("/OPT:REF"), &ref_symbol); + { + // resolve-cache lookup + U64 cache_key = ((U64)ref_symbol.obj->input_idx << 32ull) | (U64)ref_symbol.symbol_idx; + U64 cache_hash = cache_key * 0x9E3779B97F4A7C15ull; cache_hash ^= cache_hash >> 32; + U64 cache_slot = max_U64; + B32 cache_hit = 0; + for (U64 probe_idx = 0; probe_idx < 8; probe_idx += 1) { + U64 slot = (cache_hash + probe_idx) & resolve_cache_mask; + if (resolve_cache[slot].key == cache_key) { + U64 packed_ref = resolve_cache[slot].ref; + ref_symbol = packed_ref != max_U64 ? (LNK_ObjSymbolRef){ .obj = objs_by_idx[packed_ref >> 32], .symbol_idx = (U32)packed_ref } + : (LNK_ObjSymbolRef){0}; + cache_hit = 1; + break; + } + if (resolve_cache[slot].key == max_U64) { cache_slot = slot; break; } + } + + if (!cache_hit) { + // cycle detection via linear scan of the visited chain: chains are 1-3 hops in + // practice, and this keeps the tree HashMap + per-hop arena pushes off the hot path + // (exact same first-revisit semantics) + U64 chain_fixed[64]; + U64 *chain = chain_fixed; + U64 chain_count = 0; + U64 chain_cap = ArrayCount(chain_fixed); + B32 was_cyclic = 0; + + Temp temp = temp_begin(scratch2.arena); + B32 keep_walking = 1; + do { + // detect cyclic chains + U64 symbol_key = ((U64)ref_symbol.obj->input_idx << 32ull) | (U64)ref_symbol.symbol_idx; + B32 was_seen = 0; + for EachIndex(chain_idx, chain_count) { + if (chain[chain_idx] == symbol_key) { was_seen = 1; break; } + } + if (!was_seen) { + if (chain_count == chain_cap) { + U64 *new_chain = push_array_no_zero(temp.arena, U64, chain_cap * 2); + MemoryCopy(new_chain, chain, sizeof(chain[0]) * chain_count); + chain = new_chain; chain_cap *= 2; + } + chain[chain_count++] = symbol_key; + } else { + COFF_ParsedSymbol reloc_parsed = lnk_parsed_symbol_from_coff_symbol_idx(batch->v[i].obj, reloc->isymbol); + lnk_error_obj(LNK_Warning_CyclicSymbol, batch->v[i].obj, "symbol %S forms a cyclic chain (/OPT:REF)", reloc_parsed.name); + MemoryZeroStruct(&ref_symbol); + was_cyclic = 1; + break; + } + + // unpack symbol (interp needs no name decode) + COFF_ParsedSymbol ref_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref_symbol.obj, ref_symbol.symbol_idx); + COFF_SymbolValueInterpType ref_interp = coff_interp_from_parsed_symbol(ref_parsed); + + // resolve symbol + LNK_ObjSymbolRef next_ref = {0}; + if (lnk_resolve_symbol(symtab, ref_symbol, &next_ref)) { + keep_walking = (ref_interp == COFF_SymbolValueInterp_Weak || ref_interp == COFF_SymbolValueInterp_Undefined); + ref_symbol = next_ref; + } else { + keep_walking = 0; + } + } while (keep_walking); + temp_end(temp); + + // memoize (skip the cyclic-warning path so the warning replays per reloc as before) + if (!was_cyclic) { + if (cache_slot == max_U64) { cache_slot = cache_hash & resolve_cache_mask; } + U64 packed_ref = ref_symbol.obj ? Compose64Bit(ref_symbol.obj->input_idx, ref_symbol.symbol_idx) : max_U64; + resolve_cache[cache_slot] = (LNK_RefResolveSlot){ .key = cache_key, .ref = packed_ref }; + } + } + } // skip unresolved symbol if (ref_symbol.obj == 0) { continue; } - // unpack resolved symbol + // unpack resolved symbol (only interp + section_number are used -- skip the name decode) COFF_ParsedSymbol ref_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref_symbol.obj, ref_symbol.symbol_idx); COFF_SymbolValueInterpType ref_interp = coff_interp_from_parsed_symbol(ref_parsed); if (ref_interp == COFF_SymbolValueInterp_Regular) { Temp temp = temp_begin(scratch2.arena); - U32List associated_sections = lnk_obj_collect_associated_section_numbers(temp.arena, ref_symbol.obj, ref_parsed.section_number, 0); + LNK_Obj *walk_obj = ref_symbol.obj; + U32 seed_sn = ref_parsed.section_number; + + // per-walk visited set + walk stack: flat arrays with linear-scan membership -- + // associative groups are a handful of sections, and the tree HashMap + per-node + // arena pushes dominated this walk + U32 visited_fixed[64]; + U32 *visited = visited_fixed; + U64 visited_count = 0; + U64 visited_cap = ArrayCount(visited_fixed); + U32 stack_fixed[64]; + U32 *stack = stack_fixed; + U64 stack_count = 0; + U64 stack_cap = ArrayCount(stack_fixed); + stack[stack_count++] = seed_sn; + do { + U32 section_number = stack[--stack_count]; + + // detect cyclic associative sections + { + B32 was_seen = 0; + for EachIndex(visited_idx, visited_count) { + if (visited[visited_idx] == section_number) { was_seen = 1; break; } + } + if (was_seen) { continue; } + if (visited_count == visited_cap) { + U32 *new_visited = push_array_no_zero(temp.arena, U32, visited_cap * 2); + MemoryCopy(new_visited, visited, sizeof(visited[0]) * visited_count); + visited = new_visited; visited_cap *= 2; + } + visited[visited_count++] = section_number; + } + + // push associated section + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(walk_obj, section_number); + for EachIndex(associated_idx, associated_sections.count) { + U32 assoc_sn = associated_sections.v[associated_idx]; - // visit root section - u32_list_push(temp.arena, &associated_sections, ref_parsed.section_number); - for EachNode(section_n, U32Node, associated_sections.first) { - U32 section_number = section_n->data; + { + B32 assoc_seen = 0; + for EachIndex(visited_idx, visited_count) { + if (visited[visited_idx] == assoc_sn) { assoc_seen = 1; break; } + } + if (assoc_seen) { continue; } + } + if (stack_count == stack_cap) { + U32 *new_stack = push_array_no_zero(temp.arena, U32, stack_cap * 2); + MemoryCopy(new_stack, stack, sizeof(stack[0]) * stack_count); + stack = new_stack; stack_cap *= 2; + } + stack[stack_count++] = assoc_sn; + } COFF_SectionFlags section_flags = ref_symbol.obj->coff.sections.headers[section_number].flags; // on first section visit, set live flag and enqueue section - U8 was_visited = ins_atomic_u8_eval_assign(&is_live[ref_symbol.obj->input_idx][section_number], 1); + // (plain read first -- most targets are already live; the read keeps the flag + // cacheline shared instead of dirtying it with an unconditional exchange) + U8 was_visited = *(volatile U8 *)&is_live[walk_obj->input_idx][section_number]; + if (!was_visited) { was_visited = ins_atomic_u8_eval_assign(&is_live[walk_obj->input_idx][section_number], 1); } if (was_visited) { continue; } // is section eligible for walking? @@ -2789,7 +3041,8 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) } batch->v[batch->count++] = refs; - } + + } while (stack_count); temp_end(temp); } @@ -2822,74 +3075,63 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) exit:; barrier_wait(tp->barrier); - // TODO: thread - if (task_id == 0) { + // Remove unreachable sections. Section flags are per-obj (disjoint writes), so the obj list is + // strided across tasks via objs_by_idx; stats accumulate per task and are reduced on task 0, so + // the debug log totals are identical regardless of cohort width or schedule. + { ProfBegin("Remove Unreachable Sections"); + LNK_OptRefStat *stats = remove_stats + task_id * LNK_OptRefStat_Count; + for (U64 obj_idx = task_id; obj_idx < objs_count; obj_idx += tp->worker_count) { + LNK_Obj *obj = objs_by_idx[obj_idx]; - for EachIndex(obj_idx, objs_count) { - LNK_Obj *obj = objs[obj_idx]; for LNK_EachCoffSection(it, obj) { - COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, it.v.section_number); - if ( ! is_live[obj->input_idx][it.v.section_number]) { - *it.v.flags |= COFF_SectionFlag_LnkRemove; - } - } - } - - if (lnk_get_log_status(LNK_Log_Debug)) { - typedef struct { U64 vsize; U64 fsize; U64 section_count; U64 live_count; U64 live_fsize; U64 live_vsize; } Stat; - enum { Stat_Null, Stat_Code, Stat_Data, Stat_Debug, Stat_Count }; - Stat stats[Stat_Count] = {0}; - - for EachIndex(obj_idx, objs_count) { - LNK_Obj *obj = objs[obj_idx]; + U32 section_number = it.v.section_number; + if (is_live[obj->input_idx][section_number]) { continue; } - for LNK_EachCoffSection(it, obj) { - COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, it.v.section_number); + COFF_SectionHeader *section_header = it.v.header; + *it.v.flags |= COFF_SectionFlag_LnkRemove; + COFF_SectionFlags section_flags = *it.v.flags; - U64 stat_kind = Stat_Null; - if (*it.v.flags & LNK_SECTION_FLAG_DEBUG) { stat_kind = Stat_Debug; } - else if (*it.v.flags & COFF_SectionFlag_CntCode) { stat_kind = Stat_Code; } - else { stat_kind = Stat_Data; } + U64 stat_kind = LNK_OptRefStat_Null; + if (section_flags & LNK_SECTION_FLAG_DEBUG) { stat_kind = LNK_OptRefStat_Debug; } + else if (section_flags & COFF_SectionFlag_CntCode) { stat_kind = LNK_OptRefStat_Code; } + else { stat_kind = LNK_OptRefStat_Data; } - if (is_live[obj->input_idx][it.v.section_number]) { - stats[stat_kind].live_count += 1; - if (*it.v.flags & COFF_SectionFlag_CntUninitializedData) { - stats[stat_kind].live_vsize += section_header->vsize; - } else { - stats[stat_kind].live_fsize += section_header->fsize; - } - } else { - if (*it.v.flags & COFF_SectionFlag_CntUninitializedData) { - stats[stat_kind].vsize += section_header->vsize; - } else { - stats[stat_kind].fsize += section_header->fsize; - } - stats[stat_kind].section_count += 1; - } + if (section_flags & COFF_SectionFlag_CntUninitializedData) { + stats[stat_kind].vsize += section_header->vsize; + } else { + stats[stat_kind].fsize += section_header->fsize; } + stats[stat_kind].section_count += 1; } + } + ProfEnd(); + } + barrier_wait(tp->barrier); - U64 total_fsize = 0, total_section_count = 0; - U64 total_fsize_live = 0, total_section_count_live = 0; - for EachElement(i, stats) { - total_fsize += stats[i].fsize; - total_section_count += stats[i].section_count; - total_fsize_live += stats[i].live_fsize; - total_section_count_live += stats[i].live_count; + if (task_id == 0 && lnk_get_log_status(LNK_Log_Debug)) { + LNK_OptRefStat stats[LNK_OptRefStat_Count] = {0}; + for EachIndex(reduce_task_idx, tp->worker_count) { + for EachIndex(stat_idx, (U64)LNK_OptRefStat_Count) { + stats[stat_idx].vsize += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].vsize; + stats[stat_idx].fsize += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].fsize; + stats[stat_idx].section_count += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].section_count; } - String8List stat_list = {0}; - str8_list_pushf(scratch.arena, &stat_list, "Code : removed %M, %S sections; live %M, %S sections", stats[Stat_Code].fsize, str8_from_count(scratch.arena, stats[Stat_Code].section_count ), stats[Stat_Code].live_fsize, str8_from_count(scratch.arena, stats[Stat_Code].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Data : removed %M, %S sections; live %M, %S sections", stats[Stat_Data].fsize, str8_from_count(scratch.arena, stats[Stat_Data].section_count ), stats[Stat_Data].live_fsize, str8_from_count(scratch.arena, stats[Stat_Data].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Debug: removed %M, %S sections; live %M, %S sections", stats[Stat_Debug].fsize, str8_from_count(scratch.arena, stats[Stat_Debug].section_count), stats[Stat_Debug].live_fsize, str8_from_count(scratch.arena, stats[Stat_Debug].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Total: removed %M, %S sections; live %M, %S sections", total_fsize, str8_from_count(scratch.arena, total_section_count), total_fsize_live, str8_from_count(scratch.arena, total_section_count_live)); - String8 stat_str = str8_list_join(scratch.arena, &stat_list, &(StringJoin){.pre = str8_lit(" "), .sep = str8_lit("\n ")}); - lnk_log(LNK_Log_Debug, "/OPT:REF Stats:\n%S", stat_str); } - ProfEnd(); + U64 total_fsize = 0, total_section_count = 0; + for EachElement(i, stats) { + total_fsize += stats[i].fsize; + total_section_count += stats[i].section_count; + } + String8List stat_list = {0}; + str8_list_pushf(scratch.arena, &stat_list, "Code : %M, %S sections", stats[LNK_OptRefStat_Code].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Code].section_count )); + str8_list_pushf(scratch.arena, &stat_list, "Data : %M, %S sections", stats[LNK_OptRefStat_Data].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Data].section_count )); + str8_list_pushf(scratch.arena, &stat_list, "Debug: %M, %S sections", stats[LNK_OptRefStat_Debug].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Debug].section_count)); + str8_list_pushf(scratch.arena, &stat_list, "Total: %M, %S sections", total_fsize, str8_from_count(scratch.arena, total_section_count)); + String8 stat_str = str8_list_join(scratch.arena, &stat_list, &(StringJoin){.pre = str8_lit(" "), .sep = str8_lit("\n ")}); + lnk_log(LNK_Log_Debug, "/OPT:REF Stats:\n%S", stat_str); } - barrier_wait(tp->barrier); scratch_end(scratch2); scratch_end(scratch); @@ -2901,9 +3143,15 @@ lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj { ProfBegin("/OPT:REF"); Temp scratch = scratch_begin(0,0); - U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier)/tp_broadcast, + // so under /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- + // a plain tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane obj distribution so both agree on the width. + U32 C = tp_barrier_begin(tp); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, C, objs, objs_count); LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; - tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_ref_task, &task); + tp_for_parallel_reserve(tp, 0, C, lnk_opt_ref_task, &task); // BARRIER pass (path B) + tp_barrier_end(tp); scratch_end(scratch); ProfEnd(); } @@ -3023,11 +3271,11 @@ lnk_icf_scope_from_section_number(LNK_Obj *obj, U32 section_number) } internal void -lnk_icf_atomic_min_u64(U64 *dst, U64 value) +lnk_icf_atomic_min_u32(U32 *dst, U32 value) { // preserve stable leaders despite concurrent insertion - for (U64 old_value = ins_atomic_u64_eval(dst); value < old_value;) { - U64 observed = ins_atomic_u64_eval_cond_assign(dst, value, old_value); + for (U32 old_value = ins_atomic_u32_eval(dst); value < old_value;) { + U32 observed = ins_atomic_u32_eval_cond_assign(dst, value, old_value); if (observed == old_value) { break; } old_value = observed; } @@ -3048,39 +3296,44 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) // only target colors vary between rounds, so cache non-recursive relocation data // and rehash target colors each round typedef struct { - U64 *color; - U64 static_id; + union { + U64 *color; + U64 static_id; + }; U64 association_id; U32 value; COFF_SymbolValueInterpType interp; } RelocTarget; + // Contribution and table indices are bounded to U32 below. Keeping the hot contribution + // record at 64 bytes cuts both its footprint and the demand-zero work during ICF. typedef struct { ColorKey key; U128 static_hash; RelocTarget **reloc_targets; - U64 reloc_count; - U64 color_slot_idx; + U32 color_slot_idx; + U32 reloc_count; U32 obj_idx; U32 section_number; - LNK_ICF_Scope scope; } Contrib; - // reuse both tables without clearing them between refinement rounds + // Reuse both tables without clearing them between refinement rounds. U32 generations and + // indices keep these records at 32 and 16 bytes respectively. The 128-bit group hash is dead + // before old-color indexing, so that phase stores its U32 slot index in the hash bytes. The + // old-color key is then dead after split counting, so assignment overwrites it with the output + // color. These lifetime reuses avoid carrying either result through the 16M-slot table. typedef struct { - U64 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) ColorKey key; - U64 first_contrib_idx; - U64 old_color_slot_idx; - U64 color; + U32 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) + U32 first_contrib_idx; } ColorHashSlot; typedef struct { ColorHashSlot *slots; U64 slots_count; } ColorHashTable; typedef struct { - U64 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) U64 old_color; - U64 first_contrib_idx; + U32 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) + U32 first_contrib_idx; } OldColorHashSlot; typedef struct { OldColorHashSlot *slots; U64 slots_count; } OldColorHashTable; @@ -3203,6 +3456,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } shared = { .contrib_count = sum_array_u64(task->objs_count, contrib_counts) }; + Assert(shared.contrib_count <= max_U32); if (task_id == 0) { ProfBegin("Init"); @@ -3255,7 +3509,6 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) *contrib = (Contrib){ .obj_idx = safe_cast_u32(obj->input_idx), .section_number = safe_cast_u32(it.v.section_number), - .scope = scope, }; Temp temp = temp_begin(scratch.arena); @@ -3263,11 +3516,13 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) U32List associated_sections = lnk_obj_collect_associated_section_numbers(temp.arena, obj, it.v.section_number, associated_filter); u32_list_push(temp.arena, &associated_sections, it.v.section_number); + U64 reloc_count = 0; for EachNode(associated_n, U32Node, associated_sections.first) { COFF_SectionHeader *associated_header = lnk_coff_section_header_from_section_number(obj, associated_n->data); COFF_RelocArray associated_relocs = lnk_coff_relocs_from_section_header(obj, associated_header); - contrib->reloc_count += associated_relocs.count; + reloc_count += associated_relocs.count; } + contrib->reloc_count = safe_cast_u32(reloc_count); if (contrib->reloc_count) { contrib->reloc_targets = push_array(scratch2.arena, RelocTarget *, contrib->reloc_count); } @@ -3318,8 +3573,9 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } U64 child_pos = 0; - for EachNode(child_n, U32Node, target_obj->coff.sections.associated_section_numbers[parent_sect]) { - if (child_n->data == target_sect) { break; } + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(target_obj, parent_sect); + for EachIndex(child_idx, associated_sections.count) { + if (associated_sections.v[child_idx] == target_sect) { break; } child_pos += 1; } @@ -3376,14 +3632,15 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) ColorHashTable color_table = {0}; OldColorHashTable old_color_table = {0}; - U64 *table_generation = 0; + U32 *table_generation = 0; if (task_id == 0) { ProfBegin("Alloc hash tables"); color_table.slots_count = u64_up_to_pow2(Max(2, shared.contrib_count*2)); + Assert(color_table.slots_count <= (U64)max_U32 + 1); color_table.slots = push_array(scratch.arena, ColorHashSlot, color_table.slots_count); old_color_table.slots_count = color_table.slots_count; old_color_table.slots = push_array(scratch.arena, OldColorHashSlot, old_color_table.slots_count); - table_generation = push_array_no_zero(scratch.arena, U64, 1); + table_generation = push_array_no_zero(scratch.arena, U32, 1); *table_generation = 0; ProfEnd(); } @@ -3406,15 +3663,15 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) *shared.is_part_stable = 1; // update hash tables generations - Assert(*table_generation < (max_U64 >> 2)); + Assert(*table_generation < (max_U32 >> 2)); *table_generation += 1; } barrier_wait(tp->barrier); // unpack the table generation - U64 table_generation_value = *table_generation; - U64 initializing_state = (table_generation_value << 2) | 1; - U64 ready_state = (table_generation_value << 2) | 2; + U32 table_generation_value = *table_generation; + U32 initializing_state = (table_generation_value << 2) | 1; + U32 ready_state = (table_generation_value << 2) | 2; ProfBegin("Compute colored hashes"); for EachInRange(contrib_idx, shared.contrib_ranges[task_id]) { @@ -3426,7 +3683,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) lnk_hasher_update_struct(&hasher, &contrib->static_hash); for EachIndex(reloc_idx, contrib->reloc_count) { RelocTarget *target = contrib->reloc_targets[reloc_idx]; - U64 target_id = target->color ? *target->color : target->static_id; + U64 target_id = target->interp == COFF_SymbolValueInterp_Regular ? *target->color : target->static_id; lnk_hasher_update_struct(&hasher, &target_id); lnk_hasher_update_struct(&hasher, &target->association_id); } @@ -3440,29 +3697,29 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) U64 color_slot_idx = table_hash & (color_table.slots_count - 1); for (;;) { ColorHashSlot *color_slot = &color_table.slots[color_slot_idx]; - U64 state = ins_atomic_u64_eval(&color_slot->state); + U32 state = ins_atomic_u32_eval(&color_slot->state); if ((state >> 2) != table_generation_value) { - if (ins_atomic_u64_eval_cond_assign(&color_slot->state, initializing_state, state) == state) { + if (ins_atomic_u32_eval_cond_assign(&color_slot->state, initializing_state, state) == state) { color_slot->key = contrib->key; - color_slot->first_contrib_idx = contrib_idx; - contrib->color_slot_idx = color_slot_idx; - ins_atomic_u64_eval_assign(&color_slot->state, ready_state); + color_slot->first_contrib_idx = safe_cast_u32(contrib_idx); + contrib->color_slot_idx = safe_cast_u32(color_slot_idx); + ins_atomic_u32_eval_assign(&color_slot->state, ready_state); break; } continue; } if (state == initializing_state) { - do { state = ins_atomic_u64_eval(&color_slot->state); } while (state == initializing_state); + do { state = ins_atomic_u32_eval(&color_slot->state); } while (state == initializing_state); continue; } Assert(state == ready_state); if (color_slot->key.old_color == contrib->key.old_color && u128_match(color_slot->key.hash, contrib->key.hash)) { - lnk_icf_atomic_min_u64(&color_slot->first_contrib_idx, contrib_idx); - contrib->color_slot_idx = color_slot_idx; + lnk_icf_atomic_min_u32(&color_slot->first_contrib_idx, safe_cast_u32(contrib_idx)); + contrib->color_slot_idx = safe_cast_u32(color_slot_idx); break; } @@ -3478,40 +3735,40 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *slot = &color_table.slots[contrib->color_slot_idx]; - if (ins_atomic_u64_eval(&slot->first_contrib_idx) == contrib_idx) { + if (ins_atomic_u32_eval(&slot->first_contrib_idx) == contrib_idx) { Assert(old_color_table.slots_count > 0 && (old_color_table.slots_count & (old_color_table.slots_count - 1)) == 0); U64 old_color = slot->key.old_color; U64 table_hash = hash_map_hasher(str8_struct(&old_color)); U64 old_slot_idx = table_hash & (old_color_table.slots_count - 1); for (;;) { OldColorHashSlot *old_color_slot = &old_color_table.slots[old_slot_idx]; - U64 state = ins_atomic_u64_eval(&old_color_slot->state); + U32 state = ins_atomic_u32_eval(&old_color_slot->state); if ((state >> 2) != table_generation_value) { - if (ins_atomic_u64_eval_cond_assign(&old_color_slot->state, initializing_state, state) == state) { + if (ins_atomic_u32_eval_cond_assign(&old_color_slot->state, initializing_state, state) == state) { old_color_slot->old_color = old_color; - old_color_slot->first_contrib_idx = contrib_idx; - ins_atomic_u64_eval_assign(&old_color_slot->state, ready_state); + old_color_slot->first_contrib_idx = safe_cast_u32(contrib_idx); + ins_atomic_u32_eval_assign(&old_color_slot->state, ready_state); break; } continue; } if (state == initializing_state) { - do { state = ins_atomic_u64_eval(&old_color_slot->state); } while (state == initializing_state); + do { state = ins_atomic_u32_eval(&old_color_slot->state); } while (state == initializing_state); continue; } Assert(state == ready_state); if (old_color_slot->old_color == old_color) { - lnk_icf_atomic_min_u64(&old_color_slot->first_contrib_idx, contrib_idx); + lnk_icf_atomic_min_u32(&old_color_slot->first_contrib_idx, safe_cast_u32(contrib_idx)); break; } old_slot_idx = (old_slot_idx + 1) & (old_color_table.slots_count - 1); } - slot->old_color_slot_idx = old_slot_idx; + memory_write32(&slot->key.hash, safe_cast_u32(old_slot_idx)); } } ProfEnd(); @@ -3523,10 +3780,11 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; - if (ins_atomic_u64_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } + if (ins_atomic_u32_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } - OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx]; - if (ins_atomic_u64_eval(&old_color_slot->first_contrib_idx) != contrib_idx) { + U32 old_color_slot_idx = memory_read32(&color_slot->key.hash); + OldColorHashSlot *old_color_slot = &old_color_table.slots[old_color_slot_idx]; + if (ins_atomic_u32_eval(&old_color_slot->first_contrib_idx) != contrib_idx) { split_count += 1; } } @@ -3561,13 +3819,14 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; - if (ins_atomic_u64_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } + if (ins_atomic_u32_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } - OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx]; - if (ins_atomic_u64_eval(&old_color_slot->first_contrib_idx) == contrib_idx) { - color_slot->color = color_slot->key.old_color; + U32 old_color_slot_idx = memory_read32(&color_slot->key.hash); + OldColorHashSlot *old_color_slot = &old_color_table.slots[old_color_slot_idx]; + if (ins_atomic_u32_eval(&old_color_slot->first_contrib_idx) == contrib_idx) { + // key.old_color has completed its lookup lifetime; reuse it as the output color } else { - color_slot->color = ++next_split_color; + color_slot->key.old_color = ++next_split_color; } } ProfEnd(); @@ -3580,7 +3839,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) Rng1U64 obj_contrib_range = r1u64(shared.contrib_offsets[obj_idx], shared.contrib_offsets[obj_idx] + contrib_counts[obj_idx]); for EachInRange(contrib_idx, obj_contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; - shared.color_map[contrib->obj_idx][contrib->section_number] = color_table.slots[contrib->color_slot_idx].color; + shared.color_map[contrib->obj_idx][contrib->section_number] = color_table.slots[contrib->color_slot_idx].key.old_color; } } ProfEnd(); @@ -3608,20 +3867,21 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, shared.contrib_ranges[task_id]) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; - Contrib *leader = &shared.contribs[ins_atomic_u64_eval(&color_slot->first_contrib_idx)]; + Contrib *leader = &shared.contribs[ins_atomic_u32_eval(&color_slot->first_contrib_idx)]; LNK_Obj *contrib_obj = objs[contrib->obj_idx]; LNK_Obj *leader_obj = objs[leader->obj_idx]; if (fold_stats) { - FoldStats *st = fold_stats + (task_id * LNK_ICF_Scope_COUNT); - U64 fsize = lnk_coff_section_header_from_section_number(contrib_obj, contrib->section_number)->fsize; + FoldStats *st = fold_stats + (task_id * LNK_ICF_Scope_COUNT); + U64 fsize = lnk_coff_section_header_from_section_number(contrib_obj, contrib->section_number)->fsize; + LNK_ICF_Scope scope = lnk_icf_scope_from_section_number(contrib_obj, contrib->section_number); if (leader == contrib) { - st[leader->scope].live_count += 1; - st[leader->scope].live_size += fsize; + st[scope].live_count += 1; + st[scope].live_size += fsize; } else { - st[contrib->scope].count += 1; - st[contrib->scope].size += fsize; + st[scope].count += 1; + st[scope].size += fsize; } } @@ -3677,6 +3937,13 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) temp_end(assoc_temp); } + // record the fold for debug aliasing (lnk_icf_mark_folded_lines): unlike the symlink + // redirect, this distinguishes an ICF fold (different-named section joined to a leader) + // from same-name COMDAT selection and /OPT:REF removal + if (contrib_obj->icf_fold) { + contrib_obj->icf_fold[contrib->section_number] = (LNK_ICFFold){ .leader_obj_idx = (U32)leader->obj_idx, .leader_sn = leader->section_number, .set = 1 }; + } + #if LNK_PARANOID String8 section_name = lnk_obj_section_name_from_section_number(contrib_obj, contrib->section_number); String8 leader_name = lnk_obj_section_name_from_section_number(leader_obj, leader->section_number); @@ -3780,15 +4047,32 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } internal void -lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) +lnk_opt_icf(TP_Context *tp, Arena *perm, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) { ProfBegin("/OPT:ICF"); - Temp scratch = scratch_begin(0,0); - + Temp scratch = scratch_begin(&perm, 1); + lnk_log(LNK_Log_Debug, "/OPT:ICF:"); - U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + + // per-section fold map, consumed by the debug-aliasing pass after /OPT:REF + // (lnk_icf_mark_folded_lines); allocated only when that pass will run + if (config->opt_ref == LNK_SwitchState_Yes) { + ProfScope("Alloc fold maps") { + for EachIndex(obj_idx, objs_count) { + objs[obj_idx]->icf_fold = push_array(perm, LNK_ICFFold, objs[obj_idx]->coff.sections.count_no_null + 1); + } + } + } + + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier)/tp_broadcast, + // so under /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- + // a plain tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane obj distribution so both agree on the width. + U32 C = tp_barrier_begin(tp); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, C, objs, objs_count); LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; - tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_icf_task, &task); + tp_for_parallel_reserve(tp, 0, C, lnk_opt_icf_task, &task); // BARRIER pass (path B) + tp_barrier_end(tp); scratch_end(scratch); ProfEnd(); @@ -3836,6 +4120,281 @@ lnk_should_gather_section(LNK_Obj *obj, U64 section_number, COFF_SectionHeader * return 1; } +typedef struct +{ + LNK_Obj **objs; // indexed by input_idx (== task_id) +} LNK_ICFMarkFoldedLinesTask; + +// Find the COMDAT-associative .debug$S child that carries a function section's +// per-function CodeView records. +internal U32 +lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn) +{ + if (fn_sn == 0 || fn_sn > obj->coff.sections.count_no_null) { return 0; } + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(obj, fn_sn); + for EachIndex(assoc_idx, associated_sections.count) { + U32 sn = associated_sections.v[assoc_idx]; + if (sn == 0 || sn > obj->coff.sections.count_no_null) { continue; } + if (~obj->coff.sections.headers[sn].flags & LNK_SECTION_FLAG_DEBUG) { continue; } + if (str8_match(lnk_obj_section_name_from_section_number(obj, sn), str8_lit(".debug$S"), 0)) { return sn; } + } + return 0; +} + + +// FILECHKSMS of the obj-wide (non-COMDAT) .debug$S -- the table every per-function +// Lines fragment's file_off indexes into. Direct header walk with early-out instead of +// cv_debug_s_from_data: the obj-wide .debug$S is megabytes of subsections and the full +// parse pushes a list node per subsection; here we only need one slice. +internal String8 +lnk_icf_obj_file_chksms_scan(LNK_Obj *obj) +{ + for LNK_EachCoffSection(it, obj) { + COFF_SectionFlags flags = *it.v.flags; + if (~flags & LNK_SECTION_FLAG_DEBUG) { continue; } + if ( flags & COFF_SectionFlag_LnkCOMDAT) { continue; } + if (!str8_match(lnk_obj_section_name_from_section_number(obj, it.v.section_number), str8_lit(".debug$S"), 0)) { continue; } + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, it.v.frange, &indexed)) { + for EachIndex(i, indexed.count) { + LNK_CObjDebugSEntry *entry = &indexed.v[i]; + if (entry->kind == CV_C13SubSectionKind_FileChksms) { + return str8(obj->coff.data.str + entry->raw_payload_offset, entry->raw_payload_size); + } + } + continue; + } + String8 raw = lnk_obj_section_data_from_number(obj, it.v.section_number); + if (raw.size < sizeof(CV_Signature) || cv_signature_from_debug_s(raw) != CV_Signature_C13) { continue; } + for (U64 cursor = sizeof(CV_Signature); cursor + sizeof(CV_C13SubSectionHeader) <= raw.size; ) { + CV_C13SubSectionHeader header = {0}; + cursor += str8_deserial_read_struct(raw, cursor, &header); + if (header.kind == CV_C13SubSectionKind_FileChksms) { + return str8_substr(raw, r1u64(cursor, cursor + header.size)); + } + cursor += header.size; + cursor = AlignPow2(cursor, CV_C13SubSectionAlign); + } + } + return str8_zero(); +} + +// Memoized per obj: leaders are shared across many follower objs, so without the memo the +// scan reruns once per (follower obj x leader switch). The result slices the immutable +// obj->coff.data mapping, so the racy fill is idempotent (every worker writes identical bytes); +// the init flag is published last. +internal String8 +lnk_icf_obj_file_chksms(LNK_Obj *obj) +{ + if (!ins_atomic_u32_eval((U32 *)&obj->icf_file_chksms_init)) { + String8 chksms = lnk_icf_obj_file_chksms_scan(obj); + obj->icf_file_chksms = chksms; + ins_atomic_u32_eval_assign((U32 *)&obj->icf_file_chksms_init, 1); + } + return obj->icf_file_chksms; +} + +// source identity of a function: (checksum of its file, first line). Two ICF fold members +// with equal keys are the same source text (template twins) -- their locals/labels are +// identical and the leader's record tree serves both. +typedef struct +{ + B32 valid; + U32 line; + U8 chksum_kind; + String8 chksum; +} LNK_ICFSrcKey; + +internal LNK_ICFSrcKey +lnk_icf_src_key_from_fn(Arena *scratch, LNK_Obj *obj, U32 fn_sn, String8 chksms) +{ + LNK_ICFSrcKey key = {0}; + U32 child_sn = lnk_icf_debug_s_child_from_section(obj, fn_sn); + if (child_sn == 0 || chksms.size == 0) { return key; } + LNK_ObjSection sect = lnk_obj_section_from_section_number(obj, child_sn); + String8 frag = {0}; + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, sect.frange, &indexed)) { + for EachIndex(i, indexed.count) { + LNK_CObjDebugSEntry *entry = &indexed.v[i]; + if (entry->kind == CV_C13SubSectionKind_Lines) { + frag = str8(obj->coff.data.str + entry->raw_payload_offset, entry->raw_payload_size); + break; + } + } + } else { + String8 raw = lnk_obj_section_data_from_number(obj, child_sn); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + cv_debug_s_tag_prov_sect(&ds, child_sn-1); + String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); + if (lines.node_count) { frag = lines.first->string; } + } + if (frag.size == 0) { return key; } + if (frag.size < sizeof(CV_C13SubSecLinesHeader) + sizeof(CV_C13File) + sizeof(CV_C13Line)) { return key; } + CV_C13File *file = (CV_C13File *)(frag.str + sizeof(CV_C13SubSecLinesHeader)); + CV_C13Line *l0 = (CV_C13Line *)((U8 *)file + sizeof(CV_C13File)); + if ((U64)file->file_off + sizeof(CV_C13Checksum) > chksms.size) { return key; } + CV_C13Checksum *ck = (CV_C13Checksum *)(chksms.str + file->file_off); + if ((U64)file->file_off + sizeof(CV_C13Checksum) + ck->len > chksms.size) { return key; } + key.valid = 1; + key.line = (U32)(l0->flags & 0xFFFFFF); + key.chksum_kind = ck->kind; + key.chksum = str8(chksms.str + file->file_off + sizeof(CV_C13Checksum), ck->len); + return key; +} + +internal B32 +lnk_icf_debug_s_summary_has_locals(LNK_CObjDebugSView *indexed) +{ + for EachIndex(i, indexed->count) { + if (indexed->v[i].kind == CV_C13SubSectionKind_Symbols && + (indexed->summaries[i].flags & LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS)) { return 1; } + } + return 0; +} + +// does the function's record tree have anything a watch window would show? +internal B32 +lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) +{ + LNK_ObjSection sect = lnk_obj_section_from_section_number(obj, child_sn); + LNK_CObjDebugSView indexed = {0}; + String8List syms = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, sect.frange, &indexed)) { + if (indexed.summaries) { + return lnk_icf_debug_s_summary_has_locals(&indexed); + } + for EachIndex(i, indexed.count) { + if (indexed.v[i].kind == CV_C13SubSectionKind_Symbols) { + str8_list_push(scratch, &syms, str8(obj->coff.data.str + indexed.v[i].raw_payload_offset, + indexed.v[i].raw_payload_size)); + } + } + } else { + String8 raw = lnk_obj_section_data_from_number(obj, child_sn); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + cv_debug_s_tag_prov_sect(&ds, child_sn-1); + syms = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Symbols); + } + for EachNode(n, String8Node, syms.first) { + String8 s = n->string; + for (U64 o = 0; o + 4 <= s.size; ) { + U16 len, kind; + MemoryCopy(&len, s.str + o, sizeof(len)); + MemoryCopy(&kind, s.str + o + 2, sizeof(kind)); + if (len < 2) { break; } + switch (kind) { + // stack locals + case CV_SymKind_LOCAL: + case CV_SymKind_REGREL32: + // function-scoped statics (S_LDATA32 and friends): the record naming the static lives in + // this tree; if it were dropped the debugger could no longer evaluate the follower's + // static by name, even though the (folded) data itself survives in the image + case CV_SymKind_LDATA32: + case CV_SymKind_GDATA32: + case CV_SymKind_LTHREAD32: + case CV_SymKind_GTHREAD32: + case CV_SymKind_FILESTATIC: + case CV_SymKind_CONSTANT: + return 1; + } + o += len + 2; + } + } + return 0; +} + +// /OPT:ICF folded-function debug-info slimming. Without this pass a folded function's associated +// .debug$S stays collected in full (REF marked it live with its then-live parent; the ICF fold +// removes only the .text follower), so the module stream receives every folded body's WHOLE record +// tree (S_GPROC/locals + Lines) bound to the leader RVA -- link.exe-parity content at ~+30% module +// bytes. The C13 Lines subsections are ~1% of that and are all a source breakpoint needs to bind. +// So mark each folded follower's associated .debug$S LnkRemove (drops it from full collection); +// the reloc patcher still patches it and the C13 pass merges back ONLY its Lines -- their +// SECREL/SECTION relocs target the folded function symbol, which resolves to the leader RVA +// through the redirected symlink/sect_map, so the lines land on the surviving body. +internal +THREAD_POOL_TASK_FUNC(lnk_icf_mark_folded_lines_task) +{ + Temp scratch = scratch_begin(&arena, 1); + + LNK_ICFMarkFoldedLinesTask *task = raw_task; + LNK_Obj *obj = task->objs[task_id]; + if (obj->icf_fold == 0) { scratch_end(scratch); return; } + + for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { + LNK_ICFFold fold = obj->icf_fold[section_number]; + if (!fold.set) { continue; } + if (~obj->coff.sections.headers[section_number].flags & COFF_SectionFlag_LnkRemove) { continue; } // follower kept live -> its own records emit + LNK_Obj *leader_obj = task->objs[fold.leader_obj_idx]; + if (leader_obj->coff.sections.headers[fold.leader_sn].flags & COFF_SectionFlag_LnkRemove) { continue; } // whole class dead-stripped + + // Lines-only by default. Escalate to the FULL record tree (link.exe parity for this one + // fold) when the follower comes from a DIFFERENT source location than the leader (else the + // trees are textually identical -- template twins) AND it has locals to show. Measured on + // the FN editor DLL: ~6.5% of folds differ in source, most of those are empty virtuals, so + // the escalation set is small. + U8 mark = 1; + { + Temp fold_temp = temp_begin(scratch.arena); + U32 child_sn = lnk_icf_debug_s_child_from_section(obj, section_number); + B32 needs_src_key = child_sn != 0; + if (needs_src_key && obj->compressed_obj != 0) { + // Only a follower with locals can escalate to a full record tree. A + // negative summary proves that source-key reads cannot change the mark; + // avoid faulting compressed Lines/checksum payloads just to compare them. + // Without summaries, retain the old source-key-first order so raw symbol + // trees are not parsed unnecessarily for same-source folds. + LNK_ObjSection child = lnk_obj_section_from_section_number(obj, child_sn); + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, child.frange, &indexed) && indexed.summaries && + !lnk_icf_debug_s_summary_has_locals(&indexed)) { + needs_src_key = 0; + } + } + if (needs_src_key) { + String8 follower_chksms = lnk_icf_obj_file_chksms(obj); // per-obj memo -- leaders + String8 leader_chksms = lnk_icf_obj_file_chksms(leader_obj); // shared across follower objs + LNK_ICFSrcKey fk = lnk_icf_src_key_from_fn(fold_temp.arena, obj, section_number, follower_chksms); + LNK_ICFSrcKey lk = lnk_icf_src_key_from_fn(fold_temp.arena, leader_obj, fold.leader_sn, leader_chksms); + B32 same_src = fk.valid && lk.valid && + fk.line == lk.line && fk.chksum_kind == lk.chksum_kind && + str8_match(fk.chksum, lk.chksum, 0); + if (fk.valid && lk.valid && !same_src && lnk_icf_debug_s_has_locals(fold_temp.arena, obj, child_sn)) { + mark = 2; + } + } + temp_end(fold_temp); + } + + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(obj, section_number); + for EachIndex(assoc_idx, associated_sections.count) { + U32 assoc_sn = associated_sections.v[assoc_idx]; + if (assoc_sn == 0 || assoc_sn > obj->coff.sections.count_no_null) { continue; } + if (~obj->coff.sections.headers[assoc_sn].flags & LNK_SECTION_FLAG_DEBUG) { continue; } + // exclude the follower's .debug$S from full module collection (it would otherwise merge + // its whole record tree at the leader RVA, link.exe-parity size); the consumers below + // merge back just its Lines (mark 1) or, rarely, the whole tree (mark 2) + obj->coff.sections.headers[assoc_sn].flags |= COFF_SectionFlag_LnkRemove; + if (obj->icf_lines_only == 0) { + obj->icf_lines_only = push_array(arena, B8, obj->coff.sections.count_no_null + 1); + } + obj->icf_lines_only[assoc_sn] = mark; + } + } + + scratch_end(scratch); +} + +internal void +lnk_icf_mark_folded_lines(TP_Context *tp, TP_Arena *arena, LNK_Obj **objs, U64 objs_count) +{ + ProfBeginFunction(); + LNK_ICFMarkFoldedLinesTask task = { .objs = objs }; + tp_for_parallel(tp, arena, objs_count, lnk_icf_mark_folded_lines_task, &task); // arena: per-obj icf_lines_only bitmaps + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_gather_sections_task) { @@ -4421,6 +4980,48 @@ THREAD_POOL_TASK_FUNC(lnk_patch_weak_symbols_task) lnk_patch_obj_symtab(task->symtab, task->objs[task_id], task->u.patch_symtabs.was_symbol_patched[task_id], COFF_SymbolValueInterp_Weak); } +// Non-temporal (streaming) stores for the write-once image buffer. The image is +// filled, then streamed straight to disk; these bytes are not re-read by the +// filling thread, so NT stores avoid polluting L2/L3 with ~1GB of write-once +// data. A later pass (lnk_obj_reloc_patcher) DOES read the image back, so every +// caller must _mm_sfence() before that pass runs to make the NT stores globally +// visible. NT stores require 32B alignment; the unaligned head/tail and small +// (<256B) copies fall back to MemoryCopy/MemorySet (identical bytes either way). +#define LNK_STREAM_MIN_SIZE 256 + +// SSE2 (baseline on x86-64, no -mavx required) 16B non-temporal stores. +internal void +lnk_stream_copy(void *dst, void *src, U64 size) +{ + if (size < LNK_STREAM_MIN_SIZE) { MemoryCopy(dst, src, size); return; } + U8 *d = (U8 *)dst, *s = (U8 *)src; + U64 head = (U64)(0x10 - ((U64)d & 0xf)) & 0xf; // bytes to reach 16B-aligned dst + if (head) { MemoryCopy(d, s, head); d += head; s += head; size -= head; } + U64 vec = size & ~(U64)0xf; + for (U64 i = 0; i < vec; i += 0x10) { + __m128i v = _mm_loadu_si128((__m128i const *)(s + i)); + _mm_stream_si128((__m128i *)(d + i), v); + } + U64 tail = size - vec; + if (tail) { MemoryCopy(d + vec, s + vec, tail); } +} + +internal void +lnk_stream_set(void *dst, U8 byte, U64 size) +{ + if (size < LNK_STREAM_MIN_SIZE) { MemorySet(dst, byte, size); return; } + U8 *d = (U8 *)dst; + U64 head = (U64)(0x10 - ((U64)d & 0xf)) & 0xf; + if (head) { MemorySet(d, byte, head); d += head; size -= head; } + __m128i v = _mm_set1_epi8((char)byte); + U64 vec = size & ~(U64)0xf; + for (U64 i = 0; i < vec; i += 0x10) { + _mm_stream_si128((__m128i *)(d + i), v); + } + U64 tail = size - vec; + if (tail) { MemorySet(d + vec, byte, tail); } +} + internal THREAD_POOL_TASK_FUNC(lnk_image_fill_task) { @@ -4430,15 +5031,45 @@ THREAD_POOL_TASK_FUNC(lnk_image_fill_task) for EachNode(n, LNK_ImageFillNode, task->u.image_fill.fill_nodes[task_id]) { for EachIndex(i, n->sc_count) { LNK_SectionContrib *sc = n->sc[i]; + // fast-path: the vast majority of contribs are a single data-node -> one direct copy, skipping + // the list-walk + cursor bookkeeping on the hot 739MB image-write loop. + if (sc->first_data_node.next == 0) { + U64 image_off = sc->u.off + n->base_foff; + Assert(image_off + sc->first_data_node.string.size <= image_data.size); + lnk_stream_copy(image_data.str + image_off, sc->first_data_node.string.str, sc->first_data_node.string.size); + continue; + } U64 cursor = 0; for EachNode(data_n, String8Node, &sc->first_data_node) { U64 image_off = sc->u.off + n->base_foff + cursor; Assert(image_off + data_n->string.size <= image_data.size); - MemoryCopyStr8(image_data.str + image_off, data_n->string); + lnk_stream_copy(image_data.str + image_off, data_n->string.str, data_n->string.size); cursor += data_n->string.size; } } } + // NT stores above are not ordered wrt later normal reads on other cores; the + // reloc-patch pass reads the image back. Make these stores globally visible. + _mm_sfence(); + ProfEnd(); +} + +typedef struct +{ + U8 *dst; + U64 size; + U8 byte; +} LNK_FillAlignRange; + +internal +THREAD_POOL_TASK_FUNC(lnk_fill_align_bytes_task) +{ + ProfBeginFunction(); + LNK_FillAlignRange *range = &((LNK_FillAlignRange *)raw_task)[task_id]; + lnk_stream_set(range->dst, range->byte, range->size); + // make this task's NT stores globally visible before the completion counter is + // bumped, so every thread past the join (contrib-fill / reloc passes) sees them + _mm_sfence(); ProfEnd(); } @@ -4463,101 +5094,50 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) LNK_ObjRelocPatcher *task = raw_task; LNK_Obj *obj = task->objs[task_id]; - COFF_FileHeaderInfo obj_header = obj->coff.header; - String8 string_table = lnk_coff_string_table_from_obj(obj); - - U32 closest_sect = 0; - U32 closest_reloc = 0; - U32 closest_foff = max_U32; + String8 string_table = lnk_coff_string_table_from_obj(obj); for LNK_EachCoffSection(it, obj) { COFF_SectionHeader *section_header = it.v.header; COFF_SectionFlags section_flags = *it.v.flags; if (section_flags & COFF_SectionFlag_LnkInfo) { continue; } - if (section_flags & COFF_SectionFlag_LnkRemove) { continue; } + if (section_flags & COFF_SectionFlag_LnkRemove) { + // exception: ICF-folded functions' .debug$S stays dead but its Lines are merged into the + // module bound to the leader RVA -- patch it so those Lines carry real addresses + if (!(obj->icf_lines_only && obj->icf_lines_only[it.v.section_number])) { continue; } + } if (section_flags & COFF_SectionFlag_CntUninitializedData) { continue; } - // get section bytes (special case debug info because it is not copied to the image) - Rng1U64 section_frange = rng_1u64(section_header->foff, section_header->foff + section_header->fsize); - String8 section_data = section_flags & LNK_SECTION_FLAG_DEBUG ? lnk_obj_section_data_from_number(obj, it.v.section_number) : str8_substr(task->image_data, section_frange); - - // apply relocs COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); - for EachIndex(reloc_idx, relocs.count) { - COFF_Reloc *reloc = &relocs.v[reloc_idx]; - // error check relocation - if (obj->coff.header.machine == COFF_MachineType_X64) { - if (reloc->type > COFF_Reloc_X64_Last) { - lnk_error_obj(LNK_Error_IllegalRelocation, obj, "unknown relocation type 0x%x", reloc->type); - } - } else if (obj->coff.header.machine != COFF_MachineType_Unknown) { - lnk_not_implemented("relocation patching is not implemented for %S", coff_string_from_machine_type(obj->coff.header.machine)); - continue; - } - - // compute virtual offsets - U64 reloc_voff = section_header->voff + reloc->apply_off; - - // compute symbol location values - U32 symbol_secnum = 0; - U32 symbol_secoff = 0; - S64 symbol_voff = 0; - { - COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, reloc->isymbol); - COFF_SymbolValueInterpType interp = coff_interp_from_parsed_symbol(symbol); - if (interp == COFF_SymbolValueInterp_Regular) { - if (symbol.section_number == lnk_obj_get_removed_section_number(obj)) { - if (~section_flags & LNK_SECTION_FLAG_DEBUG) { - String8 sect_name = coff_name_from_section_header(string_table, section_header); - String8 symbol_name = lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol); - lnk_error_obj(LNK_Error_RelocationAgainstRemovedSection, obj, "relocating against symbol that is in a removed section (symbol: %S, reloc-section: %S 0x%llx, reloc-index: 0x%llx)", symbol_name, sect_name, it.v.section_number, reloc_idx); - } - continue; - } - symbol_secnum = symbol.section_number; - symbol_secoff = symbol.value; - symbol_voff = safe_cast_u32((U64)task->image_section_table[symbol.section_number]->voff + (U64)symbol_secoff); - } else if (interp == COFF_SymbolValueInterp_Abs) { - // There aren't enough bits in COFF symbol to store full image base address, - // so we special case __ImageBase. A better solution would be to add - // a 64-bit symbol format to COFF. - if (str8_match(lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol), str8_lit("__ImageBase"), 0)) { - symbol.value = task->image_base; - } - symbol_secnum = 0; - symbol_secoff = 0; - symbol_voff = (S64)symbol.value - (S64)task->image_base; - } else if (interp == COFF_SymbolValueInterp_Weak) { - // unresolved weak - } else if (interp == COFF_SymbolValueInterp_Undefined) { - // unresolved undefined - } else { - InvalidPath; - } - } - - // pick reloc value - COFF_RelocValue reloc_value = {0}; - switch (obj_header.machine) { - case COFF_MachineType_Unknown: {} break; - case COFF_MachineType_X64: { reloc_value = coff_pick_reloc_value_x64(reloc->type, task->image_base, reloc_voff, symbol_secnum, symbol_secoff, symbol_voff); } break; - default: { NotImplemented; } break; - } - - // read addend - Assert(reloc_value.size <= section_data.size); - U64 raw_addend = 0; - str8_deserial_read(section_data, reloc->apply_off, &raw_addend, reloc_value.size, 1); - - // compute new reloc value - S64 addend = extend_sign64(raw_addend, reloc_value.size); - U64 reloc_result = reloc_value.value + addend; - - // commit new reloc value - MemoryCopy(section_data.str + reloc->apply_off, &reloc_result, reloc_value.size); + // get section bytes (special case debug info because it is not copied to the image) + Rng1U64 section_frange = rng_1u64(section_header->foff, section_header->foff + section_header->fsize); + String8 section_data; + if (section_flags & LNK_SECTION_FLAG_DEBUG) { + // Objs excluded from debug output have no later consumer for these bytes. + if (obj->exclude_from_debug_info) { continue; } + + // A relocation-free debug section can stay on the clean input view. + if (relocs.count == 0) { continue; } + + // With the default streaming window, .debug$S is reconstructed on demand during + // module writing, so no persistent patched copy is needed. + if (g_debug_s_window && str8_match(coff_name_from_section_header(string_table, section_header), str8_lit(".debug$S"), 0)) { continue; } + + if (obj->section_data_copies == 0) { + obj->section_data_copies = push_array(arena, String8, obj->coff.sections.count_no_null + 1); + } + String8 src = str8_substr(obj->coff.data, section_frange); + U8 *copy = push_array_no_zero(g_sect_copy_arenas[worker_id], U8, src.size); + MemoryCopy(copy, src.str, src.size); + obj->section_data_copies[it.v.section_number] = str8(copy, src.size); + section_data = obj->section_data_copies[it.v.section_number]; + } else { + section_data = str8_substr(task->image_data, section_frange); } + + // apply relocs (factored: shared with the P3.3 module-write window fill) + lnk_obj_apply_relocs_to_buffer(obj, it.v.section_number, section_header, section_data, task->image_base, task->image_section_table); } ProfEnd(); @@ -5090,6 +5670,42 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) ProfEnd(); } +internal +void +lnk_gather_base_reloc_candidate(Arena *arena, LNK_BaseRelocsTask *task, LNK_Obj *obj, + HashTable *page_ht, LNK_BaseRelocPageList *pages, + U32 sect_idx, U32 apply_off, U32 isymbol, U64 is_addr) +{ + COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, isymbol); + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol); + if (symbol_interp == COFF_SymbolValueInterp_Abs) { return; } + + U64 reloc_voff = obj->coff.sections.headers[sect_idx + 1].voff + apply_off; + U64 page_voff = AlignDownPow2(reloc_voff, task->page_size); + LNK_BaseRelocPageNode *page = hash_table_search_u64_raw(page_ht, page_voff); + if (page == 0) { + page = push_array(arena, LNK_BaseRelocPageNode, 1); + page->v.voff = page_voff; + page->v.entries_addr32 = push_array(arena, U64List, 1); + page->v.entries_addr64 = push_array(arena, U64List, 1); + SLLQueuePush(pages->first, pages->last, page); + pages->count += 1; + hash_table_push_u64_raw(arena, page_ht, page_voff, page); + } + + switch (is_addr) { + case 4: { + if (task->is_large_addr_aware) { + lnk_error_obj(LNK_Error_LargeAddrAwareRequired, obj, "found out of range ADDR32 relocation for '%S', link with /LARGEADDRESSAWARE:NO", lnk_symbol_name_from_coff_symbol_idx(obj, isymbol)); + } else { + u64_list_push(arena, page->v.entries_addr32, reloc_voff); + } + } break; + case 8: { u64_list_push(arena, page->v.entries_addr64, reloc_voff); } break; + default: { InvalidPath; } break; + } +} + internal THREAD_POOL_TASK_FUNC(lnk_gather_base_reloc_pages_task) { @@ -5099,7 +5715,25 @@ THREAD_POOL_TASK_FUNC(lnk_gather_base_reloc_pages_task) LNK_Obj *obj = task->gather.objs[task_id]; ProfBeginV("%S", obj->path); + LNK_CObjBaseRelocView compressed_index = {0}; + if (lnk_compressed_obj_base_reloc_index(obj->compressed_obj, &compressed_index)) { + for EachIndex(i, compressed_index.count) { + LNK_CObjBaseRelocEntry *entry = &compressed_index.v[i]; + if (entry->sect_idx >= obj->coff.sections.count_no_null || entry->isymbol >= obj->coff.header.symbol_count || + (entry->addr_size != 4 && entry->addr_size != 8)) { + lnk_error_obj(LNK_Error_IllData, obj, "invalid compressed base relocation sidecar entry"); + continue; + } + if (obj->coff.sections.headers[entry->sect_idx + 1].flags & COFF_SectionFlag_LnkRemove) { continue; } + lnk_gather_base_reloc_candidate(arena, task, obj, page_ht, pages, entry->sect_idx, + entry->apply_off, entry->isymbol, entry->addr_size); + } + ProfEnd(); + return; + } + for LNK_EachCoffSection(it, obj) { + U32 sect_idx = safe_cast_u32(it.v.section_number - 1); COFF_SectionHeader *sect_header = it.v.header; if (*it.v.flags & COFF_SectionFlag_LnkRemove) { continue; } @@ -5107,44 +5741,10 @@ THREAD_POOL_TASK_FUNC(lnk_gather_base_reloc_pages_task) for EachIndex(reloc_idx, relocs.count) { COFF_Reloc *r = &relocs.v[reloc_idx]; - COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, r->isymbol); - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol); - if (symbol_interp == COFF_SymbolValueInterp_Abs) { continue; } - U64 is_addr = coff_is_addr_reloc(obj->coff.header.machine, r->type); if (is_addr == 0) { continue; } - - U64 reloc_voff = sect_header->voff + r->apply_off; - U64 page_voff = AlignDownPow2(reloc_voff, task->page_size); - LNK_BaseRelocPageNode *page = hash_table_search_u64_raw(page_ht, page_voff); - if (page == 0) { - // fill out page - page = push_array(arena, LNK_BaseRelocPageNode, 1); - page->v.voff = page_voff; - page->v.entries_addr32 = push_array(arena, U64List, 1); - page->v.entries_addr64 = push_array(arena, U64List, 1); - - // push page - SLLQueuePush(pages->first, pages->last, page); - pages->count += 1; - - // register page voff - hash_table_push_u64_raw(arena, page_ht, page_voff, page); - } - - switch (is_addr) { - case 4: { - if (task->is_large_addr_aware) { - lnk_error_obj(LNK_Error_LargeAddrAwareRequired, obj, "found out of range ADDR32 relocation for '%S', link with /LARGEADDRESSAWARE:NO", lnk_symbol_name_from_coff_symbol_idx(obj, r->isymbol)); - } else { - u64_list_push(arena, page->v.entries_addr32, reloc_voff); - } - } break; - case 8: { - u64_list_push(arena, page->v.entries_addr64, reloc_voff); - } break; - default: { InvalidPath; } break; - } + lnk_gather_base_reloc_candidate(arena, task, obj, page_ht, pages, (U32)sect_idx, + r->apply_off, r->isymbol, is_addr); } } @@ -5635,13 +6235,20 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfScope("Gather Sections") { TP_Temp temp = tp_temp_begin(arena); + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier), so under + // /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- a plain + // tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane ranges/defns so everything agrees on the width. + U32 C = tp_barrier_begin(tp); task.u.gather_sects.arena = arena->v[0]; - task.u.gather_sects.ranges = tp_divide_work(arena->v[0], objs_count, tp->worker_count); - task.u.gather_sects.defns = push_array(arena->v[0], HashTable *, tp->worker_count); - tp_for_parallel_prof(tp, arena, tp->worker_count, lnk_gather_sections_task, &task, "Gather Sections"); + task.u.gather_sects.ranges = tp_divide_work(arena->v[0], objs_count, C); + task.u.gather_sects.defns = push_array(arena->v[0], HashTable *, C); + ProfBegin("Gather Sections"); + tp_for_parallel_reserve(tp, arena, C, lnk_gather_sections_task, &task); // BARRIER pass (path B) + ProfEnd(); + tp_barrier_end(tp); tp_temp_end(temp); } - // ensure determinism by sorting section contribs in chunks by input index ProfScope("Sort Section Contribs") { @@ -5881,16 +6488,56 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfBeginV("Alloc Image Buffer [%M]", lnk_section_table_total_fsize(sectab)); image_data.size = lnk_section_table_total_fsize(sectab) + image_string_table.total_size; - image_data.str = push_array_no_zero(arena->v[0], U8, image_data.size); + // Standalone reservation (not the shared link arena) so it can be released the instant the image + // is written to disk -- VirtualFree returns fast and the kernel zeroes this ~1GB on its background + // thread, overlapping the rest of the run, instead of in the single-threaded exit rundown. + image_data.str = reserve_memory(image_data.size); + commit_memory(image_data.str, image_data.size); ProfEnd(); ProfBegin("Fill Align Bytes"); - for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { - LNK_Section *sect = §_n->data; - ProfBeginV("Section: %S Size: %M", sect->name, sect->fsize); - U8 fill_byte = sect->flags & COFF_SectionFlag_CntCode ? coff_code_align_byte_from_machine(config->machine) : 0; - MemorySet(image_data.str + sect->foff, fill_byte, sect->fsize); - ProfEnd(); + { + // This is the first touch of the freshly committed ~image-size buffer: every + // page is a demand-zero fault. Serial on main, that soft-fault storm (plus the + // stream-set itself) parks all workers for the duration; range-split it across + // the pool instead. Split points are PAGE-ALIGNED in the image buffer (the + // reservation is page-aligned) so no two workers ever touch the same 4K page. + // Writes are value-identical to the serial loop and byte-disjoint -> byte-safe. + Temp fill_temp = temp_begin(scratch.arena); + U64 range_quantum = MB(4); + + // upper bound on range count + U64 range_cap = 0; + for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { + range_cap += CeilIntegerDiv(sect_n->data.fsize, range_quantum) + 1; + } + + LNK_FillAlignRange *ranges = push_array_no_zero(fill_temp.arena, LNK_FillAlignRange, range_cap); + U64 range_count = 0; + for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { + LNK_Section *sect = §_n->data; + U8 fill_byte = sect->flags & COFF_SectionFlag_CntCode ? coff_code_align_byte_from_machine(config->machine) : 0; + U64 pos = sect->foff; + U64 end = sect->foff + sect->fsize; + for (; pos < end; ) { + U64 next = AlignDownPow2(pos + range_quantum, KB(4)); + next = ClampTop(next, end); + if (next <= pos) { next = end; } + Assert(range_count < range_cap); + LNK_FillAlignRange *range = &ranges[range_count++]; + range->dst = image_data.str + pos; + range->size = next - pos; + range->byte = fill_byte; + pos = next; + } + } + + // write-once into the image buffer -> stream past the cache (see lnk_stream_set); + // each task sfences its own NT stores before signalling completion, so after the + // join every fill below is globally visible to the contrib-fill / reloc passes + tp_for_parallel(tp, 0, range_count, lnk_fill_align_bytes_task, ranges); + + temp_end(fill_temp); } ProfEnd(); @@ -5955,8 +6602,23 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT // patch relocs { + // Streaming-ring P3.3: default to windowed $S consumption (patcher skips $S copies; + // the module-write visit re-reads + patches into a reused per-worker window). + // /OPT:GCTYPES needs the persistent patched+fixed-up $S backing (it reads and rewrites + // type indices in place between the merge and the PDB build) -- keep the old copy-based + // path wholesale there. + g_debug_s_window = (config->opt_gc_types != LNK_SwitchState_Yes); + + // dedicated per-worker arenas for the patched debug-section copies: free-list + // block reuse keeps the pages warm (a raw reserve+commit per copy paid ~11.6GB of + // fresh zero-page faults per link at FN scale), and lnk_build_pdb hands the whole + // set back with arena_release after the last $S reader + g_sect_copy_arena_count = tp->worker_count; + g_sect_copy_arenas = push_array(arena->v[0], Arena *, g_sect_copy_arena_count); + for EachIndex(i, g_sect_copy_arena_count) { g_sect_copy_arenas[i] = arena_alloc(.name = "SECT_DATA_COPIES"); } + LNK_ObjRelocPatcher task = { .image_data = image_data, .objs = objs, .image_base = pe.image_base, .image_section_table = image_section_table }; - tp_for_parallel_prof(tp, 0, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); + tp_for_parallel_prof(tp, arena, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); // arena: the per-obj section_data_copies String8 tables } // patch load config @@ -6409,11 +7071,269 @@ internal void lnk_write_thread(void *raw_ctx) { ProfBeginFunction(); + lnk_summary_phase_begin(LNK_SummaryPhase_Write); LNK_WriteThreadContext *ctx = raw_ctx; lnk_write_data_to_file_path(ctx->path, ctx->temp_path, ctx->data); + lnk_summary_phase_end(LNK_SummaryPhase_Write); ProfEnd(); } +//////////////////////////////////////////////////////////////////////////////// +//~ One-line end-of-link summary (always on; production triage). Everything +// needed at print time is stashed in this global as the link progresses, so +// the line can be emitted best-effort from ANY exit path (lnk_exit on error, +// entry_point on success) with whatever was known by then. + +typedef struct LNK_SummaryInfo +{ + volatile U32 printed; // print-exactly-once latch + U64 start_us; // set first thing in entry_point + U64 t0_ms; // UTC ms epoch at link start (t1 is stamped at print time) + U64 worker_count; + U64 objs_count; + U64 input_bytes; // sum of obj data sizes (lib members count their slice) + U64 libs_count; + // physical-memory samples (GlobalMemoryStatusEx): storm triage -- prod storms + // show pdb-phase kernel time exploding 54x for the same fault count, fitting + // free-list exhaustion / page-repurpose; these 3 samples prove/refute that + U64 mem_avail_t0; // ullAvailPhys at link start + U64 mem_avail_pdb; // ullAvailPhys at pdb-phase start (0 = phase never ran) + U32 mem_load_max; // max dwMemoryLoad seen across the samples + // name COPIES: config strings parsed out of an @rsp point into the response + // file buffer, whose scratch dies right after config parse -- capture the + // bytes here instead of keeping String8s into freed memory + U64 out_name_size; + U64 pool_name_size; // non-zero => /RAD_SHARED_THREAD_POOL + U8 out_name [128]; + U8 pool_name[128]; +} LNK_SummaryInfo; + +global LNK_SummaryInfo g_summary_info; + +internal void +lnk_summary_copy_name(U8 *dst, U64 dst_cap, U64 *dst_size_out, String8 name) +{ + U64 size = Min(name.size, dst_cap); + MemoryCopy(dst, name.str, size); + *dst_size_out = size; +} + +internal U64 +lnk_summary_us_from_timer(LNK_TimerType timer) +{ + // guard against a fatal exit mid-phase (begin stamped, end still zero) + return g_timers[timer].end > g_timers[timer].begin ? g_timers[timer].end - g_timers[timer].begin : 0; +} + +internal LNK_SummaryCounters +lnk_summary_counters_from_timer(LNK_TimerType timer) +{ + LNK_SummaryCounters zero = {0}; + // same mid-phase guard as lnk_summary_us_from_timer + if (g_timers[timer].end <= g_timers[timer].begin) { return zero; } + return lnk_summary_counters_sub_sat(g_timer_counters_end[timer], g_timer_counters_begin[timer]); +} + +// one GlobalMemoryStatusEx sample for the summary line: returns available +// physical bytes and folds dwMemoryLoad into the running max. 1 syscall per +// call, called 3x per link (link start, pdb-phase start, print time). +internal U64 +lnk_summary_sample_mem(void) +{ + U64 avail = 0; +#if OS_WINDOWS + MEMORYSTATUSEX msx = { sizeof(msx) }; + if (GlobalMemoryStatusEx(&msx)) { + avail = msx.ullAvailPhys; + if (msx.dwMemoryLoad > g_summary_info.mem_load_max) { g_summary_info.mem_load_max = msx.dwMemoryLoad; } + } +#endif + return avail; +} + +internal U64 +lnk_summary_utc_ms(void) +{ +#if OS_WINDOWS + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + U64 t100 = ((U64)ft.dwHighDateTime << 32) | ft.dwLowDateTime; + return (t100 - 116444736000000000ULL) / 10000; // FILETIME epoch -> unix ms epoch +#else + return 0; +#endif +} + +// one phase bucket -> "wall-ms/user-ms/kernel-ms/faults-K" (process-wide deltas +// at the bucket's boundaries; user can exceed wall on parallel phases, and a +// bucket that overlaps another thread's work counts that work too) +internal String8 +lnk_summary_str_from_counters(Arena *arena, LNK_SummaryCounters c) +{ + return push_str8f(arena, "%llu/%llu/%llu/%llu", c.wall_us / 1000, c.user_us / 1000, c.kern_us / 1000, c.faults / 1000); +} + +internal void +lnk_print_summary(int exit_code) +{ + // run exactly once, no matter which exit path gets here first + if (ins_atomic_u32_eval_cond_assign(&g_summary_info.printed, 1, 0) != 0) { + return; + } + + // detach from the shared-pool cross-process counter on every exit path, even + // when the summary line is off -- the linker leaves through _exit and never + // runs tp_release, so this is the only place the counter gets decremented + F64 pool_grant_avg = 0, pool_park_seconds = 0; + U32 pool_procs_now = 0, pool_procs_peak = 0; + B32 pool_on = (g_summary_info.pool_name_size > 0); + if (pool_on) { + tp_stats_snapshot(&pool_grant_avg, &pool_park_seconds); + tp_procs_snapshot(&pool_procs_now, &pool_procs_peak); + tp_procs_detach(); + } + + // the line itself is opt-in (/RAD_LOG:Summary) -- always-on turned out to be + // noise in build logs; farm convoy triage passes the switch explicitly + if (!lnk_get_log_status(LNK_Log_Summary)) { + return; + } + + Temp scratch = scratch_begin(0, 0); + + F64 wall = g_summary_info.start_us ? (F64)(now_time_us() - g_summary_info.start_us) / 1000000.0 : 0; + + // process CPU + memory counters + F64 user_time = 0, kernel_time = 0, peak_ws_gib = 0, page_faults_m = 0, peak_commit_gib = 0; + U32 cow_promoted_pages = 0; +#if OS_WINDOWS + { + FILETIME create_ft, exit_ft, kernel_ft, user_ft; + if (GetProcessTimes(GetCurrentProcess(), &create_ft, &exit_ft, &kernel_ft, &user_ft)) { + user_time = (F64)(((U64)user_ft.dwHighDateTime << 32) | user_ft.dwLowDateTime) / 10000000.0; + kernel_time = (F64)(((U64)kernel_ft.dwHighDateTime << 32) | kernel_ft.dwLowDateTime) / 10000000.0; + } + PROCESS_MEMORY_COUNTERS pmc = { (DWORD)sizeof(pmc) }; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) { + peak_ws_gib = (F64)pmc.PeakWorkingSetSize / (F64)GB(1); + page_faults_m = (F64)pmc.PageFaultCount / 1000000.0; + // peak pagefile-backed commit charge -- the number build-farm memory admission + // sees; with read-only input views this tracks ws minus the mapped input set + peak_commit_gib = (F64)pmc.PeakPagefileUsage / (F64)GB(1); + } + cow_promoted_pages = (U32)g_lnk_cow_promoted_pages; + } +#endif + + // process IO totals: hard page-ins on mapped inputs surface as read bytes, + // UBA-detoured output writes as write bytes + U64 io_read_mb = 0, io_write_mb = 0; +#if OS_WINDOWS + { + IO_COUNTERS ioc = {0}; + if (GetProcessIoCounters(GetCurrentProcess(), &ioc)) { + io_read_mb = ioc.ReadTransferCount / MB(1); + io_write_mb = ioc.WriteTransferCount / MB(1); + } + } +#endif + + // phase triplets. img/dbg/pdb come from the /RAD_LOG:TIMERS stamps; dbg is + // the debug-info umbrella minus the PDB/RDI sub-phases it contains. + LNK_SummaryCounters img_c = lnk_summary_counters_from_timer(LNK_Timer_Image); + LNK_SummaryCounters pdb_c = lnk_summary_counters_from_timer(LNK_Timer_Pdb); + LNK_SummaryCounters rdi_c = lnk_summary_counters_from_timer(LNK_Timer_Rdi); + LNK_SummaryCounters dbg_c = lnk_summary_counters_sub_sat(lnk_summary_counters_sub_sat(lnk_summary_counters_from_timer(LNK_Timer_Debug), pdb_c), rdi_c); + + // residual catch-alls: umbrella bucket minus the sum of its printed + // sub-buckets, clamped at 0 per field (a /PDBSTRIPPED link runs the pdb + // sub-phases a second time OUTSIDE the Timer_Pdb bracket, which can push the + // sub-bucket sum past the umbrella -- clamp instead of printing garbage). + // Storm triage: prod shows the pdbg sub-buckets covering only ~19% of pdb + // kernel time in a storm window vs ~96% locally -- other= pins the + // uncovered span without waiting for a local repro. + LNK_SummaryCounters pdb_other, dbg_other; + { + LNK_SummaryCounters pdbg_sum = g_summary_phase[LNK_SummaryPhase_PdbGsi]; + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbHsh]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbIni]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbSym]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbMod]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbTpi]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbStr]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbSc]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbMsf]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbWr]); + pdb_other = lnk_summary_counters_sub_sat(pdb_c, pdbg_sum); + + LNK_SummaryCounters dbgg_sum = lnk_summary_counters_add(g_summary_phase[LNK_SummaryPhase_DbgMcvi], g_summary_phase[LNK_SummaryPhase_DbgMerge]); + dbg_other = lnk_summary_counters_sub_sat(dbg_c, dbgg_sum); + } + + // governor stats snapshotted above, before the detach + String8 pool_stats = str8_zero(); + if (pool_on) { + pool_stats = push_str8f(scratch.arena, " pool=%S grant_avg=%.1f park=%.1f procs=%u/%u", + str8(g_summary_info.pool_name, g_summary_info.pool_name_size), pool_grant_avg, pool_park_seconds, pool_procs_now, pool_procs_peak); + } + + // final memory sample (t1) -- 3rd and last GlobalMemoryStatusEx of the link + U64 mem_avail_t1 = lnk_summary_sample_mem(); + + lnk_fprintf(stdout, + "[radlink summary] v=3 out=%S exit=%d t0=%llu t1=%llu wall=%.1f user=%.1f kern=%.1f ws=%.1fG cm=%.1fG cowp=%u pf=%.1fM io=%llu/%lluMB mem=%.1f/%.1f/%.1f/%u workers=%llu%S" + " in=%lluo/%.1fG libs=%llu" + " ph[inp=%S res=%S icf=%S ref=%S img=%S dbg=%S pdb=%S wr=%S]" + " dbgg[mcvi=%S merge=%S other=%S]" + " pdbg[hsh=%S ini=%S gsi=%S sym=%S mod=%S tpi=%S str=%S sc=%S msf=%S wr=%S other=%S]\n", + g_summary_info.out_name_size ? str8(g_summary_info.out_name, g_summary_info.out_name_size) : str8_lit("-"), + exit_code, + g_summary_info.t0_ms, + lnk_summary_utc_ms(), + wall, + user_time, + kernel_time, + peak_ws_gib, + peak_commit_gib, + cow_promoted_pages, + page_faults_m, + io_read_mb, + io_write_mb, + (F64)g_summary_info.mem_avail_t0 / (F64)GB(1), + (F64)g_summary_info.mem_avail_pdb / (F64)GB(1), + (F64)mem_avail_t1 / (F64)GB(1), + g_summary_info.mem_load_max, + g_summary_info.worker_count, + pool_stats, + g_summary_info.objs_count, + (F64)g_summary_info.input_bytes / (F64)GB(1), + g_summary_info.libs_count, + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Input]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Resolve]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Icf]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Ref]), + lnk_summary_str_from_counters(scratch.arena, img_c), + lnk_summary_str_from_counters(scratch.arena, dbg_c), + lnk_summary_str_from_counters(scratch.arena, pdb_c), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Write]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_DbgMcvi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_DbgMerge]), + lnk_summary_str_from_counters(scratch.arena, dbg_other), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbHsh]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbIni]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbGsi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbSym]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbMod]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbTpi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbStr]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbSc]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbMsf]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbWr]), + lnk_summary_str_from_counters(scratch.arena, pdb_other)); + + scratch_end(scratch); +} + internal void lnk_log_timers(void) { @@ -6443,10 +7363,105 @@ lnk_log_timers(void) StringJoin new_line_join = { str8_lit_comp(""), str8_lit_comp("\n"), str8_lit_comp("") }; String8 output = str8_list_join(scratch.arena, &output_list, &new_line_join); lnk_log(LNK_Log_Timers, "%S\n", output); - + + // Diagnostic: when RADLINK_PHASE_LOG is set, also write machine-parseable raw + // per-phase micros to that file (for automated perf A/B). Env-unset -> no-op, + // so normal/validation links are byte-identical; this never touches DLL/PDB bytes. + char *phase_log_path = getenv("RADLINK_PHASE_LOG"); + if (phase_log_path != 0 && phase_log_path[0] != 0) { + String8List raw_list = {0}; + for (U64 i = 0; i < LNK_Timer_Count; ++i) { + str8_list_pushf(scratch.arena, &raw_list, "%S %llu\n", lnk_string_from_timer_type(i), g_timers[i].end - g_timers[i].begin); + } + str8_list_pushf(scratch.arena, &raw_list, "TOTAL %llu\n", total_build_time_micro); + String8 raw_str = str8_list_join(scratch.arena, &raw_list, 0); + lnk_write_data_to_file_path(str8_cstring(phase_log_path), str8_zero(), raw_str); + } + scratch_end(scratch); } +// scratch free-list blocks detached during the decommit pass, released on a +// background thread (see lnk_scratch_decommit_worker) +global Arena *g_detached_scratch_blocks = 0; +global Thread g_scratch_freelist_reaper = {0}; + +internal +THREAD_POOL_TASK_FUNC(lnk_scratch_decommit_worker) +{ + // Each worker decommits the committed-but-unused pages of its OWN equipped + // tctx scratch arenas. Runs on the worker thread, so tctx_selected() yields + // that worker's scratch. No cross-thread arena access. + // + // The barrier (dispatched with task_count == worker_count) guarantees every + // worker runs the body exactly once -- otherwise the work-stealing loop could + // let one fast worker grab several tasks and leave other workers' scratch + // committed. All worker_count threads are woken, so all must reach the barrier. + // + // NOTE(perf): redistributing these decommits in chunks across the pool does + // NOT help: MEM_DECOMMIT serializes in the kernel on the process address-space + // lock (~14 GB/s aggregate no matter the thread count; measured 9.3 GiB in + // 677 ms chunked-parallel vs ~500 ms with this per-worker scheme). And handing + // the ACTIVE-CHAIN decommit to a background thread is UNSAFE here: workers push + // to these scratch arenas as soon as the PDB build starts, and a push would + // re-commit pages that the background decommit then rips out. + // + // The FREE-LIST blocks are different: they hold no live data and are only + // touched again when a grow pops them. On the editor link ~84% of the + // decommitted bytes (9.4 of 11.3 GiB) sit in free-list blocks, so instead of + // decommitting them here (serialized kernel work on the critical path), each + // worker DETACHES its arenas' free chains (pointer ops, same thread => safe) + // onto a global list that a background thread releases while the PDB build + // runs. A post-detach grow simply sees an empty free list and reserves a + // fresh block -- same cost as the re-commit it would have paid anyway. + TCTX *tctx = tctx_selected(); + for EachIndex(arena_idx, ArrayCount(tctx->arenas)) { + Arena *arena = tctx->arenas[arena_idx]; + if (arena == 0) { continue; } +#if ARENA_FREE_LIST + // detach this arena's free chain and publish the blocks for background release + for (Arena *block = arena->free_last, *block_next = 0; block != 0; block = block_next) { + block_next = block->prev; + for (;;) { + Arena *head = (Arena *)ins_atomic_u64_eval(&g_detached_scratch_blocks); + block->prev = head; + if ((Arena *)ins_atomic_u64_eval_cond_assign((U64 *)&g_detached_scratch_blocks, (U64)block, (U64)head) == head) { break; } + } + } + arena->free_last = 0; +#endif + // decommit the committed-but-unused pages above the live pos (active chain) + arena_decommit_unused(arena); + } + barrier_wait(tp->barrier); +} + +// Releases the scratch free-list blocks detached by lnk_scratch_decommit_worker. +// Runs in the background: MEM_RELEASE serializes on the process address-space +// lock in the kernel, so on the main thread this would extend the decommit +// window 1:1; off the main thread it overlaps the PDB build. +internal void +lnk_detached_scratch_release_thread(void *raw) +{ + ProfBeginFunction(); + U64 begin_us = now_time_us(); + + U64 released_bytes = 0; + U64 released_count = 0; + Arena *chain = (Arena *)ins_atomic_u64_eval_assign((U64 *)&g_detached_scratch_blocks, 0); + for (Arena *block = chain, *block_next = 0; block != 0; block = block_next) { + block_next = block->prev; + released_bytes += block->cmt; + released_count += 1; + AsanUnpoisonMemoryRegion(block, block->cmt); + release_memory(block, block->res); + } + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu detached scratch blocks (%llu MiB committed) took %.2f ms (off main thread)", + released_count, released_bytes / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_p2r_worker) { @@ -6529,6 +7544,15 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_Obj **objs = lnk_array_from_obj_list(scratch.arena, link.objs); LNK_Lib **libs = lnk_array_from_lib_list(scratch.arena, link.libs); + // summary: input volume (lib members count their member slice) + { + U64 input_bytes = 0; + for EachIndex(obj_idx, objs_count) { input_bytes += objs[obj_idx]->coff.data.size; } + g_summary_info.objs_count = objs_count; + g_summary_info.libs_count = libs_count; + g_summary_info.input_bytes = input_bytes; + } + // // Layout Image // @@ -6581,8 +7605,55 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // CodeView // LNK_RRT_Array rrt_input = lnk_rrt_array_from_config(arena->v[0], config); + lnk_summary_phase_begin(LNK_SummaryPhase_DbgMcvi); LNK_CodeViewInput cv = lnk_make_code_view_input(tp, arena, config, debug_info_objs_count, debug_info_objs, rrt_input); + lnk_summary_phase_end(LNK_SummaryPhase_DbgMcvi); + lnk_summary_phase_begin(LNK_SummaryPhase_DbgMerge); LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); + lnk_summary_phase_end(LNK_SummaryPhase_DbgMerge); + + // Streaming-ring P2 slice A: $S TI/kind fixups are journaled in lnk_merge_types and + // normally replayed per obj into the window at the module-write visit + // (lnk_write_pdb_modules). /OPT:GCTYPES consumes fixed-up $S bytes in place right below + // (mark roots + compaction rewrite) and runs with the window disabled (persistent patched + // copies), so it still needs the eager whole-input replay. P3.3: a /PDBSTRIPPED-only + // build no longer takes this path -- its pre-build stripping loop re-reads each Symbols + // node through lnk_obj_window_debug_s, which applies relocs AND replays the journal per + // node, so nothing is written into the raw mapped views. + if (config->opt_gc_types == LNK_SwitchState_Yes) { + lnk_apply_debug_s_fixups_eager(tp, &cv); + } + + // prune merged types not reachable from any surviving symbol (PDB-size win). OFF by default: + // it removes types that a debugger can still legitimately cast to in the watch window + // (reachable-from-symbols is a subset of castable-types). Opt in with /OPT:GCTYPES. + if (config->opt_gc_types == LNK_SwitchState_Yes) { + lnk_gc_types(tp, arena->v[0], &cv, &cv_types); + } + + // merge-types reached the scratch high-water (~9GB of per-thread tctx scratch + // stays committed but idle). Release those unused scratch pages back to the OS + // before the PDB build re-grows, dropping the recorded peak working set. Each + // worker decommits its own scratch; do the main thread's scratch too. Only + // pages strictly above each arena's live `pos` are touched, so output stays + // byte-identical and the push path re-commits on demand during PDB build. + { + ProfBegin("Decommit Scratch"); + U64 decommit_begin_us = now_time_us(); + // task_count == worker_count + the in-worker barrier => every worker + // (worker 0 IS the main thread) runs exactly once, covering main's scratch. + tp_for_parallel_reserve(tp, 0, tp->worker_count, lnk_scratch_decommit_worker, 0); // BARRIER pass (path B) + if (g_detached_scratch_blocks != 0) { + g_scratch_freelist_reaper = thread_launch(lnk_detached_scratch_release_thread, 0); + } + lnk_log(LNK_Log_Timers, "[teardown] scratch decommit pass in %.2f ms", (F64)(now_time_us() - decommit_begin_us) / 1000.0); + ProfEnd(); + } + + // Type merging is the last bulk consumer of type payload pages. Apply the configured cache + // generation transition before PDB construction so type residency does not stack with the + // later GSI and module-stream allocations. + lnk_compressed_obj_trim_working_set(); // // Debug Info @@ -6592,8 +7663,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) if (config->debug_mode == LNK_DebugMode_Full || config->rad_debug == LNK_SwitchState_Yes) { LNK_FileArtifact pdb_artifact = {0}; { + g_summary_info.mem_avail_pdb = lnk_summary_sample_mem(); lnk_timer_begin(LNK_Timer_Pdb); - + lnk_summary_phase_begin(LNK_SummaryPhase_PdbHsh); if (config->pdb_hash_type_names != LNK_TypeNameHashMode_None) { lnk_replace_type_names_with_hashes(tp, arena, @@ -6603,10 +7675,12 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) config->pdb_hash_type_name_length, config->pdb_hash_type_name_map); } - + lnk_summary_phase_end(LNK_SummaryPhase_PdbHsh); pdb_writer.output_path = config->debug_mode == LNK_DebugMode_Full ? config->pdb_name : str8_zero(); pdb_writer.temp_output_path = config->debug_mode == LNK_DebugMode_Full ? config->temp_pdb_name : str8_zero(); - pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_writer, LNK_PDB_BuilderFlag_All); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); + pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_writer, LNK_PDB_BuilderFlag_All, inputer); + lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); lnk_timer_end(LNK_Timer_Pdb); } @@ -6615,7 +7689,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) lnk_timer_begin(LNK_Timer_Rdi); LNK_P2R p2r = { .config = config, .pdb_data = lnk_data_from_file_artifact(lnk_get_huge_arena(), &pdb_artifact), .image_data = image_ctx.image_data }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_p2r_worker, &p2r); + tp_for_parallel_reserve(tp, arena, tp->worker_count, lnk_p2r_worker, &p2r); // BARRIER pass (path B) String8List rdi_blobs = rdim_file_blobs_from_section_bundle(scratch.arena, &p2r.bake_results.section_bundle); lnk_write_data_list_to_file_path(config->rad_debug_name, config->temp_rad_debug_name, rdi_blobs); @@ -6628,13 +7702,36 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // stripped PDB // if (config->pdb_stripped_name.size != 0) { + // P3.3 fold: with the $S window enabled there are no patched copies and no persisted + // fixup replay -- the raw mapped Symbols bytes are pre-reloc and pre-TI-fixup. Re-read + // each obj's Symbols nodes through lnk_obj_window_debug_s (raw view -> window copy + + // relocs + journal replay: exactly the bytes the module-write pass consumed); the + // journal was kept alive across lnk_build_pdb for this (released below). The strip + // loop already copies every surviving record out, so the window is transient per obj. + // With the window disabled (/OPT:GCTYPES) the old flow is intact: free_sect_copies==0 + // kept the patched+eagerly-fixed copies alive and the nodes are read in place. + PE_BinInfo stripped_pe = {0}; + COFF_SectionHeader **stripped_image_sectab = 0; + Temp wscratch = scratch_begin(&scratch.arena, 1); + if (g_debug_s_window) { + stripped_pe = pe_bin_info_from_data(scratch.arena, image_ctx.image_data); + stripped_image_sectab = coff_section_table_from_data(scratch.arena, image_ctx.image_data, stripped_pe.section_table_range); + } + CV_DebugS *debug_s_arr = push_array(scratch.arena, CV_DebugS, cv.obj_count); for EachIndex(obj_idx, cv.obj_count) { + Temp wtemp = temp_begin(wscratch.arena); + CV_DebugS *debug_s_dst = &debug_s_arr[obj_idx]; - CV_DebugS *debug_s_src = &cv.debug_s_arr[obj_idx]; - String8List *dst = &debug_s_dst->data_list[CV_C13SubSectionIdxKind_Symbols]; - String8List *src = &debug_s_src->data_list[CV_C13SubSectionIdxKind_Symbols]; + CV_DebugS debug_s_win = {0}; + String8List *src; + if (g_debug_s_window) { + debug_s_win = lnk_obj_window_debug_s(wtemp.arena, &cv, obj_idx, stripped_pe.image_base, stripped_image_sectab, 1 /* symbols_only */); + src = cv_sub_section_ptr_from_debug_s(&debug_s_win, CV_C13SubSectionKind_Symbols); + } else { + src = cv_sub_section_ptr_from_debug_s(&cv.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); + } U64 proc_count = 0; U64 proc_size = 0; @@ -6667,18 +7764,28 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); if (symbol.kind == CV_SymKind_SKIP) { continue; } if (cv_is_lproc(symbol)) { - CV_SymProc32 *src_proc = str8_deserial_get_raw_ptr(symbol.data, 0, sizeof(*src_proc)); - memory_write32(&src_proc->itype, 0); // strip type index + // strip the type index in the DESTINATION copy -- the source $$S stays untouched + // (patching the source would dirty its private/CoW backing pages for no reason) + U64 rec_off = buffer_cursor; buffer_cursor += cv_write_symbol(buffer, buffer_cursor, buffer_size, &symbol, CV_SymbolAlign); + memory_write32(buffer + rec_off + sizeof(CV_SymbolHeader) + OffsetOf(CV_SymProc32, itype), 0); buffer_cursor += cv_write_symbol(buffer, buffer_cursor, buffer_size, &(CV_Symbol){ .kind = CV_SymKind_END }, CV_SymbolAlign); } } } Assert(buffer_cursor == buffer_size); - str8_list_push(scratch.arena, dst, str8(buffer, buffer_size)); + // synthesized bytes (TI-stripped copies): provenance marked synthetic + cv_debug_s_push_synthetic_sub_section(scratch.arena, debug_s_dst, CV_C13SubSectionKind_Symbols, str8(buffer, buffer_size)); } + + temp_end(wtemp); // window bytes are consumed (records copied into `buffer`) } + scratch_end(wscratch); + + // last $S-journal reader on the stripped path is done (lnk_build_pdb kept the journal + // alive when a /PDBSTRIPPED build follows); no-op when already consumed/never built + lnk_release_debug_s_fixup_journal(&cv); LNK_CodeViewInput stripped_cv = {0}; stripped_cv.config = config; @@ -6689,8 +7796,12 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) stripped_cv.debug_s_arr = debug_s_arr; stripped_cv.symbol_input_ranges = push_array(scratch.arena, Rng1U64, tp->worker_count); - LNK_FileArtifact pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, (LNK_PdbWriter){0}, LNK_PDB_BuilderFlag_All); + // inputer==0: never early-release from the stripped build (and the first build was + // already gated off by pdb_stripped_name) -- its window fills read the raw views + LNK_FileArtifact pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, (LNK_PdbWriter){0}, LNK_PDB_BuilderFlag_All, 0); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); lnk_write_data_list_to_file_path(config->pdb_stripped_name, str8f(scratch.arena, "%S.tmp", config->pdb_stripped_name), pdb_artifact.data); + lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); } lnk_timer_end(LNK_Timer_Debug); @@ -6700,7 +7811,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) #if OS_WINDOWS // for unexplained reasons, file mappings on Windows cause slow process exit times ProfBegin("Release Input File Maps"); - lnk_inputer_release_file_maps(tp, inputer); + lnk_inputer_release_file_maps(tp, config->debug_worker_cap, inputer); ProfEnd(); #endif @@ -6711,11 +7822,31 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // reap the background arena-release thread, if one is still in flight + if (g_arena_reaper_thread.u64[0] != 0) { + thread_join(g_arena_reaper_thread, max_U64); + MemoryZeroStruct(&g_arena_reaper_thread); + } + + // reap the background scratch free-list release thread, if one was launched + if (g_scratch_freelist_reaper.u64[0] != 0) { + thread_join(g_scratch_freelist_reaper, max_U64); + MemoryZeroStruct(&g_scratch_freelist_reaper); + } + + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it + // concurrently with the remaining work + exit, not single-threaded in the process rundown. + release_memory(image_ctx.image_data.str, image_ctx.image_data.size); // // Timers // - if (lnk_get_log_status(LNK_Log_Timers)) { - lnk_log_timers(); + { + lnk_compressed_obj_log_stats(); + lnk_obj_log_compressed_census(); + char *phase_log_env = getenv("RADLINK_PHASE_LOG"); + if (lnk_get_log_status(LNK_Log_Timers) || (phase_log_env != 0 && phase_log_env[0] != 0)) { + lnk_log_timers(); + } } scratch_end(scratch); @@ -6891,7 +8022,7 @@ lnk_run_type_server(TP_Context *tp, TP_Arena *arena, LNK_Config *config) ProfScope("Pack Type Data & Data Ranges") { LNK_RRTTypeDataSerializer task = { &cv_types, &rrt.type_data_raw, rrt.type_data_ranges }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_serialize_rrt_type_data_task, &task); + tp_for_parallel_reserve(tp, arena, tp->worker_count, lnk_serialize_rrt_type_data_task, &task); // BARRIER pass (path B) // pack type index ranges for EachIndex(i, CV_TypeIndexSource_COUNT) { @@ -6977,10 +8108,20 @@ internal void entry_point(CmdLine *cmdline) { Temp scratch = scratch_begin(0,0); + g_summary_info.start_us = now_time_us(); + g_summary_info.t0_ms = lnk_summary_utc_ms(); + g_summary_info.mem_avail_t0 = lnk_summary_sample_mem(); lnk_log_begin(); // init config from the command line LNK_Config *config = lnk_config_init(cmdline->argc, cmdline->argv); + lnk_compressed_obj_configure(config); + + // Snapshot summary identity immediately after command-line parsing, before + // pool initialization and later scratch allocations. + lnk_summary_copy_name(g_summary_info.out_name, sizeof(g_summary_info.out_name), &g_summary_info.out_name_size, str8_skip_last_slash(config->out_path)); + lnk_summary_copy_name(g_summary_info.pool_name, sizeof(g_summary_info.pool_name), &g_summary_info.pool_name_size, config->shared_thread_pool_name); + g_summary_info.worker_count = config->worker_count; if (lnk_get_log_status(LNK_Log_Debug)) { lnk_fprintf(stderr, "--------------------------------------------------------------------------------\n"); @@ -6999,6 +8140,8 @@ entry_point(CmdLine *cmdline) case LNK_BootMode_TypeServer: lnk_run_type_server(tp, tp_arena, config); break; } + lnk_print_summary(0); + lnk_log_end(); scratch_end(scratch); } diff --git a/src/linker/lnk.h b/src/linker/lnk.h index 59b3d9a27..b35a701c9 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -32,6 +32,9 @@ typedef struct LNK_Input { String8 path; String8 data; + String8 compressed_data; + LNK_CompressedObj *compressed_obj; + U8 *compressed_eager_base; B32 disallow; B32 is_thin; B32 owns_file_map; @@ -206,7 +209,6 @@ typedef struct LNK_BaseRelocPageArray typedef struct { B32 search_anti_deps; - B32 reset_search_cursor; LNK_Link *link; HashMap *imports_hm; LNK_SymbolTable *symtab; @@ -233,6 +235,13 @@ typedef struct COFF_SectionHeader **image_section_table; } LNK_ObjRelocPatcher; +// Per-worker arenas backing the patched debug-section copies (lnk_obj_reloc_patcher +// pushes on g_sect_copy_arenas[worker_id]; arena_alloc recycles free-list blocks so the +// pages are warm). Released wholesale in lnk_build_pdb after the last $S reader +// (module write + global-record materialize), gated off when /PDBSTRIPPED follows. +global Arena **g_sect_copy_arenas = 0; +global U64 g_sect_copy_arena_count = 0; + typedef struct { U64 page_size; @@ -378,7 +387,7 @@ internal LNK_Input * lnk_inputer_push_lib_thin(LNK_Inputer *inputer, LNK_Config internal B32 lnk_inputer_has_items(LNK_Inputer *inputer); internal LNK_InputPtrArray lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Flags io_flags, LNK_InputList *all_inputs, LNK_InputList *new_inputs); -internal void lnk_inputer_release_file_maps(TP_Context *tp, LNK_Inputer *inputer); +internal void lnk_inputer_release_file_maps(TP_Context *tp, U64 worker_cap, LNK_Inputer *inputer); // --- Link Context ------------------------------------------------------------ @@ -396,7 +405,9 @@ internal LNK_LinkResult lnk_link_image (TP_Context *tp, TP_Arena *arena, LNK_Con // --- Optimizations ----------------------------------------------------------- internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); -internal void lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); +internal void lnk_opt_icf(TP_Context *tp, Arena *perm, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); +internal U32 lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn); +internal void lnk_icf_mark_folded_lines(TP_Context *tp, TP_Arena *arena, LNK_Obj **objs, U64 objs_count); // --- Win32 Image ------------------------------------------------------------- @@ -409,3 +420,9 @@ internal LNK_ImageContext lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_C internal void lnk_log_link_stats(LNK_ObjList obj_list, LNK_LibList *lib_index, LNK_SectionTable *sectab); internal void lnk_log_timers(void); + +// One-line end-of-link summary for production triage (always on). Prints +// exactly once; safe to call from any exit path (values best-effort on early +// error exits). Defined in lnk.c; called from lnk_exit and entry_point. +internal void lnk_print_summary(int exit_code); + diff --git a/src/linker/lnk_compressed_obj.c b/src/linker/lnk_compressed_obj.c new file mode 100644 index 000000000..7178aa741 --- /dev/null +++ b/src/linker/lnk_compressed_obj.c @@ -0,0 +1,1867 @@ +#if OS_WINDOWS && defined(LNK_OODLE) + +#include "lnk_compressed_obj_format.h" +#include + +//////////////////////////////// +// Runtime state + +#define LNK_COBJ_MAX_INITIAL_CACHE_GIB 17u +#define LNK_COBJ_MAX_POST_CACHE_GIB 6u +#define LNK_COBJ_MAX_TOTAL_CACHE_GIB (LNK_COBJ_MAX_INITIAL_CACHE_GIB + LNK_COBJ_MAX_POST_CACHE_GIB) +#define LNK_COBJ_MIN_TOTAL_CACHE_GIB 4u +#define LNK_COBJ_REGION_CAP (1u << 20) +#define LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS 64u +#define LNK_COBJ_FILE_MAPPING_GRANULARITY KB(64) +#define LNK_COBJ_STALL_ABORT_US 30000000ull +#define LNK_COBJ_KNOWN_HEADER_FLAGS (LNK_COBJ_FLAG_TYPE_INDEX | \ + LNK_COBJ_FLAG_UDT_HASH_INDEX | \ + LNK_COBJ_FLAG_BASE_RELOC_INDEX | \ + LNK_COBJ_FLAG_PORTABLE_RAW_MAP | \ + LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR | \ + LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2 | \ + LNK_COBJ_FLAG_DEBUG_S_INDEX | \ + LNK_COBJ_FLAG_DEBUG_S_SUMMARY | \ + LNK_COBJ_TYPE_INDEX_COUNT_MASK) + +// Every compressed input reserves a virtual range matching the original OBJ. Raw segments map +// directly from the portable file. Compressed segments remain placeholders until the vectored +// exception handler decodes them into a bounded pagefile-backed cache and maps the cache slot at +// the faulting logical address. The linker therefore keeps its ordinary pointer-based OBJ parser. + +typedef enum LNK_CObjSegState +{ + LNK_CObjSegState_Empty, + LNK_CObjSegState_Loading, + LNK_CObjSegState_Ready, + LNK_CObjSegState_Pinned, + LNK_CObjSegState_Evicting, +} LNK_CObjSegState; + +typedef struct LNK_CObjRuntimeSegment +{ + SRWLOCK lock; + volatile LONG state; + U32 slot_idx; + U32 decode_count; + B32 isolated_placeholder; + U8 *address; +} LNK_CObjRuntimeSegment; + +struct LNK_CompressedObj +{ + U8 *base; + U64 reserve_size; + U64 raw_size; + U32 segment_size; + U32 segment_count; + String8 mapped_file; + LNK_CObjHeader *header; + LNK_CObjSegment *directory; + LNK_CObjTypeIndex *type_indices; + LNK_CObjUdtHashIndex *udt_hash_indices; + LNK_CObjBaseRelocEntry *base_reloc_entries; + U32 base_reloc_entry_count; + B32 has_base_reloc_index; + LNK_CObjDebugSEntry *debug_s_entries; + LNK_CObjDebugSSummary *debug_s_summaries; + U32 debug_s_entry_count; + U32 type_index_count; + LNK_CObjRuntimeSegment *segments; + HANDLE portable_mapping; + SRWLOCK placeholder_lock; + volatile LONG active; +}; + +typedef struct LNK_CObjWriteGroup +{ + SRWLOCK lock; + U8 *view; + volatile LONG completed_count; + U32 slot_count; + volatile LONG sealed; +} LNK_CObjWriteGroup; + +typedef struct LNK_CObjCache +{ + SRWLOCK lock; + HANDLE slot_mapping; + LNK_CObjWriteGroup *write_groups; + U64 write_group_count; + U32 write_group_slot_count; + U64 slot_count; + U64 active_slot_count; + U64 segment_size; + U64 resident_count; + U64 victim_cursor; + LNK_CObjRuntimeSegment **slot_segments; + LNK_CompressedObj **regions; + U64 region_count; + U64 sorted_region_count; + U64 region_cap; + PVOID (WINAPI *map_view_of_file_3)(HANDLE,HANDLE,PVOID,ULONG64,SIZE_T,ULONG,ULONG,MEM_EXTENDED_PARAMETER*,ULONG); + BOOL (WINAPI *unmap_view_of_file_2)(HANDLE,PVOID,ULONG); + PVOID (WINAPI *virtual_alloc_2)(HANDLE,PVOID,SIZE_T,ULONG,ULONG,MEM_EXTENDED_PARAMETER*,ULONG); + PVOID veh; + volatile LONG initialized; + volatile LONG failed; + U64 faults; + U64 decodes; + U64 redecodes; + U64 evictions; + U64 eviction_failures; + U32 last_eviction_error; + U64 raw_segments; + U64 decoded_bytes; + U64 stored_bytes_read; + U64 occupancy_hwm; + U64 frozen_segments; + U64 raw_mapped_segments; + U64 input_compressed_bytes; + U64 input_raw_bytes; + U64 initial_cache_bytes; + U64 post_boundary_cache_bytes; + U64 decode_us; + U64 fault_us; + U64 eviction_us; + U64 freeze_us; + U64 trim_us; + U64 cleanup_us; + B32 skip_cleanup; +} LNK_CObjCache; + +typedef enum LNK_CObjPolicySource +{ + LNK_CObjPolicySource_Adaptive, + LNK_CObjPolicySource_Environment, + LNK_CObjPolicySource_CommandLine, +} LNK_CObjPolicySource; + +typedef struct LNK_CObjPolicy +{ + U64 initial_cache_bytes; + U64 post_boundary_cache_bytes; + U32 trim_mode; + B32 freeze_generation; + B32 one_shot; + B32 configured; + LNK_CObjPolicySource initial_source; + LNK_CObjPolicySource post_source; + LNK_CObjPolicySource trim_source; + LNK_CObjPolicySource freeze_source; + LNK_CObjPolicySource one_shot_source; +} LNK_CObjPolicy; + +typedef struct LNK_CObjDecoderScratch +{ + void *memory; + OO_SINTa size; + OodleLZ_Compressor compressor; + B32 configured; +} LNK_CObjDecoderScratch; + +global LNK_CObjCache g_lnk_cobj_cache; +global LNK_CObjPolicy g_lnk_cobj_policy; +global U64 g_lnk_cobj_input_capacity_hint; +global volatile LONG g_lnk_cobj_cache_init_state; + +// Oodle otherwise creates and destroys decoder state through the process heap for every segment. +// Links decode tens of thousands of independent segments concurrently, so those tiny allocations +// serialize on the heap lock. One block per calling thread is sufficient because every decode is +// unthreaded and synchronous. The container records its codec, so each block is sized for the +// actual Kraken, Mermaid, or Selkie decoder instead of pessimistically reserving Oodle's maximum. +global thread_static LNK_CObjDecoderScratch g_lnk_cobj_decoder_scratch; +global U64 g_lnk_cobj_decoder_scratch_count; +global U64 g_lnk_cobj_decoder_scratch_bytes; + +global U64 g_lnk_cobj_window_decodes; +global U64 g_lnk_cobj_window_bytes; +global U64 g_lnk_cobj_window_decoded_bytes; +global B32 g_lnk_cobj_cache_shrunk; +global U64 g_lnk_cobj_redecode_debug_s; +global U64 g_lnk_cobj_redecode_types; +global U64 g_lnk_cobj_redecode_segment_zero; + +//////////////////////////////// +// Configuration and diagnostics + +internal LNK_CObjDecoderScratch +lnk_cobj_decoder_scratch(U32 compressor_value) +{ + OodleLZ_Compressor compressor = (OodleLZ_Compressor)compressor_value; + if (g_lnk_cobj_decoder_scratch.memory != 0 && + g_lnk_cobj_decoder_scratch.configured && + g_lnk_cobj_decoder_scratch.compressor == compressor) { + return g_lnk_cobj_decoder_scratch; + } + + OO_S32 needed = OodleLZDecoder_MemorySizeNeeded(compressor, -1); + if (needed <= 0) { + compressor = OodleLZ_Compressor_Invalid; + needed = OodleLZDecoder_MemorySizeNeeded(compressor, -1); + } + if (needed > 0 && needed > g_lnk_cobj_decoder_scratch.size) { + void *memory = VirtualAlloc(0, (SIZE_T)needed, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + if (memory != 0) { + void *old_memory = g_lnk_cobj_decoder_scratch.memory; + OO_SINTa old_size = g_lnk_cobj_decoder_scratch.size; + g_lnk_cobj_decoder_scratch.memory = memory; + g_lnk_cobj_decoder_scratch.size = needed; + if (old_memory == 0) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_decoder_scratch_count); + } else { + VirtualFree(old_memory, 0, MEM_RELEASE); + } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_decoder_scratch_bytes, + needed - old_size); + } + } + + if (needed > 0 && + g_lnk_cobj_decoder_scratch.memory != 0 && + g_lnk_cobj_decoder_scratch.size >= needed) { + g_lnk_cobj_decoder_scratch.compressor = compressor; + g_lnk_cobj_decoder_scratch.configured = 1; + return g_lnk_cobj_decoder_scratch; + } + + // Preserve Oodle's existing internal-allocation fallback if caller memory could not be made. + LNK_CObjDecoderScratch fallback = {0}; + return fallback; +} + +internal OO_SINTa +lnk_cobj_oodle_decompress(U32 compressor, const void *stored, OO_SINTa stored_size, + void *raw, OO_SINTa raw_size) +{ + LNK_CObjDecoderScratch scratch = lnk_cobj_decoder_scratch(compressor); + return OodleLZ_Decompress(stored, stored_size, raw, raw_size, + OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, + 0, 0, 0, 0, scratch.memory, scratch.size, + OodleLZ_Decode_Unthreaded); +} + +internal char * +lnk_cobj_policy_source_string(LNK_CObjPolicySource source) +{ + switch (source) { + case LNK_CObjPolicySource_CommandLine: return "command-line"; + case LNK_CObjPolicySource_Environment: return "environment"; + default: return "adaptive"; + } +} + +internal B32 +lnk_cobj_env_u64(char *name, U64 *value_out) +{ + char value[64]; + DWORD len = GetEnvironmentVariableA(name, value, sizeof(value)); + if (len == 0 || len >= sizeof(value)) { return 0; } + char *end = 0; + U64 value_u64 = strtoull(value, &end, 10); + if (end == value || *end != 0) { return 0; } + *value_out = value_u64; + return 1; +} + +internal B32 +lnk_cobj_env_switch(char *name, B32 *enabled_out) +{ + char value[16]; + DWORD len = GetEnvironmentVariableA(name, value, sizeof(value)); + if (len == 0 || len >= sizeof(value)) { return 0; } + *enabled_out = value[0] != '0' && _stricmp(value, "no") != 0 && _stricmp(value, "false") != 0; + return 1; +} + +internal void +lnk_cobj_adaptive_capacities(U64 *initial_bytes_out, U64 *post_bytes_out) +{ + U64 total_gib = 8; + MEMORYSTATUSEX memory = { sizeof(memory) }; + if (GlobalMemoryStatusEx(&memory)) { + U64 physical_gib = memory.ullTotalPhys / GB(1); + U64 commit_available_gib = memory.ullAvailPageFile / GB(1); + U64 physical_budget_gib = (physical_gib * 3) / 8; + U64 commit_budget_gib = commit_available_gib / 2; + total_gib = Clamp(LNK_COBJ_MIN_TOTAL_CACHE_GIB, + Min(physical_budget_gib, commit_budget_gib), + LNK_COBJ_MAX_TOTAL_CACHE_GIB); + } + U64 post_gib = Max(1, (total_gib * LNK_COBJ_MAX_POST_CACHE_GIB) / LNK_COBJ_MAX_TOTAL_CACHE_GIB); + *initial_bytes_out = (total_gib - post_gib) * GB(1); + *post_bytes_out = post_gib * GB(1); +} + +internal void +lnk_compressed_obj_configure(LNK_Config *config) +{ + LNK_CObjPolicy policy = {0}; + lnk_cobj_adaptive_capacities(&policy.initial_cache_bytes, &policy.post_boundary_cache_bytes); + policy.freeze_generation = 1; + policy.trim_mode = 1; + + U64 env_u64 = 0; + if (lnk_cobj_env_u64("RAD_COBJ_CACHE_GIB", &env_u64) && env_u64 > 0) { + policy.initial_cache_bytes = env_u64 * GB(1); + policy.initial_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_u64("RAD_COBJ_CACHE_MIB", &env_u64) && env_u64 > 0) { + policy.initial_cache_bytes = env_u64 * MB(1); + policy.initial_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_u64("RAD_COBJ_CACHE_SHRINK_GIB", &env_u64) && env_u64 > 0) { + policy.post_boundary_cache_bytes = env_u64 * GB(1); + policy.post_source = LNK_CObjPolicySource_Environment; + } + + B32 env_switch = 0; + if (lnk_cobj_env_switch("RAD_COBJ_CACHE_FREEZE", &env_switch)) { + policy.freeze_generation = env_switch; + policy.freeze_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_u64("RAD_COBJ_TRIM_WS", &env_u64)) { + policy.trim_mode = (U32)Clamp(0, env_u64, 2); + policy.trim_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_switch("RAD_COBJ_SKIP_CLEANUP", &env_switch)) { + policy.one_shot = env_switch; + policy.one_shot_source = LNK_CObjPolicySource_Environment; + } + + if (config != 0) { + if (config->cobj_cache_gib > 0) { + policy.initial_cache_bytes = config->cobj_cache_gib * GB(1); + policy.initial_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_cache_shrink_gib > 0) { + policy.post_boundary_cache_bytes = config->cobj_cache_shrink_gib * GB(1); + policy.post_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_cache_freeze != LNK_SwitchState_Null) { + policy.freeze_generation = config->cobj_cache_freeze == LNK_SwitchState_Yes; + policy.freeze_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_trim_ws != LNK_SwitchState_Null) { + policy.trim_mode = config->cobj_trim_ws == LNK_SwitchState_Yes; + policy.trim_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_one_shot != LNK_SwitchState_Null) { + policy.one_shot = config->cobj_one_shot == LNK_SwitchState_Yes; + policy.one_shot_source = LNK_CObjPolicySource_CommandLine; + } + } + + policy.initial_cache_bytes = Max(MB(1), policy.initial_cache_bytes); + policy.post_boundary_cache_bytes = Max(MB(1), policy.post_boundary_cache_bytes); + policy.configured = 1; + g_lnk_cobj_policy = policy; +} + +internal void +lnk_compressed_obj_prepare_cache(String8 *mapped_files, U64 count) +{ + if (g_lnk_cobj_cache_init_state != 0) { return; } + U64 segment_capacity = 0; + for EachIndex(i, count) { + String8 mapped_file = mapped_files[i]; + if (mapped_file.size < sizeof(LNK_CObjHeader)) { continue; } + LNK_CObjHeader *header = (LNK_CObjHeader *)mapped_file.str; + if (header->magic != LNK_COBJ_MAGIC || header->version != LNK_COBJ_VERSION) { continue; } + // Every OBJ consumes an integral number of cache slots. Summing raw byte counts before + // aligning underestimates capacity badly for targets containing many small OBJs: sixty-four + // 20 KiB OBJs need sixty-four slots, not three slots for their ~1.25 MiB byte sum. + U64 obj_capacity = (U64)header->segment_count * header->segment_size; + segment_capacity = obj_capacity > max_U64 - segment_capacity ? max_U64 : + segment_capacity + obj_capacity; + } + if (segment_capacity > g_lnk_cobj_input_capacity_hint) { + g_lnk_cobj_input_capacity_hint = segment_capacity; + } +} + +internal LNK_CompressedObj *lnk_cobj_region_from_address(U8 *address); + +internal int +lnk_cobj_address_ptr_compare(const void *a, const void *b) +{ + U8 *pa = *(U8 **)a; + U8 *pb = *(U8 **)b; + return pa < pb ? -1 : pa > pb; +} + +internal B32 +lnk_cobj_runtime_segment_overlaps_types(LNK_CObjRuntimeSegment *segment) +{ + LNK_CompressedObj *region = segment && segment->address ? + lnk_cobj_region_from_address(segment->address) : 0; + if (region == 0 || region->type_indices == 0) { return 0; } + U64 seg_idx = (U64)(segment->address - region->base) / region->segment_size; + Rng1U64 seg_range = r1u64(seg_idx * region->segment_size, + Min(region->raw_size, (seg_idx + 1) * region->segment_size)); + for EachIndex(type_idx, region->type_index_count) { + LNK_CObjTypeIndex *type = ®ion->type_indices[type_idx]; + Rng1U64 type_range = r1u64(type->raw_section_offset, + type->raw_section_offset + type->raw_section_size); + if (dim_1u64(intersect_1u64(seg_range, type_range)) != 0) { return 1; } + } + return 0; +} + +internal B32 +lnk_cobj_runtime_segment_overlaps_debug_s(LNK_CObjRuntimeSegment *segment) +{ + LNK_CompressedObj *region = segment && segment->address ? + lnk_cobj_region_from_address(segment->address) : 0; + if (region == 0 || region->debug_s_entries == 0) { return 0; } + U64 seg_idx = (U64)(segment->address - region->base) / region->segment_size; + Rng1U64 seg_range = r1u64(seg_idx * region->segment_size, + Min(region->raw_size, (seg_idx + 1) * region->segment_size)); + for EachIndex(entry_idx, region->debug_s_entry_count) { + LNK_CObjDebugSEntry *entry = ®ion->debug_s_entries[entry_idx]; + // Retain the complete span from the section signature through each indexed payload. Taking + // the union over a section's entries covers its C13 headers/padding as well as payload bytes. + Rng1U64 debug_s_range = r1u64(entry->raw_section_offset, + (U64)entry->raw_payload_offset + entry->raw_payload_size); + if (dim_1u64(intersect_1u64(seg_range, debug_s_range)) != 0) { return 1; } + } + return 0; +} + +internal void +lnk_compressed_obj_log_phase_stats(char *tag) +{ + if (g_lnk_cobj_cache.region_count == 0) { return; } + char *phase_stats_env = getenv("RAD_COBJ_PHASE_STATS"); + if (phase_stats_env != 0 && phase_stats_env[0] != 0 && phase_stats_env[0] != '0') { + lnk_log(LNK_Log_Timers, "[cobj phase] %s faults=%llu decodes=%llu redecodes=%llu evictions=%llu window=%llu", + tag, g_lnk_cobj_cache.faults, g_lnk_cobj_cache.decodes, g_lnk_cobj_cache.redecodes, + g_lnk_cobj_cache.evictions, g_lnk_cobj_window_decodes); + } +} + +internal B32 +lnk_cobj_skip_cleanup_enabled(void) +{ + return g_lnk_cobj_policy.one_shot; +} + +//////////////////////////////// +// Indexed and explicit-decode access + +internal String8 +lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range) +{ + String8 result = {0}; + if (!obj || range.min >= range.max || range.max > obj->raw_size || !obj->mapped_file.str) { return result; } + U32 first = (U32)(range.min / obj->segment_size); + U32 last = (U32)((range.max - 1) / obj->segment_size); + if (last >= obj->segment_count) { return result; } + + LNK_CObjSegment *first_entry = &obj->directory[first]; + if (!(first_entry->flags & LNK_COBJ_SEGMENT_RAW) || first_entry->stored_size != first_entry->raw_size) { return result; } + U64 expected_file_off = first_entry->file_offset + first_entry->stored_size; + for (U32 seg_idx = first + 1; seg_idx <= last; ++seg_idx) { + LNK_CObjSegment *entry = &obj->directory[seg_idx]; + if (!(entry->flags & LNK_COBJ_SEGMENT_RAW) || entry->stored_size != entry->raw_size || + entry->file_offset != expected_file_off) { + return result; + } + expected_file_off += entry->stored_size; + } + + // Raw runs are mapped directly into the reserved logical OBJ address range. + return str8(obj->base + range.min, dim_1u64(range)); +} + +internal B32 +lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out) +{ + MemoryZeroStruct(out); + if (!obj || !obj->type_indices) { return 0; } + for (U32 i = 0; i < obj->type_index_count; ++i) { + LNK_CObjTypeIndex *idx = &obj->type_indices[i]; + if (idx->raw_section_offset == section_range.min && idx->raw_section_size == dim_1u64(section_range)) { + out->count = idx->leaf_count; + if (obj->header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR) { + U64 group_count = ((U64)idx->leaf_count + + (((U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT) - 1)) >> + LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_bytes = group_count * 2 * sizeof(U32); + out->packed_v2_offset_groups = (U32 *)(obj->mapped_file.str + idx->offsets_file_offset); + out->packed_v2_offset_payload = obj->mapped_file.str + idx->offsets_file_offset + + ((group_bytes + 7) & ~(U64)7); + out->offset_checkpoint_shift = LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + out->packed_kind_dictionary = (U16 *)(obj->mapped_file.str + idx->sizes_file_offset); + out->packed_kind_codes = obj->mapped_file.str + idx->kinds_file_offset; + out->packed_sidecar = 1; + } else { + out->offsets = (U32 *)(obj->mapped_file.str + idx->offsets_file_offset); + out->sizes = (U16 *)(obj->mapped_file.str + idx->sizes_file_offset); + out->kinds = (U16 *)(obj->mapped_file.str + idx->kinds_file_offset); + } + if (obj->udt_hash_indices) { + LNK_CObjUdtHashIndex *udt = &obj->udt_hash_indices[i]; + out->complete_udt_hashes = (U64 *)(obj->mapped_file.str + udt->hashes_file_offset); + out->complete_udt_hash_count = udt->hash_count; + } + return 1; + } + } + return 0; +} + +internal B32 +lnk_compressed_obj_base_reloc_index(LNK_CompressedObj *obj, LNK_CObjBaseRelocView *out) +{ + MemoryZeroStruct(out); + if (!obj || !obj->has_base_reloc_index) { return 0; } + out->count = obj->base_reloc_entry_count; + out->v = obj->base_reloc_entries; + return 1; +} + +internal B32 +lnk_compressed_obj_debug_s_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjDebugSView *out) +{ + MemoryZeroStruct(out); + if (!obj || !obj->debug_s_entries || section_range.min > max_U32) { return 0; } + U32 wanted = (U32)section_range.min; + U32 lo = 0, hi = obj->debug_s_entry_count; + while (lo < hi) { + U32 mid = lo + (hi - lo) / 2; + if (obj->debug_s_entries[mid].raw_section_offset < wanted) { lo = mid + 1; } + else { hi = mid; } + } + U32 first = lo; + while (lo < obj->debug_s_entry_count && obj->debug_s_entries[lo].raw_section_offset == wanted) { lo += 1; } + if (first == lo) { return 0; } + out->v = obj->debug_s_entries + first; + out->summaries = obj->debug_s_summaries ? obj->debug_s_summaries + first : 0; + out->count = lo - first; + return 1; +} + +internal B32 +lnk_compressed_obj_copy_range(LNK_CompressedObj *obj, Rng1U64 range, void *raw_dst, LNK_CObjDecodeWindow *window) +{ + if (!obj || range.min > range.max || range.max > obj->raw_size) { return 0; } + if (!window->buffer || window->buffer_cap < obj->segment_size) { + if (window->buffer) { VirtualFree(window->buffer, 0, MEM_RELEASE); } + window->buffer = VirtualAlloc(0, obj->segment_size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + window->buffer_cap = window->buffer ? obj->segment_size : 0; + window->seg_idx = max_U32; + if (!window->buffer) { return 0; } + } + if (window->obj != obj) { window->obj = obj; window->seg_idx = max_U32; } + + U8 *dst = raw_dst; + U64 cursor = range.min; + while (cursor < range.max) { + U32 seg_idx = (U32)(cursor / obj->segment_size); + LNK_CObjSegment *entry = &obj->directory[seg_idx]; + U64 in_seg = cursor - (U64)seg_idx * obj->segment_size; + U64 amount = Min(range.max - cursor, (U64)entry->raw_size - in_seg); + if (window->seg_idx != seg_idx) { + U8 *stored = obj->mapped_file.str + entry->file_offset; + B32 ok = 1; + B32 decode_direct = in_seg == 0 && amount == entry->raw_size; + U8 *decode_dst = decode_direct ? dst : window->buffer; + if (entry->flags & LNK_COBJ_SEGMENT_RAW) { + MemoryCopy(decode_dst, stored, entry->raw_size); + } else { + OO_SINTa got = lnk_cobj_oodle_decompress(obj->header->compressor, + stored, entry->stored_size, + decode_dst, entry->raw_size); + ok = got == entry->raw_size; + } + if (!ok) { return 0; } + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_window_decodes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_window_bytes, entry->stored_size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_window_decoded_bytes, entry->raw_size); + if (decode_direct) { + // Large $S and symbol-fixup walks consume complete interior segments once. Decode those + // bytes into their final arena instead of filling the window and copying them again. + window->seg_idx = max_U32; + dst += amount; cursor += amount; + continue; + } + window->seg_idx = seg_idx; + window->valid_size = entry->raw_size; + } + MemoryCopy(dst, window->buffer + in_seg, amount); + dst += amount; cursor += amount; + } + return 1; +} + +internal B32 +lnk_compressed_obj_copy_string(LNK_CompressedObj *obj, String8 src, void *dst, LNK_CObjDecodeWindow *window) +{ + if (!obj) { return 0; } + U64 base = (U64)obj->base; + U64 ptr = (U64)src.str; + if (ptr < base || ptr - base > obj->raw_size || src.size > obj->raw_size - (ptr - base)) { return 0; } + return lnk_compressed_obj_copy_range(obj, r1u64(ptr - base, ptr - base + src.size), dst, window); +} + +internal void +lnk_compressed_obj_release_window(LNK_CObjDecodeWindow *window) +{ + if (window->buffer) { VirtualFree(window->buffer, 0, MEM_RELEASE); } + MemoryZeroStruct(window); +} + +internal U32 +lnk_compressed_obj_segment_count(LNK_CompressedObj *obj) +{ + return obj ? obj->segment_count : 0; +} + +internal U32 +lnk_compressed_obj_segment_size(LNK_CompressedObj *obj) +{ + return obj ? obj->segment_size : 0; +} + +internal Rng1U64 +lnk_compressed_obj_stored_segment_range(LNK_CompressedObj *obj, U32 segment_idx) +{ + Rng1U64 result = {0}; + if (obj && segment_idx < obj->segment_count) { + LNK_CObjSegment *segment = &obj->directory[segment_idx]; + U64 min = (U64)(obj->mapped_file.str + segment->file_offset); + result = r1u64(min, min + segment->stored_size); + } + return result; +} + +//////////////////////////////// +// Decoded-segment cache + + +internal LNK_CObjWriteGroup * +lnk_cobj_alloc_write_groups(U64 slot_count, U32 group_slot_count, U64 *group_count_out) +{ + U64 group_count = CeilIntegerDiv(slot_count, group_slot_count); + LNK_CObjWriteGroup *groups = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, + group_count * sizeof(*groups)); + if (groups) { + for EachIndex(group_idx, group_count) { + U64 first_slot = group_idx * group_slot_count; + groups[group_idx].slot_count = (U32)Min((U64)group_slot_count, slot_count - first_slot); + } + *group_count_out = group_count; + } + return groups; +} + +internal void +lnk_cobj_release_write_groups(LNK_CObjWriteGroup *groups, U64 group_count) +{ + if (!groups) { return; } + for EachIndex(group_idx, group_count) { + if (groups[group_idx].view) { UnmapViewOfFile(groups[group_idx].view); } + } + HeapFree(GetProcessHeap(), 0, groups); +} + +internal U8 * +lnk_cobj_write_group_acquire(U32 slot_idx, U32 *group_idx_out) +{ + if (!g_lnk_cobj_cache.write_groups) { return 0; } + U32 group_idx = slot_idx / g_lnk_cobj_cache.write_group_slot_count; + LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; + AcquireSRWLockExclusive(&group->lock); + U8 *result = 0; + if (!group->sealed) { + // Return the group index even when the large write view cannot be mapped. The caller can + // populate this slot through a temporary per-slot view, but must still count it complete; + // otherwise the group never seals and none of its slots can ever be recycled. + *group_idx_out = group_idx; + if (!group->view) { + U64 group_off = (U64)group_idx * g_lnk_cobj_cache.write_group_slot_count * + g_lnk_cobj_cache.segment_size; + U64 group_size = (U64)group->slot_count * g_lnk_cobj_cache.segment_size; + group->view = MapViewOfFile(g_lnk_cobj_cache.slot_mapping, FILE_MAP_WRITE, + (DWORD)(group_off >> 32), (DWORD)group_off, group_size); + } + if (group->view) { + U32 in_group_idx = slot_idx % g_lnk_cobj_cache.write_group_slot_count; + result = group->view + (U64)in_group_idx * g_lnk_cobj_cache.segment_size; + } + } + ReleaseSRWLockExclusive(&group->lock); + return result; +} + +internal void +lnk_cobj_write_group_complete(U32 group_idx) +{ + LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; + AcquireSRWLockExclusive(&group->lock); + LONG completed_count = InterlockedIncrement(&group->completed_count); + if (completed_count == group->slot_count) { + if (group->view) { UnmapViewOfFile(group->view); } + group->view = 0; + InterlockedExchange(&group->sealed, 1); + } + ReleaseSRWLockExclusive(&group->lock); +} + +internal void +lnk_compressed_obj_trim_working_set(void) +{ + if (g_lnk_cobj_policy.trim_mode == 0) { return; } + + U64 begin_us = now_time_us(); + if (g_lnk_cobj_policy.trim_mode == 2) { + SetProcessWorkingSetSize(GetCurrentProcess(), (SIZE_T)-1, (SIZE_T)-1); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.trim_us, now_time_us() - begin_us); + lnk_log(LNK_Log_Timers, "[cobj trim] whole-process elapsed=%.2f ms", + (F64)(now_time_us() - begin_us) / 1000.0); + return; + } + + // Drop only the decoded cache views from the process working set. Unlike the + // whole-process SetProcessWorkingSetSize path this preserves the linker's hot + // hash tables and arenas. The pagefile-section mappings stay intact, so a + // later pointer access can soft-fault the decoded bytes without running Oodle + // again; VirtualUnlock on an unlocked range intentionally returns + // ERROR_NOT_LOCKED after removing resident pages from the caller's working set. + U64 cache_view_count = 0; + U64 cache_view_bytes = 0; + U64 purged_count = 0; + U64 post_boundary_cache_bytes = g_lnk_cobj_cache.post_boundary_cache_bytes; + if (post_boundary_cache_bytes > 0 && !g_lnk_cobj_cache_shrunk && g_lnk_cobj_cache.segment_size) { + U64 wanted_slots = Clamp(1, post_boundary_cache_bytes / g_lnk_cobj_cache.segment_size, + g_lnk_cobj_cache.slot_count); + B32 freeze_generation = g_lnk_cobj_policy.freeze_generation; + if (freeze_generation) { + U64 freeze_begin_us = now_time_us(); + // Preserve the complete first-generation mapping but never recycle one of its slots again. + // Existing logical OBJ pointers therefore remain valid. PDB misses use a fresh, independently + // evicting cache generation; this costs shrink_gib additional system commit but avoids the + // thousands of UnmapViewOfFile2 calls and forced re-decodes of a destructive phase reset. + U64 new_mapping_size = wanted_slots * g_lnk_cobj_cache.segment_size; + HANDLE new_mapping = CreateFileMappingW(INVALID_HANDLE_VALUE, 0, PAGE_READWRITE, + (DWORD)(new_mapping_size >> 32), + (DWORD)new_mapping_size, 0); + U64 new_write_group_count = 0; + LNK_CObjWriteGroup *new_write_groups = g_lnk_cobj_cache.write_groups ? + lnk_cobj_alloc_write_groups(wanted_slots, g_lnk_cobj_cache.write_group_slot_count, + &new_write_group_count) : 0; + LNK_CObjRuntimeSegment **new_slots = VirtualAlloc(0, wanted_slots * sizeof(void *), + MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + B32 can_freeze = new_mapping != 0 && new_slots != 0 && + (!g_lnk_cobj_cache.write_groups || new_write_groups != 0); + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (can_freeze) { + for EachIndex(slot_idx, g_lnk_cobj_cache.slot_count) { + LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; + if (segment != 0 && segment->state != LNK_CObjSegState_Ready && + segment->state != LNK_CObjSegState_Pinned) { + can_freeze = 0; + break; + } + } + } + if (can_freeze) { + HANDLE old_mapping = g_lnk_cobj_cache.slot_mapping; + LNK_CObjWriteGroup *old_write_groups = g_lnk_cobj_cache.write_groups; + U64 old_write_group_count = g_lnk_cobj_cache.write_group_count; + LNK_CObjRuntimeSegment **old_slots = g_lnk_cobj_cache.slot_segments; + U64 old_slot_count = g_lnk_cobj_cache.slot_count; + for EachIndex(slot_idx, old_slot_count) { + LNK_CObjRuntimeSegment *segment = old_slots[slot_idx]; + if (segment != 0) { + // max_U32 denotes a frozen slot: it is mapped and readable, but is not owned by the + // active generation's slot table and must never participate in victim selection. + segment->slot_idx = max_U32; + g_lnk_cobj_cache.frozen_segments += 1; + } + } + g_lnk_cobj_cache.slot_mapping = new_mapping; + g_lnk_cobj_cache.write_groups = new_write_groups; + g_lnk_cobj_cache.write_group_count = new_write_group_count; + g_lnk_cobj_cache.slot_segments = new_slots; + g_lnk_cobj_cache.slot_count = wanted_slots; + g_lnk_cobj_cache.active_slot_count = wanted_slots; + g_lnk_cobj_cache.resident_count = 0; + g_lnk_cobj_cache.victim_cursor = 0; + g_lnk_cobj_cache_shrunk = 1; + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + // Mapped views retain the old pagefile section object. Only its handle and now-unused + // reverse slot table are released here; ordinary close unmaps each frozen logical view. + lnk_cobj_release_write_groups(old_write_groups, old_write_group_count); + CloseHandle(old_mapping); + U64 trimmed_frozen = 0; + U64 trim_calls = 0; + U64 frozen_trim_us = 0; + char *freeze_trim_env = getenv("RAD_COBJ_CACHE_FREEZE_TRIM"); + if (freeze_trim_env != 0 && freeze_trim_env[0] != 0 && freeze_trim_env[0] != '0') { + U64 frozen_trim_begin_us = now_time_us(); + // Keep every view valid but offer its resident pages back to the OS. Later consumers + // incur only soft page faults into the frozen pagefile section, never Oodle re-decodes. + // This is deliberately optional: it trades some kernel fault work for a lower PDB-phase + // working set, so deployments can choose the appropriate memory/speed point. + B32 types_only = _stricmp(freeze_trim_env, "types") == 0; + B32 except_debug_s = _stricmp(freeze_trim_env, "except_debug_s") == 0 || + _stricmp(freeze_trim_env, "except_debug_s_runs") == 0; + B32 runs_only = _stricmp(freeze_trim_env, "except_debug_s_runs") == 0; + U8 **trim_addresses = VirtualAlloc(0, old_slot_count * sizeof(*trim_addresses), + MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + for EachIndex(slot_idx, old_slot_count) { + LNK_CObjRuntimeSegment *segment = old_slots[slot_idx]; + B32 should_trim = segment != 0 && segment->address != 0; + if (should_trim && types_only) { + should_trim = lnk_cobj_runtime_segment_overlaps_types(segment); + } else if (should_trim && except_debug_s) { + should_trim = !lnk_cobj_runtime_segment_overlaps_debug_s(segment); + } + if (should_trim) { + if (trim_addresses != 0) { + trim_addresses[trimmed_frozen++] = segment->address; + } else { + VirtualUnlock(segment->address, g_lnk_cobj_cache.segment_size); + trimmed_frozen += 1; + trim_calls += 1; + } + } + } + if (trim_addresses != 0) { + qsort(trim_addresses, trimmed_frozen, sizeof(*trim_addresses), lnk_cobj_address_ptr_compare); + U64 selected_count = trimmed_frozen; + trimmed_frozen = 0; + for (U64 first = 0; first < selected_count;) { + U64 opl = first + 1; + while (opl < selected_count && + trim_addresses[opl] == trim_addresses[opl - 1] + g_lnk_cobj_cache.segment_size) { + opl += 1; + } + U64 run_count = opl - first; + if (!runs_only || run_count > 1) { + VirtualUnlock(trim_addresses[first], run_count * g_lnk_cobj_cache.segment_size); + trimmed_frozen += run_count; + trim_calls += 1; + } + first = opl; + } + VirtualFree(trim_addresses, 0, MEM_RELEASE); + } + frozen_trim_us = now_time_us() - frozen_trim_begin_us; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.trim_us, frozen_trim_us); + } + VirtualFree(old_slots, 0, MEM_RELEASE); + U64 freeze_us = now_time_us() - freeze_begin_us; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.freeze_us, freeze_us); + lnk_log(LNK_Log_Timers, "[cobj freeze] retained=%llu trimmed=%llu trim-calls=%llu new-cache=%llu MiB elapsed=%.2f ms", + g_lnk_cobj_cache.frozen_segments, trimmed_frozen, trim_calls, new_mapping_size / MB(1), + (F64)freeze_us / 1000.0); + return; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + lnk_cobj_release_write_groups(new_write_groups, new_write_group_count); + if (new_mapping) { CloseHandle(new_mapping); } + if (new_slots) { VirtualFree(new_slots, 0, MEM_RELEASE); } + } + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + // Reset the phase cache completely. Keeping arbitrary retained mappings across the + // limit transition is unsafe because later victim selection owns an index prefix. + for EachIndex(slot_idx, g_lnk_cobj_cache.slot_count) { + LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; + if (!segment || InterlockedCompareExchange(&segment->state, LNK_CObjSegState_Evicting, + LNK_CObjSegState_Ready) != LNK_CObjSegState_Ready) { + continue; + } + if (segment->address && + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), segment->address, + MEM_PRESERVE_PLACEHOLDER)) { + g_lnk_cobj_cache.slot_segments[slot_idx] = 0; + segment->slot_idx = max_U32; + InterlockedExchange(&segment->state, LNK_CObjSegState_Empty); + purged_count += 1; + } else { + InterlockedExchange(&segment->state, LNK_CObjSegState_Ready); + } + } + // Write-pinned segments (and the unlikely view that failed to unmap) still own their + // original slots. Count survivors in the active prefix instead of pretending the reset + // emptied it. If that prefix is completely occupied, extend it through the next free slot + // so a later cache miss cannot spin forever with no possible victim. + U64 active_slot_count = wanted_slots; + U64 retained_active_count = 0; + for EachIndex(slot_idx, active_slot_count) { + retained_active_count += g_lnk_cobj_cache.slot_segments[slot_idx] != 0; + } + if (retained_active_count == active_slot_count) { + for (U64 slot_idx = active_slot_count; slot_idx < g_lnk_cobj_cache.slot_count; ++slot_idx) { + retained_active_count += g_lnk_cobj_cache.slot_segments[slot_idx] != 0; + active_slot_count = slot_idx + 1; + if (g_lnk_cobj_cache.slot_segments[slot_idx] == 0) { break; } + } + } + g_lnk_cobj_cache.active_slot_count = active_slot_count; + g_lnk_cobj_cache.resident_count = retained_active_count; + g_lnk_cobj_cache.victim_cursor %= active_slot_count; + g_lnk_cobj_cache_shrunk = 1; + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + } + if (post_boundary_cache_bytes == 0) { + for EachIndex(region_idx, g_lnk_cobj_cache.region_count) { + LNK_CompressedObj *region = g_lnk_cobj_cache.regions[region_idx]; + if (!region || !region->active) { continue; } + for EachIndex(seg_idx, region->segment_count) { + LNK_CObjRuntimeSegment *segment = ®ion->segments[seg_idx]; + if ((segment->state == LNK_CObjSegState_Ready || segment->state == LNK_CObjSegState_Pinned) && + segment->address) { + VirtualUnlock(segment->address, region->segment_size); + cache_view_count += 1; + cache_view_bytes += region->segment_size; + } + } + } + } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.trim_us, now_time_us() - begin_us); + lnk_log(LNK_Log_Timers, "[cobj trim] purged=%llu active=%llu MiB cache_views=%llu cache=%llu GiB elapsed=%.2f ms", + purged_count, (g_lnk_cobj_cache.active_slot_count * g_lnk_cobj_cache.segment_size) / MB(1), + cache_view_count, cache_view_bytes / GB(1), + (F64)(now_time_us() - begin_us) / 1000.0); +} + +internal int +lnk_cobj_region_ptr_compare(const void *a, const void *b) +{ + LNK_CompressedObj *ra = *(LNK_CompressedObj **)a; + LNK_CompressedObj *rb = *(LNK_CompressedObj **)b; + return ra->base < rb->base ? -1 : ra->base > rb->base; +} + +internal LNK_CompressedObj * +lnk_cobj_region_from_address(U8 *address) +{ + U64 min = 0, max = g_lnk_cobj_cache.region_count; + while (min < max) { + U64 mid = min + (max - min) / 2; + LNK_CompressedObj *region = g_lnk_cobj_cache.regions[mid]; + if (address < region->base) { + max = mid; + } else if (address >= region->base + region->reserve_size) { + min = mid + 1; + } else { + return region->active ? region : 0; + } + } + return 0; +} + +internal B32 +lnk_cobj_evict_one_locked(U32 *slot_idx_out) +{ + U64 victim_slot = max_U64; + LNK_CObjRuntimeSegment *victim = 0; + for (U64 probe = 0; probe < g_lnk_cobj_cache.active_slot_count; ++probe) { + U64 slot_idx = (g_lnk_cobj_cache.victim_cursor + probe) % g_lnk_cobj_cache.active_slot_count; + LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; + if (segment == 0) { + *slot_idx_out = (U32)slot_idx; + g_lnk_cobj_cache.victim_cursor = (slot_idx + 1) % g_lnk_cobj_cache.active_slot_count; + return 1; + } + B32 write_group_recyclable = 1; + if (g_lnk_cobj_cache.write_groups) { + U64 group_idx = slot_idx / g_lnk_cobj_cache.write_group_slot_count; + write_group_recyclable = g_lnk_cobj_cache.write_groups[group_idx].sealed != 0; + } + if (segment->state == LNK_CObjSegState_Ready && write_group_recyclable) { + victim_slot = slot_idx; + victim = segment; + break; + } + } + if (victim == 0 || InterlockedCompareExchange(&victim->state, LNK_CObjSegState_Evicting, LNK_CObjSegState_Ready) != LNK_CObjSegState_Ready) { + return 0; + } + + U8 *victim_address = victim->address; + U64 eviction_begin_us = now_time_us(); + B32 unmapped = victim_address != 0 && + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), victim_address, + MEM_PRESERVE_PLACEHOLDER); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.eviction_us, + now_time_us() - eviction_begin_us); + if (!unmapped) { + g_lnk_cobj_cache.eviction_failures += 1; + g_lnk_cobj_cache.last_eviction_error = GetLastError(); + InterlockedExchange(&victim->state, LNK_CObjSegState_Ready); + return 0; + } + g_lnk_cobj_cache.slot_segments[victim_slot] = 0; + Assert(g_lnk_cobj_cache.resident_count > 0); + g_lnk_cobj_cache.resident_count -= 1; + victim->slot_idx = max_U32; + InterlockedExchange(&victim->state, LNK_CObjSegState_Empty); + g_lnk_cobj_cache.evictions += 1; + *slot_idx_out = (U32)victim_slot; + g_lnk_cobj_cache.victim_cursor = (victim_slot + 1) % g_lnk_cobj_cache.active_slot_count; + return 1; +} + +internal B32 lnk_cobj_isolate_portable_segment(LNK_CompressedObj *region, U32 seg_idx); + +internal B32 +lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) +{ + LNK_CObjRuntimeSegment *runtime = ®ion->segments[seg_idx]; + AcquireSRWLockExclusive(&runtime->lock); + if (runtime->state == LNK_CObjSegState_Ready || runtime->state == LNK_CObjSegState_Pinned) { + ReleaseSRWLockExclusive(&runtime->lock); + return 1; + } + while (runtime->state == LNK_CObjSegState_Evicting) { YieldProcessor(); } + InterlockedExchange(&runtime->state, LNK_CObjSegState_Loading); + + U32 slot_idx = max_U32; + U64 stall_begin_us = now_time_us(); + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + while (!lnk_cobj_evict_one_locked(&slot_idx)) { + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + SwitchToThread(); + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + U64 elapsed_us = now_time_us() - stall_begin_us; + if (elapsed_us >= LNK_COBJ_STALL_ABORT_US) { + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + InterlockedExchange(&runtime->state, LNK_CObjSegState_Empty); + InterlockedExchange(&g_lnk_cobj_cache.failed, 1); + ReleaseSRWLockExclusive(&runtime->lock); + return 0; + } + } + g_lnk_cobj_cache.slot_segments[slot_idx] = runtime; + runtime->slot_idx = slot_idx; + g_lnk_cobj_cache.resident_count += 1; + Assert(g_lnk_cobj_cache.resident_count <= g_lnk_cobj_cache.active_slot_count); + if (g_lnk_cobj_cache.resident_count > g_lnk_cobj_cache.occupancy_hwm) { + g_lnk_cobj_cache.occupancy_hwm = g_lnk_cobj_cache.resident_count; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + + LNK_CObjSegment *entry = ®ion->directory[seg_idx]; + U64 slot_off = (U64)slot_idx * region->segment_size; + U32 write_group_idx = max_U32; + U8 *write_view = lnk_cobj_write_group_acquire(slot_idx, &write_group_idx); + B32 temporary_write_view = write_view == 0; + if (temporary_write_view) { + write_view = (U8 *)MapViewOfFile(g_lnk_cobj_cache.slot_mapping, FILE_MAP_WRITE, + (DWORD)(slot_off >> 32), (DWORD)slot_off, region->segment_size); + } + B32 ok = write_view != 0; + B32 slot_written = 0; + U64 decode_begin_us = now_time_us(); + if (ok) { + U8 *stored = region->mapped_file.str + entry->file_offset; + if (entry->flags & LNK_COBJ_SEGMENT_RAW) { + MemoryCopy(write_view, stored, entry->raw_size); + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.raw_segments); + } else { + OO_SINTa decoded = lnk_cobj_oodle_decompress(region->header->compressor, + stored, entry->stored_size, + write_view, entry->raw_size); + ok = decoded == entry->raw_size; + } + if (ok && entry->raw_size < region->segment_size) { + MemoryZero(write_view + entry->raw_size, region->segment_size - entry->raw_size); + } + slot_written = ok; + if (temporary_write_view) { UnmapViewOfFile(write_view); } + } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.decode_us, + now_time_us() - decode_begin_us); + U8 *target = region->base + (U64)seg_idx * region->segment_size; + if (ok) { ok = lnk_cobj_isolate_portable_segment(region, seg_idx); } + if (ok) { + void *mapped = g_lnk_cobj_cache.map_view_of_file_3(g_lnk_cobj_cache.slot_mapping, GetCurrentProcess(), target, + slot_off, region->segment_size, MEM_REPLACE_PLACEHOLDER, + PAGE_READONLY, 0, 0); + ok = mapped == target; + } + if (slot_written && write_group_idx != max_U32) { + lnk_cobj_write_group_complete(write_group_idx); + } + if (!ok) { + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (g_lnk_cobj_cache.slot_segments[slot_idx] == runtime) { + g_lnk_cobj_cache.slot_segments[slot_idx] = 0; + Assert(g_lnk_cobj_cache.resident_count > 0); + g_lnk_cobj_cache.resident_count -= 1; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + runtime->slot_idx = max_U32; + InterlockedExchange(&runtime->state, LNK_CObjSegState_Empty); + InterlockedExchange(&g_lnk_cobj_cache.failed, 1); + ReleaseSRWLockExclusive(&runtime->lock); + return 0; + } + + if (runtime->decode_count++ > 0) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.redecodes); + if (lnk_cobj_runtime_segment_overlaps_debug_s(runtime)) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_redecode_debug_s); + } + if (lnk_cobj_runtime_segment_overlaps_types(runtime)) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_redecode_types); + } + if (seg_idx == 0) { InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_redecode_segment_zero); } + } + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.decodes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.decoded_bytes, entry->raw_size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.stored_bytes_read, entry->stored_size); + InterlockedExchange(&runtime->state, LNK_CObjSegState_Ready); + ReleaseSRWLockExclusive(&runtime->lock); + return 1; +} + +// In-place COFF patches are rare, but they must retain FILE_MAP_COPY semantics. Raw-stored +// segments already map the portable file directly and can fall through to the generic COW VEH. +// Compressed segments first need a decoded mapping. Pin it so eviction can never discard a +// private modification, then promote the faulting page here instead of depending on the order of +// independently registered vectored exception handlers. +internal B32 +lnk_cobj_pin_compressed_segment_for_write(LNK_CompressedObj *region, U32 seg_idx) +{ + LNK_CObjRuntimeSegment *runtime = ®ion->segments[seg_idx]; + for (;;) { + LONG state = runtime->state; + if (state == LNK_CObjSegState_Pinned) { return 1; } + if (state == LNK_CObjSegState_Ready) { + if (InterlockedCompareExchange(&runtime->state, LNK_CObjSegState_Pinned, + LNK_CObjSegState_Ready) == LNK_CObjSegState_Ready) { + return 1; + } + continue; + } + if (!lnk_cobj_materialize(region, seg_idx)) { return 0; } + } +} + +internal LONG NTAPI +lnk_cobj_veh(EXCEPTION_POINTERS *info) +{ + EXCEPTION_RECORD *er = info->ExceptionRecord; + if (er->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && er->NumberParameters >= 2) { + U8 *address = (U8 *)er->ExceptionInformation[1]; + LNK_CompressedObj *region = lnk_cobj_region_from_address(address); + if (region != 0) { + U64 fault_begin_us = now_time_us(); + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.faults); + U32 seg_idx = (U32)((address - region->base) / region->segment_size); + U64 access_kind = er->ExceptionInformation[0]; + if (access_kind == 0 && seg_idx < region->segment_count && + lnk_cobj_materialize(region, seg_idx)) { + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.fault_us, + now_time_us() - fault_begin_us); + return EXCEPTION_CONTINUE_EXECUTION; + } + if (access_kind == 1 && seg_idx < region->segment_count && + !(region->directory[seg_idx].flags & LNK_COBJ_SEGMENT_RAW)) { + if (lnk_cobj_pin_compressed_segment_for_write(region, seg_idx)) { + // Complete the write-fault transition in this handler. Both this handler and the + // ordinary input COW handler are front-inserted, so relying on which one runs next + // would make a write-first access depend on their registration order. + void *page = (void *)((UINT_PTR)address & ~(UINT_PTR)(KB(4) - 1)); + DWORD old_protect = 0; + if (VirtualProtect(page, KB(4), PAGE_WRITECOPY, &old_protect)) { + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.fault_us, + now_time_us() - fault_begin_us); + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.fault_us, + now_time_us() - fault_begin_us); + } + } + return EXCEPTION_CONTINUE_SEARCH; +} + +internal B32 +lnk_cobj_cache_init(U32 segment_size) +{ + LONG init_state = InterlockedCompareExchange(&g_lnk_cobj_cache_init_state, 1, 0); + if (init_state != 0) { + while (g_lnk_cobj_cache_init_state == 1) { SwitchToThread(); } + return g_lnk_cobj_cache_init_state == 2 && g_lnk_cobj_cache.failed == 0 && + g_lnk_cobj_cache.segment_size == segment_size; + } + InitializeSRWLock(&g_lnk_cobj_cache.lock); + HMODULE kernelbase = GetModuleHandleA("KernelBase.dll"); + g_lnk_cobj_cache.map_view_of_file_3 = (void *)GetProcAddress(kernelbase, "MapViewOfFile3"); + g_lnk_cobj_cache.unmap_view_of_file_2 = (void *)GetProcAddress(kernelbase, "UnmapViewOfFile2"); + g_lnk_cobj_cache.virtual_alloc_2 = (void *)GetProcAddress(kernelbase, "VirtualAlloc2"); + if (!g_lnk_cobj_cache.map_view_of_file_3 || !g_lnk_cobj_cache.unmap_view_of_file_2 || !g_lnk_cobj_cache.virtual_alloc_2) { + g_lnk_cobj_cache.failed = 1; + g_lnk_cobj_cache.initialized = 1; + InterlockedExchange(&g_lnk_cobj_cache_init_state, 2); + return 0; + } + + if (!g_lnk_cobj_policy.configured) { lnk_compressed_obj_configure(0); } + U64 cache_bytes = g_lnk_cobj_policy.initial_cache_bytes; + U64 post_boundary_cache_bytes = g_lnk_cobj_policy.post_boundary_cache_bytes; + U64 segment_size_u64 = segment_size; + U64 input_capacity_hint = g_lnk_cobj_input_capacity_hint; + if (g_lnk_cobj_policy.initial_source == LNK_CObjPolicySource_Adaptive) { + // Keep at least one first-generation slot per write-group lane. This is only 32 MiB for + // 512 KiB segments, avoids serializing a wave of parallel first faults on tiny targets, + // and does not override explicit cache-size experiments. + input_capacity_hint = Max(input_capacity_hint, + segment_size_u64 * LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS); + } + if (input_capacity_hint > 0) { + if (input_capacity_hint < cache_bytes) { + cache_bytes = AlignPow2(input_capacity_hint, segment_size_u64); + } + if (input_capacity_hint < post_boundary_cache_bytes) { + post_boundary_cache_bytes = AlignPow2(input_capacity_hint, segment_size_u64); + } + } + g_lnk_cobj_cache.skip_cleanup = lnk_cobj_skip_cleanup_enabled(); + g_lnk_cobj_cache.initial_cache_bytes = cache_bytes; + g_lnk_cobj_cache.post_boundary_cache_bytes = post_boundary_cache_bytes; + g_lnk_cobj_cache.slot_count = Max(1, cache_bytes / segment_size); + g_lnk_cobj_cache.active_slot_count = g_lnk_cobj_cache.slot_count; + g_lnk_cobj_cache.segment_size = segment_size; + U64 mapping_size = g_lnk_cobj_cache.slot_count * segment_size; + g_lnk_cobj_cache.slot_mapping = CreateFileMappingW(INVALID_HANDLE_VALUE, 0, PAGE_READWRITE, + (DWORD)(mapping_size >> 32), (DWORD)mapping_size, 0); + if (g_lnk_cobj_cache.slot_mapping) { + g_lnk_cobj_cache.write_group_slot_count = LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS; + g_lnk_cobj_cache.write_groups = lnk_cobj_alloc_write_groups( + g_lnk_cobj_cache.slot_count, g_lnk_cobj_cache.write_group_slot_count, + &g_lnk_cobj_cache.write_group_count); + } + g_lnk_cobj_cache.slot_segments = VirtualAlloc(0, g_lnk_cobj_cache.slot_count * sizeof(void *), MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + // Keep the VEH lookup table fixed after publication, but make the ceiling high enough that + // it cannot constrain practical links. One million pointers cost only 8 MiB on x64. + g_lnk_cobj_cache.region_cap = LNK_COBJ_REGION_CAP; + g_lnk_cobj_cache.regions = VirtualAlloc(0, g_lnk_cobj_cache.region_cap * sizeof(void *), MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + if (!g_lnk_cobj_cache.slot_mapping || !g_lnk_cobj_cache.slot_segments || !g_lnk_cobj_cache.regions) { + g_lnk_cobj_cache.failed = 1; + } else { + g_lnk_cobj_cache.veh = AddVectoredExceptionHandler(1, lnk_cobj_veh); + if (!g_lnk_cobj_cache.veh) { g_lnk_cobj_cache.failed = 1; } + } + g_lnk_cobj_cache.initialized = 1; + InterlockedExchange(&g_lnk_cobj_cache_init_state, 2); + return g_lnk_cobj_cache.failed == 0; +} + +//////////////////////////////// +// Portable logical-address mapping + +internal B32 +lnk_cobj_segment_is_direct_raw(LNK_CObjSegment *segment) +{ + return (segment->flags & LNK_COBJ_SEGMENT_RAW) != 0; +} + +internal B32 +lnk_cobj_split_portable_raw_boundaries(LNK_CompressedObj *region) +{ + // Leave compressed runs intact. Raw runs still need exact placeholders because they are mapped + // directly from the portable file before any OBJ pointers are published. + for (U32 seg_idx = 1; seg_idx < region->segment_count; ++seg_idx) { + B32 prev_raw = lnk_cobj_segment_is_direct_raw(®ion->directory[seg_idx - 1]); + B32 curr_raw = lnk_cobj_segment_is_direct_raw(®ion->directory[seg_idx]); + if (prev_raw != curr_raw) { + U64 off = (U64)seg_idx * region->segment_size; + if (!VirtualFree(region->base + off, region->reserve_size - off, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER)) { + return 0; + } + } + } + LNK_CObjSegment *last = ®ion->directory[region->segment_count - 1]; + if (lnk_cobj_segment_is_direct_raw(last) && last->raw_size < region->segment_size) { + U64 map_end = (U64)(region->segment_count - 1) * region->segment_size + + AlignPow2(last->raw_size, LNK_COBJ_FILE_MAPPING_GRANULARITY); + if (map_end < region->reserve_size && + !VirtualFree(region->base + map_end, region->reserve_size - map_end, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER)) { + return 0; + } + } + return 1; +} + +internal B32 +lnk_cobj_isolate_portable_segment(LNK_CompressedObj *region, U32 seg_idx) +{ + LNK_CObjRuntimeSegment *runtime = ®ion->segments[seg_idx]; + AcquireSRWLockExclusive(®ion->placeholder_lock); + B32 ok = 1; + if (!runtime->isolated_placeholder) { + U32 first = seg_idx; + while (first > 0 && !region->segments[first - 1].isolated_placeholder && + !lnk_cobj_segment_is_direct_raw(®ion->directory[first - 1])) { + first -= 1; + } + U32 opl = seg_idx + 1; + while (opl < region->segment_count && !region->segments[opl].isolated_placeholder && + !lnk_cobj_segment_is_direct_raw(®ion->directory[opl])) { + opl += 1; + } + U8 *target = region->base + (U64)seg_idx * region->segment_size; + if (seg_idx > first) { + ok = VirtualFree(target, (U64)(opl - seg_idx) * region->segment_size, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER); + } + if (ok && seg_idx + 1 < opl) { + ok = VirtualFree(target + region->segment_size, + (U64)(opl - seg_idx - 1) * region->segment_size, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER); + } + if (ok) { runtime->isolated_placeholder = 1; } + } + ReleaseSRWLockExclusive(®ion->placeholder_lock); + return ok; +} + +internal void +lnk_cobj_unmap_portable_raw_runs(LNK_CompressedObj *region) +{ + for (U32 first = 0; first < region->segment_count;) { + LNK_CObjSegment *entry = ®ion->directory[first]; + if (!lnk_cobj_segment_is_direct_raw(entry)) { first += 1; continue; } + U32 opl = first + 1; + while (opl < region->segment_count && + lnk_cobj_segment_is_direct_raw(®ion->directory[opl]) && + region->directory[opl].file_offset == region->directory[opl - 1].file_offset + region->segment_size) { + opl += 1; + } + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), + region->base + (U64)first * region->segment_size, + MEM_PRESERVE_PLACEHOLDER); + first = opl; + } +} + +internal B32 +lnk_cobj_map_portable_raw_runs(LNK_CompressedObj *region) +{ + for (U32 first = 0; first < region->segment_count;) { + LNK_CObjSegment *entry = ®ion->directory[first]; + if (!lnk_cobj_segment_is_direct_raw(entry)) { first += 1; continue; } + U32 opl = first + 1; + while (opl < region->segment_count && + lnk_cobj_segment_is_direct_raw(®ion->directory[opl]) && + region->directory[opl].file_offset == region->directory[opl - 1].file_offset + region->segment_size) { + opl += 1; + } + U64 map_size = (U64)(opl - first) * region->segment_size; + if (opl == region->segment_count) { + LNK_CObjSegment *last = ®ion->directory[opl - 1]; + if (last->raw_size < region->segment_size) { + map_size -= region->segment_size - AlignPow2(last->raw_size, LNK_COBJ_FILE_MAPPING_GRANULARITY); + } + } + U64 file_off = entry->file_offset; + U8 *target = region->base + (U64)first * region->segment_size; + if ((file_off & (LNK_COBJ_FILE_MAPPING_GRANULARITY - 1)) != 0 || + file_off > region->mapped_file.size || + map_size > region->mapped_file.size - file_off || + g_lnk_cobj_cache.map_view_of_file_3(region->portable_mapping, GetCurrentProcess(), target, + file_off, map_size, MEM_REPLACE_PLACEHOLDER, + PAGE_READONLY, 0, 0) != target) { + lnk_cobj_unmap_portable_raw_runs(region); + return 0; + } + first = opl; + } + return 1; +} + +internal void +lnk_cobj_release_placeholder_reservation(LNK_CompressedObj *region) +{ + LNK_CObjSegment *last = ®ion->directory[region->segment_count - 1]; + B32 split_final_raw = lnk_cobj_segment_is_direct_raw(last) && + last->raw_size < region->segment_size; + if (region->segment_count > 1 || split_final_raw) { + VirtualFree(region->base, region->reserve_size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS); + } + VirtualFree(region->base, 0, MEM_RELEASE); +} + +internal void +lnk_cobj_discard_unpublished_region(LNK_CompressedObj *region) +{ + if (!region) { return; } + region->active = 0; + lnk_cobj_unmap_portable_raw_runs(region); + lnk_cobj_release_placeholder_reservation(region); + if (region->portable_mapping) { CloseHandle(region->portable_mapping); } + HeapFree(GetProcessHeap(), 0, region); +} + +//////////////////////////////// +// Container validation and lifetime + +internal B32 +lnk_cobj_open_portable(LNK_Input *input, String8 mapped_file) +{ + if (mapped_file.size < sizeof(LNK_CObjHeader)) { return 0; } + LNK_CObjHeader *header = (LNK_CObjHeader *)mapped_file.str; + if (header->magic != LNK_COBJ_MAGIC) { return 0; } + B32 has_invalid_flag_dependency = + ((header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) && + !(header->flags & LNK_COBJ_FLAG_TYPE_INDEX)) || + ((header->flags & LNK_COBJ_FLAG_DEBUG_S_SUMMARY) && + !(header->flags & LNK_COBJ_FLAG_DEBUG_S_INDEX)) || + ((header->flags & LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2) && + !(header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR)) || + ((header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR) && + !(header->flags & LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2)) || + ((header->flags & LNK_COBJ_TYPE_INDEX_COUNT_MASK) && + !(header->flags & LNK_COBJ_FLAG_TYPE_INDEX)); + if (header->version != LNK_COBJ_VERSION || header->header_size != sizeof(*header) || + header->raw_size == 0 || + header->segment_size < LNK_COBJ_FILE_MAPPING_GRANULARITY || + (header->segment_size & (header->segment_size - 1)) != 0 || + header->raw_size > max_U64 - (header->segment_size - 1) || + header->segment_count != CeilIntegerDiv(header->raw_size, header->segment_size) || + header->directory_offset < header->header_size || header->directory_offset > mapped_file.size || + (header->flags & ~LNK_COBJ_KNOWN_HEADER_FLAGS) != 0 || has_invalid_flag_dependency || + !(header->flags & LNK_COBJ_FLAG_PORTABLE_RAW_MAP) || + (U64)header->segment_count * sizeof(LNK_CObjSegment) > mapped_file.size - header->directory_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object container: %S", input->path); + return 0; + } + LNK_CObjSegment *directory = (LNK_CObjSegment *)(mapped_file.str + header->directory_offset); + B32 has_portable_raw_segments = 0; + U64 raw_mapped_segment_count = 0; + for (U32 i = 0; i < header->segment_count; ++i) { + LNK_CObjSegment *entry = &directory[i]; + U64 segment_raw_offset = (U64)i * header->segment_size; + U32 expected_raw_size = (U32)Min((U64)header->segment_size, + header->raw_size - segment_raw_offset); + if (entry->raw_size != expected_raw_size || entry->stored_size == 0 || + (entry->flags & ~LNK_COBJ_SEGMENT_RAW) != 0 || + ((entry->flags & LNK_COBJ_SEGMENT_RAW) && entry->stored_size != entry->raw_size) || + entry->file_offset > mapped_file.size || entry->stored_size > mapped_file.size - entry->file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object segment: %S", input->path); + return 0; + } + has_portable_raw_segments |= !!(entry->flags & LNK_COBJ_SEGMENT_RAW); + raw_mapped_segment_count += !!(entry->flags & LNK_COBJ_SEGMENT_RAW); + } + U32 type_index_count = 0; + LNK_CObjTypeIndex *type_indices = 0; + LNK_CObjUdtHashIndex *udt_hash_indices = 0; + LNK_CObjBaseRelocIndex *base_reloc_index = 0; + LNK_CObjDebugSIndex *debug_s_index = 0; + if (header->flags & LNK_COBJ_FLAG_TYPE_INDEX) { + type_index_count = (header->flags & LNK_COBJ_TYPE_INDEX_COUNT_MASK) >> LNK_COBJ_TYPE_INDEX_COUNT_SHIFT; + U64 dir_bytes = (U64)type_index_count * sizeof(LNK_CObjTypeIndex); + if (type_index_count == 0 || header->reserved > mapped_file.size || dir_bytes > mapped_file.size - header->reserved) { + lnk_error(LNK_Error_IllData, "invalid compressed object type index: %S", input->path); + return 0; + } + type_indices = (LNK_CObjTypeIndex *)(mapped_file.str + header->reserved); + for (U32 i = 0; i < type_index_count; ++i) { + LNK_CObjTypeIndex *idx = &type_indices[i]; + U64 offset_count = idx->leaf_count; + U64 offsets_bytes = offset_count * sizeof(U32); + U64 sizes_bytes = offset_count * sizeof(U16); + U64 kinds_bytes = offset_count * sizeof(U16); + if (header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR) { + sizes_bytes = 256 * sizeof(U16); + kinds_bytes = offset_count; + U64 group_size = (U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_count = (offset_count + group_size - 1) / group_size; + U64 group_bytes = group_count * 2 * sizeof(U32); + U64 payload_rel = (group_bytes + 7) & ~(U64)7; + if (idx->offsets_file_offset > mapped_file.size || group_bytes > mapped_file.size - idx->offsets_file_offset || + idx->sizes_file_offset < idx->offsets_file_offset || + idx->sizes_file_offset - idx->offsets_file_offset < payload_rel) { + lnk_error(LNK_Error_IllData, "invalid packed v2 type offset directory: %S", input->path); + return 0; + } + U64 payload_bytes = idx->sizes_file_offset - idx->offsets_file_offset - payload_rel; + U32 *groups = (U32 *)(mapped_file.str + idx->offsets_file_offset); + for (U64 group_idx = 0; group_idx < group_count; ++group_idx) { + U64 first = group_idx * group_size; + U64 count = Min(group_size, offset_count - first); + U32 descriptor = groups[group_idx*2 + 1]; + U64 rel = descriptor & ~(U32)1; + U64 width = (descriptor & 1) ? 3 : 2; + if (rel > payload_bytes || count * width > payload_bytes - rel) { + lnk_error(LNK_Error_IllData, "invalid packed v2 type offset payload: %S", input->path); + return 0; + } + } + offsets_bytes = idx->sizes_file_offset - idx->offsets_file_offset; + } + if (idx->raw_section_offset > header->raw_size || idx->raw_section_size > header->raw_size - idx->raw_section_offset || + idx->offsets_file_offset > mapped_file.size || offsets_bytes > mapped_file.size - idx->offsets_file_offset || + idx->sizes_file_offset > mapped_file.size || sizes_bytes > mapped_file.size - idx->sizes_file_offset || + idx->kinds_file_offset > mapped_file.size || kinds_bytes > mapped_file.size - idx->kinds_file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object type index arrays: %S", input->path); + return 0; + } + } + if (header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) { + U64 udt_dir_off = header->reserved + dir_bytes; + U64 udt_dir_bytes = (U64)type_index_count * sizeof(LNK_CObjUdtHashIndex); + if (udt_dir_off > mapped_file.size || udt_dir_bytes > mapped_file.size - udt_dir_off) return 0; + udt_hash_indices = (LNK_CObjUdtHashIndex *)(mapped_file.str + udt_dir_off); + for (U32 i = 0; i < type_index_count; ++i) { + LNK_CObjUdtHashIndex *idx = &udt_hash_indices[i]; + U64 bytes = (U64)idx->hash_count * sizeof(U64); + if (idx->hashes_file_offset > mapped_file.size || bytes > mapped_file.size - idx->hashes_file_offset) return 0; + } + } + } + + if (header->flags & LNK_COBJ_FLAG_BASE_RELOC_INDEX) { + U64 dir_off = header->reserved; + if (header->flags & LNK_COBJ_FLAG_TYPE_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjTypeIndex); + if (header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjUdtHashIndex); + } + } + if (dir_off > mapped_file.size || sizeof(LNK_CObjBaseRelocIndex) > mapped_file.size - dir_off) { + lnk_error(LNK_Error_IllData, "invalid compressed object base relocation index: %S", input->path); + return 0; + } + base_reloc_index = (LNK_CObjBaseRelocIndex *)(mapped_file.str + dir_off); + U64 entry_bytes = (U64)base_reloc_index->entry_count * sizeof(LNK_CObjBaseRelocEntry); + if (base_reloc_index->entries_file_offset > mapped_file.size || + entry_bytes > mapped_file.size - base_reloc_index->entries_file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object base relocation entries: %S", input->path); + return 0; + } + } + if (header->flags & LNK_COBJ_FLAG_DEBUG_S_INDEX) { + U64 dir_off = header->reserved; + if (header->flags & LNK_COBJ_FLAG_TYPE_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjTypeIndex); + if (header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjUdtHashIndex); + } + } + if (header->flags & LNK_COBJ_FLAG_BASE_RELOC_INDEX) { dir_off += sizeof(LNK_CObjBaseRelocIndex); } + if (dir_off > mapped_file.size || sizeof(LNK_CObjDebugSIndex) > mapped_file.size - dir_off) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S index: %S", input->path); + return 0; + } + debug_s_index = (LNK_CObjDebugSIndex *)(mapped_file.str + dir_off); + U64 entry_bytes = (U64)debug_s_index->entry_count * sizeof(LNK_CObjDebugSEntry); + if (debug_s_index->entries_file_offset > mapped_file.size || + entry_bytes > mapped_file.size - debug_s_index->entries_file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S entries: %S", input->path); + return 0; + } + LNK_CObjDebugSEntry *entries = (LNK_CObjDebugSEntry *)(mapped_file.str + debug_s_index->entries_file_offset); + for (U32 i = 0; i < debug_s_index->entry_count; ++i) { + if ((U64)entries[i].raw_payload_offset + entries[i].raw_payload_size > header->raw_size || + (i && entries[i].raw_section_offset < entries[i-1].raw_section_offset)) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S entry: %S", input->path); + return 0; + } + } + if (header->flags & LNK_COBJ_FLAG_DEBUG_S_SUMMARY) { + U64 summary_off = AlignPow2(debug_s_index->entries_file_offset + entry_bytes, 8); + U64 summary_bytes = (U64)debug_s_index->entry_count * sizeof(LNK_CObjDebugSSummary); + if (summary_off > mapped_file.size || summary_bytes > mapped_file.size - summary_off) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S summaries: %S", input->path); + return 0; + } + } + } + if (!lnk_cobj_cache_init(header->segment_size)) { + lnk_error(LNK_Error_IllData, "compressed object cache initialization failed"); + return 0; + } + + HANDLE portable_mapping = 0; + if (has_portable_raw_segments) { + Temp scratch = scratch_begin(0, 0); + String16 path16 = str16_from_8(scratch.arena, input->path); + HANDLE file = CreateFileW(path16.str, GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + if (file != INVALID_HANDLE_VALUE) { + // Match ordinary OBJ mapping semantics: expose raw runs through a read-only view of a + // WRITECOPY section. Untouched pages stay file-backed without up-front commit, while the + // generic COW VEH can privately promote the rare page patched in place by the linker. + portable_mapping = CreateFileMappingW(file, 0, PAGE_WRITECOPY, 0, 0, 0); + CloseHandle(file); + } + scratch_end(scratch); + } + // A fully compressed container is self-contained in mapped_file and deliberately needs no + // second handle to its path. Only raw-stored segments require a WRITECOPY section that can be + // mapped at their logical OBJ offsets. + if (has_portable_raw_segments && !portable_mapping) { + lnk_error(LNK_Error_IllData, "unable to open portable compressed object mapping: %S", input->path); + return 0; + } + + U64 reserve_size = (header->raw_size + (U64)header->segment_size - 1) & ~((U64)header->segment_size - 1); + U8 *base = g_lnk_cobj_cache.virtual_alloc_2(GetCurrentProcess(), 0, reserve_size, + MEM_RESERVE|MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, 0, 0); + if (!base) { + if (portable_mapping) { CloseHandle(portable_mapping); } + lnk_error(LNK_Error_IllData, "unable to reserve compressed object view: %S", input->path); + return 0; + } + + // These descriptors need stable addresses, not independent VADs. One heap block per OBJ avoids + // two VirtualAlloc VADs and also halves allocator calls versus separate descriptor/table blocks. + U64 segments_off = AlignPow2(sizeof(LNK_CompressedObj), 64); + U64 metadata_size = segments_off + sizeof(LNK_CObjRuntimeSegment) * header->segment_count; + LNK_CompressedObj *region = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, metadata_size); + LNK_CObjRuntimeSegment *segments = region ? (LNK_CObjRuntimeSegment *)((U8 *)region + segments_off) : 0; + if (!region) { + VirtualFree(base, 0, MEM_RELEASE); + if (portable_mapping) { CloseHandle(portable_mapping); } + lnk_error(LNK_Error_IllData, "unable to allocate compressed object metadata: %S", input->path); + return 0; + } + for (U32 i = 0; i < header->segment_count; ++i) { + InitializeSRWLock(&segments[i].lock); + segments[i].slot_idx = max_U32; + segments[i].address = base + (U64)i * header->segment_size; + } + region->base = base; + region->reserve_size = reserve_size; + region->raw_size = header->raw_size; + region->segment_size = header->segment_size; + region->segment_count = header->segment_count; + region->mapped_file = mapped_file; + region->header = header; + region->directory = directory; + region->type_indices = type_indices; + region->udt_hash_indices = udt_hash_indices; + region->segments = segments; + region->portable_mapping = portable_mapping; + if (base_reloc_index) { + region->base_reloc_entries = (LNK_CObjBaseRelocEntry *)(mapped_file.str + base_reloc_index->entries_file_offset); + region->base_reloc_entry_count = base_reloc_index->entry_count; + region->has_base_reloc_index = 1; + } + if (debug_s_index) { + region->debug_s_entries = (LNK_CObjDebugSEntry *)(mapped_file.str + debug_s_index->entries_file_offset); + region->debug_s_entry_count = debug_s_index->entry_count; + if (header->flags & LNK_COBJ_FLAG_DEBUG_S_SUMMARY) { + U64 bytes = (U64)debug_s_index->entry_count * sizeof(LNK_CObjDebugSEntry); + U64 off = AlignPow2(debug_s_index->entries_file_offset + bytes, 8); + region->debug_s_summaries = (LNK_CObjDebugSSummary *)(mapped_file.str + off); + } + } + region->type_index_count = type_index_count; + region->active = 1; + + B32 placeholders_ok = lnk_cobj_split_portable_raw_boundaries(region) && + lnk_cobj_map_portable_raw_runs(region); + if (!placeholders_ok) { + lnk_cobj_discard_unpublished_region(region); + lnk_error(LNK_Error_IllData, "unable to map portable compressed object view: %S", input->path); + return 0; + } + + B32 registered = 0; + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (g_lnk_cobj_cache.region_count < g_lnk_cobj_cache.region_cap) { + g_lnk_cobj_cache.regions[g_lnk_cobj_cache.region_count++] = region; + g_lnk_cobj_cache.raw_mapped_segments += raw_mapped_segment_count; + g_lnk_cobj_cache.input_compressed_bytes += mapped_file.size; + g_lnk_cobj_cache.input_raw_bytes += header->raw_size; + registered = 1; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (!registered) { + lnk_cobj_discard_unpublished_region(region); + lnk_error(LNK_Error_IllData, "too many compressed object inputs"); + return 0; + } + + input->compressed_obj = region; + input->compressed_data = mapped_file; + input->data = str8(base, header->raw_size); + return 1; +} + +internal B32 +lnk_compressed_obj_open(LNK_Input *input, String8 mapped_file) +{ + return lnk_cobj_open_portable(input, mapped_file); +} + +internal void +lnk_compressed_obj_finalize_open(void) +{ + // Parallel input mapping appends regions in completion order. The caller invokes this joined + // boundary before starting any COFF parser task, so no logical compressed range can fault until + // the address table has been sorted for lock-free VEH lookup. + // Archive, generated-object and empty input batches usually add no compressed + // regions. The table is append-only; do not sort the same addresses again. + if (!g_lnk_cobj_cache.initialized || g_lnk_cobj_cache.region_count < 2 || + g_lnk_cobj_cache.region_count == g_lnk_cobj_cache.sorted_region_count) { return; } + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + qsort(g_lnk_cobj_cache.regions, g_lnk_cobj_cache.region_count, + sizeof(*g_lnk_cobj_cache.regions), lnk_cobj_region_ptr_compare); + g_lnk_cobj_cache.sorted_region_count = g_lnk_cobj_cache.region_count; + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); +} + +internal void +lnk_compressed_obj_close(LNK_Input *input) +{ + LNK_CompressedObj *region = input->compressed_obj; + if (!region) { return; } + U64 cleanup_begin_us = now_time_us(); + if (g_lnk_cobj_cache.skip_cleanup) { + // Benchmark/diagnostic mode: the process is about to exit and Windows tears the address + // space down more efficiently than issuing one UnmapViewOfFile2 call per resident segment. + // Useful for separating link work from explicit VAD teardown; not the bounded-cache path. + input->owns_file_map = 0; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.cleanup_us, + now_time_us() - cleanup_begin_us); + return; + } + InterlockedExchange(®ion->active, 0); + lnk_cobj_unmap_portable_raw_runs(region); + for (U32 i = 0; i < region->segment_count; ++i) { + LNK_CObjRuntimeSegment *segment = ®ion->segments[i]; + if (segment->state == LNK_CObjSegState_Ready || segment->state == LNK_CObjSegState_Pinned) { + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), region->base + (U64)i * region->segment_size, + MEM_PRESERVE_PLACEHOLDER); + if (segment->slot_idx != max_U32 && segment->slot_idx < g_lnk_cobj_cache.slot_count) { + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (g_lnk_cobj_cache.slot_segments[segment->slot_idx] == segment) { + g_lnk_cobj_cache.slot_segments[segment->slot_idx] = 0; + // Destructive shrinking can leave pinned slots above the active prefix. They remain + // mapped for pointer stability but are intentionally excluded from resident_count. + if (segment->slot_idx < g_lnk_cobj_cache.active_slot_count) { + Assert(g_lnk_cobj_cache.resident_count > 0); + g_lnk_cobj_cache.resident_count -= 1; + } + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + } + } + } + lnk_cobj_release_placeholder_reservation(region); + if (region->portable_mapping) { CloseHandle(region->portable_mapping); } + file_map_view_close((FileMap){0}, input->compressed_data.str, r1u64(0, input->compressed_data.size)); + // The sorted address lookup retains region pointers for the life of the process. Keep this + // small inactive descriptor allocated so an unrelated later access violation cannot make the + // vectored exception handler dereference freed metadata. + region->mapped_file = str8_zero(); + region->header = 0; + region->directory = 0; + region->type_indices = 0; + region->udt_hash_indices = 0; + region->base_reloc_entries = 0; + region->debug_s_entries = 0; + region->debug_s_summaries = 0; + region->portable_mapping = 0; + input->data = str8_zero(); + input->compressed_data = str8_zero(); + input->compressed_obj = 0; + input->owns_file_map = 0; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.cleanup_us, + now_time_us() - cleanup_begin_us); +} + +//////////////////////////////// +// Statistics + +internal void +lnk_compressed_obj_log_stats(void) +{ + if (!g_lnk_cobj_cache.initialized || g_lnk_cobj_cache.region_count == 0) { + if (g_lnk_cobj_window_decodes) { lnk_log(LNK_Log_Timers, "[cobj window] decodes=%llu stored=%llu MiB", g_lnk_cobj_window_decodes, g_lnk_cobj_window_bytes / MB(1)); } + return; + } + U64 unique_decodes = g_lnk_cobj_cache.decodes - g_lnk_cobj_cache.redecodes; + U64 total_decodes = g_lnk_cobj_cache.decodes + g_lnk_cobj_window_decodes; + U64 total_decoded_bytes = g_lnk_cobj_cache.decoded_bytes + g_lnk_cobj_window_decoded_bytes; + U64 total_stored_bytes = g_lnk_cobj_cache.stored_bytes_read + g_lnk_cobj_window_bytes; + lnk_log(LNK_Log_Timers, + "[cobj] files=%llu input=%llu/%llu MiB segment=%llu KiB cache=%llu MiB(%s)->%llu MiB(%s) freeze=%u(%s) trim=%u(%s) one-shot=%u(%s) resident-hwm=%llu faults=%llu unique=%llu decodes=%llu decoded=%llu MiB read=%llu MiB evictions=%llu redecodes=%llu frozen=%llu/%llu MiB raw-mapped=%llu decoder-scratch=%llu/%llu KiB time-ms{decode=%.2f fault=%.2f evict=%.2f freeze=%.2f trim=%.2f cleanup=%.2f}\n", + g_lnk_cobj_cache.region_count, + g_lnk_cobj_cache.input_compressed_bytes / MB(1), + g_lnk_cobj_cache.input_raw_bytes / MB(1), + g_lnk_cobj_cache.segment_size / KB(1), + g_lnk_cobj_cache.initial_cache_bytes / MB(1), + lnk_cobj_policy_source_string(g_lnk_cobj_policy.initial_source), + g_lnk_cobj_cache.post_boundary_cache_bytes / MB(1), + lnk_cobj_policy_source_string(g_lnk_cobj_policy.post_source), + g_lnk_cobj_policy.freeze_generation, + lnk_cobj_policy_source_string(g_lnk_cobj_policy.freeze_source), + g_lnk_cobj_policy.trim_mode, + lnk_cobj_policy_source_string(g_lnk_cobj_policy.trim_source), + g_lnk_cobj_policy.one_shot, + lnk_cobj_policy_source_string(g_lnk_cobj_policy.one_shot_source), + g_lnk_cobj_cache.occupancy_hwm, g_lnk_cobj_cache.faults, unique_decodes, total_decodes, + total_decoded_bytes / MB(1), total_stored_bytes / MB(1), + g_lnk_cobj_cache.evictions, + g_lnk_cobj_cache.redecodes, + g_lnk_cobj_cache.frozen_segments, + (g_lnk_cobj_cache.frozen_segments * g_lnk_cobj_cache.segment_size) / MB(1), + g_lnk_cobj_cache.raw_mapped_segments, + g_lnk_cobj_decoder_scratch_count, + g_lnk_cobj_decoder_scratch_bytes / KB(1), + (F64)g_lnk_cobj_cache.decode_us / 1000.0, + (F64)g_lnk_cobj_cache.fault_us / 1000.0, + (F64)g_lnk_cobj_cache.eviction_us / 1000.0, + (F64)g_lnk_cobj_cache.freeze_us / 1000.0, + (F64)g_lnk_cobj_cache.trim_us / 1000.0, + (F64)g_lnk_cobj_cache.cleanup_us / 1000.0); + if (g_lnk_cobj_cache.frozen_segments) { + lnk_log(LNK_Log_Timers, "[cobj frozen] segments=%llu", g_lnk_cobj_cache.frozen_segments); + } + if (g_lnk_cobj_cache.redecodes) { + lnk_log(LNK_Log_Timers, "[cobj redecodes] debug_s=%llu types=%llu segment0=%llu", + g_lnk_cobj_redecode_debug_s, g_lnk_cobj_redecode_types, + g_lnk_cobj_redecode_segment_zero); + } + if (g_lnk_cobj_window_decodes) { + lnk_log(LNK_Log_Timers, "[cobj window] decodes=%llu stored=%llu MiB", + g_lnk_cobj_window_decodes, g_lnk_cobj_window_bytes / MB(1)); + } +} + +#else +internal void lnk_compressed_obj_configure(struct LNK_Config *config) { (void)config; } +internal void lnk_compressed_obj_prepare_cache(String8 *mapped_files, U64 count) { (void)mapped_files; (void)count; } +internal void lnk_compressed_obj_trim_working_set(void) {} +internal String8 lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range) { return str8_zero(); } +internal B32 lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out) { MemoryZeroStruct(out); return 0; } +internal B32 lnk_compressed_obj_base_reloc_index(LNK_CompressedObj *obj, LNK_CObjBaseRelocView *out) { MemoryZeroStruct(out); return 0; } +internal B32 lnk_compressed_obj_debug_s_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjDebugSView *out) { MemoryZeroStruct(out); return 0; } +internal B32 lnk_compressed_obj_copy_range(LNK_CompressedObj *obj, Rng1U64 range, void *dst, LNK_CObjDecodeWindow *window) { return 0; } +internal B32 lnk_compressed_obj_copy_string(LNK_CompressedObj *obj, String8 src, void *dst, LNK_CObjDecodeWindow *window) { return 0; } +internal void lnk_compressed_obj_release_window(LNK_CObjDecodeWindow *window) {} +internal U32 lnk_compressed_obj_segment_count(LNK_CompressedObj *obj) { return 0; } +internal U32 lnk_compressed_obj_segment_size(LNK_CompressedObj *obj) { return 0; } +internal Rng1U64 lnk_compressed_obj_stored_segment_range(LNK_CompressedObj *obj, U32 segment_idx) { Rng1U64 r = {0}; return r; } +internal B32 +lnk_compressed_obj_open(struct LNK_Input *input, String8 mapped_file) +{ + if (mapped_file.size >= sizeof(LNK_CObjHeader) && + ((LNK_CObjHeader *)mapped_file.str)->magic == LNK_COBJ_MAGIC) { + lnk_error(LNK_Error_IllData, + "compressed object input requires an Oodle-enabled RAD Link build: %S", input->path); + } + return 0; +} +internal void lnk_compressed_obj_finalize_open(void) {} +internal void lnk_compressed_obj_close(struct LNK_Input *input) {} +internal void lnk_compressed_obj_log_stats(void) {} +internal void lnk_compressed_obj_log_phase_stats(char *tag) {} +#endif diff --git a/src/linker/lnk_compressed_obj.h b/src/linker/lnk_compressed_obj.h new file mode 100644 index 000000000..0631de85c --- /dev/null +++ b/src/linker/lnk_compressed_obj.h @@ -0,0 +1,68 @@ +#pragma once + +// The portable container layout is codec-independent and is also needed by the no-Oodle stubs. +// Oodle headers and calls remain private to the enabled branch in lnk_compressed_obj.c. +#include "lnk_compressed_obj_format.h" + +typedef struct LNK_CompressedObj LNK_CompressedObj; +typedef struct LNK_CObjBaseRelocEntry LNK_CObjBaseRelocEntry; +typedef struct LNK_CObjDebugSEntry LNK_CObjDebugSEntry; +typedef struct LNK_CObjDebugSSummary LNK_CObjDebugSSummary; + +typedef struct LNK_CObjTypeIndexView +{ + U64 count; + U32 *offsets; + U16 *sizes; + U16 *kinds; + U32 *packed_v2_offset_groups; + U8 *packed_v2_offset_payload; + U16 *packed_kind_dictionary; + U8 *packed_kind_codes; + U64 *complete_udt_hashes; + U64 complete_udt_hash_count; + U8 offset_checkpoint_shift; + B8 packed_sidecar; +} LNK_CObjTypeIndexView; + +typedef struct LNK_CObjBaseRelocView +{ + U32 count; + LNK_CObjBaseRelocEntry *v; +} LNK_CObjBaseRelocView; + +typedef struct LNK_CObjDebugSView +{ + U32 count; + LNK_CObjDebugSEntry *v; + LNK_CObjDebugSSummary *summaries; +} LNK_CObjDebugSView; + +typedef struct LNK_CObjDecodeWindow LNK_CObjDecodeWindow; +struct LNK_CObjDecodeWindow +{ + LNK_CompressedObj *obj; + U8 *buffer; + U64 buffer_cap; + U32 seg_idx; + U32 valid_size; +}; + +internal void lnk_compressed_obj_configure(struct LNK_Config *config); +internal void lnk_compressed_obj_prepare_cache(String8 *mapped_files, U64 count); +internal void lnk_compressed_obj_trim_working_set(void); +internal String8 lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range); +internal B32 lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out); +internal B32 lnk_compressed_obj_base_reloc_index(LNK_CompressedObj *obj, LNK_CObjBaseRelocView *out); +internal B32 lnk_compressed_obj_debug_s_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjDebugSView *out); +internal B32 lnk_compressed_obj_copy_range(LNK_CompressedObj *obj, Rng1U64 range, void *dst, LNK_CObjDecodeWindow *window); +internal B32 lnk_compressed_obj_copy_string(LNK_CompressedObj *obj, String8 src, void *dst, LNK_CObjDecodeWindow *window); +internal void lnk_compressed_obj_release_window(LNK_CObjDecodeWindow *window); +internal U32 lnk_compressed_obj_segment_count(LNK_CompressedObj *obj); +internal U32 lnk_compressed_obj_segment_size(LNK_CompressedObj *obj); +internal Rng1U64 lnk_compressed_obj_stored_segment_range(LNK_CompressedObj *obj, U32 segment_idx); +internal B32 lnk_compressed_obj_open(struct LNK_Input *input, String8 mapped_file); +internal void lnk_compressed_obj_finalize_open(void); +internal void lnk_compressed_obj_close(struct LNK_Input *input); +internal void lnk_compressed_obj_log_stats(void); +internal void lnk_compressed_obj_log_phase_stats(char *tag); diff --git a/src/linker/lnk_compressed_obj_format.h b/src/linker/lnk_compressed_obj_format.h new file mode 100644 index 000000000..b76b2c3fc --- /dev/null +++ b/src/linker/lnk_compressed_obj_format.h @@ -0,0 +1,124 @@ +// Portable segmented-object format shared by the writer and RAD Link reader. +// Files are self-contained, all integers are little endian, and each compressed +// segment is independently decodable. + +#pragma once + +#define LNK_COBJ_MAGIC 0x3130304A424F4C52ull /* "RLOBJ001" */ +#define LNK_COBJ_VERSION 1u +#define LNK_COBJ_SEGMENT_RAW 0x00000001u +#define LNK_COBJ_FLAG_TYPE_INDEX 0x00000001u +#define LNK_COBJ_FLAG_RETIRED_SPARSE_RAW_VIEW 0x00000002u // Reserved; never accepted by the portable reader. +#define LNK_COBJ_FLAG_UDT_HASH_INDEX 0x00000004u +#define LNK_COBJ_FLAG_BASE_RELOC_INDEX 0x00000008u +#define LNK_COBJ_FLAG_PORTABLE_RAW_MAP 0x00000010u +#define LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR 0x00000040u +#define LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2 0x00000080u +#define LNK_COBJ_FLAG_DEBUG_S_INDEX 0x00002000u +#define LNK_COBJ_FLAG_DEBUG_S_SUMMARY 0x00004000u +#define LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT 9u +#define LNK_COBJ_TYPE_INDEX_COUNT_SHIFT 16u +#define LNK_COBJ_TYPE_INDEX_COUNT_MASK 0xffff0000u + +typedef struct LNK_CObjHeader +{ + U64 magic; + U32 version; + U32 header_size; + U64 raw_size; + U32 segment_size; + U32 segment_count; + U64 directory_offset; + U64 data_offset; + U32 compressor; + U32 flags; + U64 reserved; +} LNK_CObjHeader; + +typedef struct LNK_CObjSegment +{ + U64 file_offset; + U32 stored_size; + U32 raw_size; + U32 flags; + U32 reserved; +} LNK_CObjSegment; + +// Optional leaf index for .debug$T/.debug$P sections. The three arrays are +// deliberately stored separately: type dedup scans only kinds, while offsets +// and sizes are touched only for the winning leaves. All offsets are file +// offsets in the compressed container; leaf offsets are relative to the first +// byte after the four-byte CodeView signature. +typedef struct LNK_CObjTypeIndex +{ + U64 raw_section_offset; + U32 raw_section_size; + U32 leaf_count; + // Normal layout: U32 offsets, U16 sizes, U16 kinds. + // PACKED_TYPE_SIDECAR: + // U32 group[ceil(leaf_count/512)][2] = {absolute_base, payload_offset|is_u24}; + // followed by U16 deltas for normal groups and U24 deltas for rare wide groups; + // sizes_file_offset -> U16 kind_dictionary[256] (sizes derive from adjacent offsets); + // kinds_file_offset -> U8 kind_code[leaf_count]. + U64 offsets_file_offset; + U64 sizes_file_offset; + U64 kinds_file_offset; +} LNK_CObjTypeIndex; + +typedef struct LNK_CObjUdtHashIndex +{ + U64 hashes_file_offset; + U32 hash_count; + U32 reserved; +} LNK_CObjUdtHashIndex; + +// Compact candidates for PE base relocation generation. The compressor filters the full COFF +// relocation streams down to address relocations; RAD Link still applies section-liveness and +// symbol-interpretation checks because those depend on the final link. +typedef struct LNK_CObjBaseRelocIndex +{ + U64 entries_file_offset; + U32 entry_count; + U32 reserved; +} LNK_CObjBaseRelocIndex; + +typedef struct LNK_CObjBaseRelocEntry +{ + U32 sect_idx; + U32 apply_off; + U32 isymbol; + U8 addr_size; + U8 reserved[3]; +} LNK_CObjBaseRelocEntry; + +// Compact directory for C13 subsection payloads. Parsing .debug$S normally walks every +// subsection header in the logical COFF view, which needlessly decodes all compressed symbol +// segments just to construct String8 slices. Entries are grouped by raw_section_offset and let +// the linker construct those slices without touching payload pages. COFF section file offsets +// are U32 by definition, so the complete entry stays at 16 bytes. +typedef struct LNK_CObjDebugSIndex +{ + U64 entries_file_offset; + U32 entry_count; + U32 reserved; +} LNK_CObjDebugSIndex; + +typedef struct LNK_CObjDebugSEntry +{ + U32 raw_section_offset; + U32 raw_payload_offset; + U32 raw_payload_size; + U32 kind; +} LNK_CObjDebugSEntry; + +// Size-only module/GSI prepass results for the corresponding DEBUG_S_INDEX entry. These values +// depend only on symbol size/kind fields, which relocation and type-index fixups never alter. +typedef struct LNK_CObjDebugSSummary +{ + U32 module_symbol_size; + U32 gsi_candidate_count; + U32 proc_ref_count; + U32 flags; +} LNK_CObjDebugSSummary; + +#define LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS 0x1u diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 4e0993319..1b441e784 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -68,6 +68,11 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_WholeArchive, 0, LNK_CmdValueKind_Scalar, "WHOLEARCHIVE", "[:LIBNAME]", "Force linker to pull in all objs from the specified lib." }, { LNK_CmdSwitch_Rad_Age, 0, LNK_CmdValueKind_Scalar, "RAD_AGE", ":#", "Age embeded in EXE and PDB, used to validate incremental build. Default is 1." }, + { LNK_CmdSwitch_Rad_CObjCacheGiB, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_CACHE_GIB", ":#", "Initial decoded compressed-OBJ cache capacity in GiB." }, + { LNK_CmdSwitch_Rad_CObjCacheShrinkGiB, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_CACHE_SHRINK_GIB", ":#", "Post-type-merge decoded compressed-OBJ cache capacity in GiB." }, + { LNK_CmdSwitch_Rad_CObjCacheFreeze, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_CACHE_FREEZE", "[:NO]", "Retain the initial decoded generation when starting the post-boundary cache." }, + { LNK_CmdSwitch_Rad_CObjTrimWs, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_TRIM_WS", "[:NO]", "Trim decoded cache working-set pages at the type-merge boundary." }, + { LNK_CmdSwitch_Rad_CObjOneShot, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_ONE_SHOT", "[:NO]", "Use process-teardown cleanup for this one-shot linker invocation." }, //{ LNK_CmdSwitch_Rad_BuildExp, 0, LNK_CmdValueKind_Scalar, "RAD_BUILD_EXP", "[:NO]", "Build export data." }, { LNK_CmdSwitch_Rad_BuildInfo, 0, LNK_CmdValueKind_Null, "RAD_BUILD_INFO", "", "Print build info and exit." }, { LNK_CmdSwitch_Rad_BuildImpLib, 0, LNK_CmdValueKind_Scalar, "RAD_BUILD_IMPLIB", "[:NO]", "Build import library." }, @@ -104,15 +109,19 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_Rad_WriteTempFiles, 0, LNK_CmdValueKind_Scalar, "RAD_WRITE_TEMP_FILES", "[:NO]", "When speicifed linker writes image and debug info to temporary files and renames after link is done." }, { LNK_CmdSwitch_Rad_TimeStamp, 0, LNK_CmdValueKind_Scalar, "RAD_TIME_STAMP", ":#", "Time stamp embeded in EXE and PDB." }, { LNK_CmdSwitch_Rad_DebugTypeHash, 0, LNK_CmdValueKind_Scalar, "RAD_DEBUG_TYPE_HASH", ":{BLAKE3|XXHASH}", "Sets hashing algorithm for debug type merging." }, + { LNK_CmdSwitch_Rad_DebugTypeHash, 0, LNK_CmdValueKind_Scalar, "RAD_TYPEHASHALG", ":{BLAKE3|XXHASH}", "Alias of RAD_DEBUG_TYPE_HASH (spelling used by UnrealBuildTool)." }, { LNK_CmdSwitch_Rad_UnresolvedSymbolLimit, 0, LNK_CmdValueKind_Scalar, "RAD_UNRESOLVED_SYMBOL_LIMIT", ":#", "Limits number of unresolved symbol errors linker reports." }, { LNK_CmdSwitch_Rad_UnresolvedSymbolRefLimit, 0, LNK_CmdValueKind_Scalar, "RAD_UNRESOLVED_SYMBOL_REF_LIMIT", ":#", "Limit number of unresolved symbol references linker reports." }, { LNK_CmdSwitch_Rad_Version, 0, LNK_CmdValueKind_Null, "RAD_VERSION", "", "Print version and exit." }, { LNK_CmdSwitch_Rad_Workers, 0, LNK_CmdValueKind_Scalar, "RAD_WORKERS", ":#", "Set number of workers created in the pool. Number is capped at 1024. When /RAD_SHARED_THREAD_POOL is specified this number cant exceed /RAD_SHARED_THREAD_POOL_MAX_WORKERS." }, + { LNK_CmdSwitch_Rad_DebugWorkers, 0, LNK_CmdValueKind_Scalar, "RAD_DEBUG_WORKERS", ":#", "Cap concurrent workers in page-fault-bound debug-input stages (parse/prefetch). Default 20; 0 = uncapped. Output is identical either way; the cap only trades idle spinning in the kernel page-fault path for free cores." }, { LNK_CmdSwitch_Rad_WorkDir, 0, LNK_CmdValueKind_Scalar, "RAD_WORK_DIR", ":PATH", "Working directory used for stable debug paths." }, { LNK_CmdSwitch_RadTypeServer, 0, LNK_CmdValueKind_Scalar, "RAD_TYPE_SERVER", ":FILENAME", "Merge types and store them in the specified file. The filename must have the .rrt extension." }, { LNK_CmdSwitch_LLVM_AddrSig, 0, LNK_CmdValueKind_Scalar, "LLVM_ADDRSIG", "[:NO]", "Use .llvm_addrsig to guide ICF." }, + { LNK_CmdSwitch_IfcMap, 1, LNK_CmdValueKind_Scalar, "IFCMAP", ":FILENAME", "Map a header-unit module interface (.ifc) for debug-record resolution (TOML)." }, + { LNK_CmdSwitch_IfcDebugRecords, 0, LNK_CmdValueKind_Scalar, "IFCDEBUGRECORDS", "[:NO]", "Resolve MSVC header-unit IFC debug records into real CodeView types." }, { LNK_CmdSwitch_Help, 0, LNK_CmdValueKind_Null, "HELP", "", "" }, { LNK_CmdSwitch_Help, 0, LNK_CmdValueKind_Null, "?", "", "" }, @@ -1208,7 +1217,17 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val switch (cmd_switch) { case LNK_CmdSwitch_Null: { - lnk_error_obj(LNK_Warning_UnknownSwitch, obj, "unknown switch: \"/%S%s%S\"", cmd_name, value.size ? ":" : "", value); + // Unknown /RAD_* switches on the command line warn and are ignored: the + // RAD_ namespace is owned by this linker, but newer build scripts must + // keep working against older radlink binaries (forward compatibility), + // so an unrecognized /RAD_* switch must not fail the link. Use + // LNK_Warning_Cmdl so the warning stays visible even though the + // release-default /RAD_IGNORE mutes LNK_Warning_UnknownSwitch. + if (obj == 0 && str8_match_lit("RAD_", str8_prefix(cmd_name, 4), StringMatchFlag_CaseInsensitive)) { + lnk_error(LNK_Warning_Cmdl, "unknown switch \"/%S%s%S\"; this radlink build does not support it -- switch ignored", cmd_name, value.size ? ":" : "", value); + } else { + lnk_error_obj(LNK_Warning_UnknownSwitch, obj, "unknown switch: \"/%S%s%S\"", cmd_name, value.size ? ":" : "", value); + } } break; default: break; @@ -1245,6 +1264,9 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val alt_name.to = push_str8_copy(config->arena, alt_name.to); lnk_alt_name_list_push(config->arena, &config->alt_name_list, alt_name); + if (str8_ends_with(alt_name.from, str8_lit("$fo$"), 0)) { + lnk_alt_name_list_push(config->arena, &config->function_override_list, alt_name); + } hash_map_push_string_string(config->arena, &config->alt_name_ht, alt_name.from, alt_name.to); } } @@ -1721,12 +1743,20 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val } } config->opt_icf = LNK_SwitchState_Yes; + } else if (str8_match_lit("icfstatic", param, StringMatchFlag_CaseInsensitive)) { + // compatibility: build systems that drove the fork's /OPT:ICFSTATIC. Internal-linkage + // COMDATs are always fold candidates here, so this is plain ICF. + config->opt_icf = LNK_SwitchState_Yes; } else if (str8_match_lit("noicf", param, StringMatchFlag_CaseInsensitive)) { config->opt_icf = LNK_SwitchState_No; } else if (str8_match_lit("lbr", param, StringMatchFlag_CaseInsensitive)) { config->opt_lbr = LNK_SwitchState_Yes; } else if (str8_match_lit("nolibr", param, StringMatchFlag_CaseInsensitive)) { config->opt_lbr = LNK_SwitchState_No; + } else if (str8_match_lit("gctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_Yes; + } else if (str8_match_lit("nogctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_No; } else { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "unknown option \"%S\"", param); } @@ -1948,6 +1978,32 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val lnk_cmd_switch_parse_u32(obj, cmd_switch, value, &config->age, 0); } break; + case LNK_CmdSwitch_Rad_CObjCacheGiB: { + if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value, &config->cobj_cache_gib, 0) && + config->cobj_cache_gib == 0) { + lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "cache capacity must be greater than zero"); + } + } break; + + case LNK_CmdSwitch_Rad_CObjCacheShrinkGiB: { + if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value, &config->cobj_cache_shrink_gib, 0) && + config->cobj_cache_shrink_gib == 0) { + lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "cache capacity must be greater than zero"); + } + } break; + + case LNK_CmdSwitch_Rad_CObjCacheFreeze: { + lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->cobj_cache_freeze); + } break; + + case LNK_CmdSwitch_Rad_CObjTrimWs: { + lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->cobj_trim_ws); + } break; + + case LNK_CmdSwitch_Rad_CObjOneShot: { + lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->cobj_one_shot); + } break; + //case LNK_CmdSwitch_Rad_BuildExp: { // LNK_SwitchState state; // if (lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &state)) { @@ -2198,7 +2254,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val if (value.size == 0) { config->shared_thread_pool_name = str8_lit(LNK_DEFAULT_THREAD_POOL_NAME); } else { - lnk_cmd_switch_parse_string(obj, cmd_switch, value, &config->shared_thread_pool_name); + // NOTE: must copy into the config arena -- the parsed string points into + // response-file/cmdline scratch that is freed long before late consumers + // (pool init, summary) read it + lnk_cmd_switch_parse_string_copy(config->arena, obj, cmd_switch, value, &config->shared_thread_pool_name); if (config->shared_thread_pool_name.size == 0) { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "invalid empty string for thread pool name"); } @@ -2305,6 +2364,14 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val } } break; + case LNK_CmdSwitch_Rad_DebugWorkers: { + U64 cap; + if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value, &cap, 0)) { + config->debug_worker_cap = cap; + } + } break; + + case LNK_CmdSwitch_Rad_WorkDir: { lnk_cmd_switch_parse_string_copy(config->arena, obj, cmd_switch, value, &config->work_dir); } break; @@ -2332,6 +2399,19 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val case LNK_CmdSwitch_LLVM_AddrSig: { lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->llvm_addrsig); } break; + case LNK_CmdSwitch_IfcMap: { + // collect .toml paths (header-unit -> .ifc); parsed lazily during debug-info build + String8 path = {0}; + if (lnk_cmd_switch_parse_string(obj, cmd_switch, value, &path)) { + str8_list_push(config->arena, &config->ifc_map_list, push_str8_copy(config->arena, path)); + } + } break; + case LNK_CmdSwitch_IfcDebugRecords: { + LNK_SwitchState state = LNK_SwitchState_Null; + if (lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &state)) { + config->ifc_debug_records = state; + } + } break; } scratch_end(scratch); @@ -2941,6 +3021,11 @@ lnk_config_init(U64 argc, char **argv) config->work_dir = get_current_path(arena); config->force = lnk_cmd_line_has_switch(cmd_line, LNK_CmdSwitch_Force); + // fault-bound debug-input stages spin on the kernel page-fault path past ~20 + // concurrent workers (current throughput knee); default cap trades that spin for + // free cores, /RAD_DEBUG_WORKERS:0 restores full width + config->debug_worker_cap = 20; + // apply command line switches for EachNode(cmd, LNK_CmdOption, cmd_line.first_option) { lnk_apply_cmd_option_to_config(config, cmd->string, cmd->value, 0); diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 5ad6bb20c..8ffdd5cde 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -102,6 +102,11 @@ typedef enum LNK_CmdSwitch_Rad_Age, LNK_CmdSwitch_Rad_BootMode, + LNK_CmdSwitch_Rad_CObjCacheGiB, + LNK_CmdSwitch_Rad_CObjCacheShrinkGiB, + LNK_CmdSwitch_Rad_CObjCacheFreeze, + LNK_CmdSwitch_Rad_CObjTrimWs, + LNK_CmdSwitch_Rad_CObjOneShot, LNK_CmdSwitch_Rad_BuildExp, LNK_CmdSwitch_Rad_BuildImpLib, LNK_CmdSwitch_Rad_BuildInfo, @@ -138,6 +143,7 @@ typedef enum LNK_CmdSwitch_Rad_SortImports, LNK_CmdSwitch_Rad_TimeStamp, LNK_CmdSwitch_Rad_DebugTypeHash, + LNK_CmdSwitch_Rad_DebugWorkers, LNK_CmdSwitch_Rad_UnresolvedSymbolLimit, LNK_CmdSwitch_Rad_UnresolvedSymbolRefLimit, LNK_CmdSwitch_Rad_Version, @@ -149,6 +155,8 @@ typedef enum LNK_CmdSwitch_RadTypeServer_MatchObj, LNK_CmdSwitch_LLVM_AddrSig, + LNK_CmdSwitch_IfcMap, + LNK_CmdSwitch_IfcDebugRecords, LNK_CmdSwitch_Help, @@ -324,6 +332,7 @@ typedef struct LNK_Config LNK_SwitchState opt_ref; LNK_SwitchState opt_icf; LNK_SwitchState opt_lbr; + LNK_SwitchState opt_gc_types; // /OPT:GCTYPES -- prune unreferenced CodeView types. Default OFF: shrinks PDB but a pruned type can't be cast-to in the debugger watch window. U64 opt_iter_count; LNK_SwitchState import_table_emit_biat; LNK_SwitchState import_table_emit_uiat; @@ -344,7 +353,13 @@ typedef struct LNK_Config U64 pdb_page_size; U64 worker_count; U64 max_worker_count; + U64 debug_worker_cap; String8 shared_thread_pool_name; + U64 cobj_cache_gib; + U64 cobj_cache_shrink_gib; + LNK_SwitchState cobj_cache_freeze; + LNK_SwitchState cobj_trim_ws; + LNK_SwitchState cobj_one_shot; LNK_SwitchState do_function_pad_min; B32 infer_function_pad_min; U64 function_pad_min; @@ -394,6 +409,7 @@ typedef struct LNK_Config String8 rad_debug_alt_path; LNK_IncludeSymbolList include_symbol_list; LNK_AltNameList alt_name_list; + LNK_AltNameList function_override_list; LNK_MergeDirectiveList merge_list; LNK_SectionDirectiveList section_list; U64 data_dir_count; @@ -425,6 +441,8 @@ typedef struct LNK_Config LNK_SwitchState type_server; LNK_SwitchState sort_imports; LNK_SwitchState llvm_addrsig; + LNK_SwitchState ifc_debug_records; // resolve LF_IFC_RECORD (0x1522) into real CodeView types + String8List ifc_map_list; // .toml paths from /ifcMap (header-unit -> .ifc) } LNK_Config; // --- MSVC Error Codes -------------------------------------------------------- diff --git a/src/linker/lnk_debug_helper.c b/src/linker/lnk_debug_helper.c index fbf9bb360..c7fb5694d 100644 --- a/src/linker/lnk_debug_helper.c +++ b/src/linker/lnk_debug_helper.c @@ -26,7 +26,9 @@ lnk_make_debug_s(Arena *arena, String8List symbols) cv_patch_symbol_tree_offsets(symbols, sizeof(CV_Signature), CV_SymbolAlign); - CV_DebugS debug_s = { .data_list[CV_C13SubSectionIdxKind_Symbols] = symbols }; + // synthetic construction (linker-made symbols): provenance untracked by design + CV_DebugS debug_s = {0}; + *cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols) = symbols; String8List debug_s_data_list = cv_data_from_debug_s_c13(scratch.arena, &debug_s, 1); String8 debug_s_data = str8_list_join(arena, &debug_s_data_list, 0); diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index d18791cbb..26601ddec 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -7,24 +7,289 @@ internal Arena * lnk_get_huge_arena(void) { if (g_huge_arena == 0) { - g_huge_arena = arena_alloc(.name = "HUGE"); + // 2MB commit quantum (vs the 64KB default): this arena backs multi-GB debug + // info merges; the larger quantum cuts VirtualAlloc(MEM_COMMIT) syscalls + // (all serialized on the process address-space lock) ~32x for at most 2MB + // of slack past the high-water mark. + g_huge_arena = arena_alloc(.commit_size = MB(2), .name = "HUGE"); } return g_huge_arena; } +// Handle of the in-flight background arena-release thread (at most one). Joined +// (a) before launching the next reaper and (b) at the end of the link, next to +// the image-write-thread join. NOTE: do NOT thread_detach right after +// thread_launch -- that releases the W32_Entity the new thread's entry point is +// about to read (startup race). +static Thread g_arena_reaper_thread = {0}; + +internal void +lnk_arena_release_thread(void *raw_arena) +{ + // REAPER: releasing a huge arena costs ~50-100ms/GB of committed pages in the + // kernel (MiDeleteVaDirect/MiDecommitFreePage walk every PTE under + // VirtualFree(MEM_RELEASE)), and that work serializes on the process + // address-space lock -- chunking it across the thread pool does NOT make it + // faster (measured: 8 GiB in 860 ms chunked-parallel vs 371 ms serial). So + // instead take it off the critical path entirely: release on a background + // thread while the main thread proceeds. Caller must hand over EXCLUSIVE + // ownership -- no reference to the arena (or memory inside it) may survive + // the thread_launch. + ProfBeginFunction(); + U64 begin_us = now_time_us(); + Arena *arena = raw_arena; + + U64 committed_size = 0; + for (Arena *n = arena->current; n != 0; n = n->prev) { committed_size += n->cmt; } +#if ARENA_FREE_LIST + for (Arena *n = arena->free_last; n != 0; n = n->prev) { committed_size += n->cmt; } +#endif + + arena_release(arena); + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu MiB arena took %.2f ms (off main thread)", + committed_size / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + +// Reaper entry point for a per-worker arena array (TP_Arena). Same ownership rule as +// lnk_arena_release_thread: caller hands over EXCLUSIVE ownership. The TP_Arena header and its +// v[] array live inside v[0] (tp_arena_alloc layout), which tp_arena_release frees last, so the +// walk below and the release order are safe. +internal void +lnk_tp_arena_release_thread(void *raw_arena) +{ + ProfBeginFunction(); + U64 begin_us = now_time_us(); + TP_Arena *tp_arena = raw_arena; + + U64 committed_size = 0; + for EachIndex(i, tp_arena->count) { + for (Arena *n = tp_arena->v[i]->current; n != 0; n = n->prev) { committed_size += n->cmt; } +#if ARENA_FREE_LIST + for (Arena *n = tp_arena->v[i]->free_last; n != 0; n = n->prev) { committed_size += n->cmt; } +#endif + } + + tp_arena_release(&tp_arena); + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu MiB worker arenas took %.2f ms (off main thread)", + committed_size / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + +//////////////////////////////////////////////////////////////////////////////// +//~ Fault-storm mitigation: batched PrefetchVirtualMemory over mapped input +// ranges. The .debug$S/$T parse and type-merge loops first-touch tens of GB of +// memory-mapped obj sections one 4K page fault at a time; with ~100+ links in +// flight on a build farm those per-page traps saturate the kernel machine-wide +// (prod, 126 concurrent links: dbg phase 682s kernel vs 303s user, 42M faults +// in mcvi alone). PrefetchVirtualMemory populates the ranges in bulk (large MM +// batches, no per-page trap), so issue it over each phase's input ranges right +// before the parse walk. Purely a paging hint: no output byte depends on it, +// and failure is silently ignored (pre-Win8 OS / memory pressure). A lone +// link skips the hint because walking its already-cached 80+ GiB input set is +// measurable overhead; concurrent shared-pool links retain it to reduce the +// machine-wide fault storm. + +#if OS_WINDOWS +// declared locally so we do not depend on the SDK's _WIN32_WINNT gate for +// WIN32_MEMORY_RANGE_ENTRY; layout matches memoryapi.h exactly +typedef struct LNK_Win32MemoryRangeEntry +{ + void *VirtualAddress; + SIZE_T NumberOfBytes; +} LNK_Win32MemoryRangeEntry; +typedef BOOL LNK_Win32PrefetchVirtualMemoryFunc(HANDLE process, ULONG_PTR count, LNK_Win32MemoryRangeEntry *ranges, ULONG flags); // WINAPI omitted: x64-only convention + +// entries per task: the kernel's per-page population work dominates the +// syscall overhead, so small batches fanned out over the pool parallelize the +// MM work (14 GiB of mcvi input: ~0.9 s serial -> a wide parallel burst) +#define LNK_PREFETCH_BATCH_SIZE 256 + +typedef struct +{ + LNK_Win32PrefetchVirtualMemoryFunc *proc; + U64 entry_count; + LNK_Win32MemoryRangeEntry *entries; +} LNK_PrefetchTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_prefetch_task) +{ + LNK_PrefetchTask *task = raw_task; + U64 lo = task_id * LNK_PREFETCH_BATCH_SIZE; + U64 hi = Min(lo + LNK_PREFETCH_BATCH_SIZE, task->entry_count); + if (lo < hi) { + task->proc(GetCurrentProcess(), (ULONG_PTR)(hi - lo), task->entries + lo, 0); + } +} +#endif + +// Run a per-item parallel-for on at most `cap` workers. The debug-input stages +// this wraps are page-fault-bound: the kernel working-set-insert path tops out +// near ~3M pages/s regardless of thread count, so lanes past ~20 only convert +// free cores into spin inside the fault handler (the knee moved up after giant-input +// jobification; 24+ lanes still regress wall and sharply increase kernel CPU). Items +// are pulled from a shared cursor, so per-item outputs land in the same +// item-indexed slots as the uncapped path -- output is byte-identical. +typedef struct +{ + TP_TaskFunc *func; + void *data; + U64 item_count; + U64 cursor; +} LNK_CappedForTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_capped_for_task) +{ + LNK_CappedForTask *wrap = raw_task; + for (;;) { + U64 item_idx = ins_atomic_u64_inc_eval(&wrap->cursor) - 1; + if (item_idx >= wrap->item_count) { break; } + wrap->func(arena, worker_id, item_idx, wrap->data, tp); + } +} + +internal void +lnk_tp_for_parallel_capped(TP_Context *tp, TP_Arena *task_arena, U64 cap, U64 item_count, TP_TaskFunc *func, void *data) +{ + if (cap == 0 || cap >= item_count) { + tp_for_parallel(tp, task_arena, item_count, func, data); + } else { + LNK_CappedForTask wrap = { .func = func, .data = data, .item_count = item_count }; + tp_for_parallel(tp, task_arena, cap, lnk_capped_for_task, &wrap); + } +} + +#define lnk_tp_for_parallel_capped_prof(pool, arena, cap, item_count, task_func, task_data, zone_name) ProfBegin(zone_name); lnk_tp_for_parallel_capped(pool, arena, cap, item_count, task_func, task_data); ProfEnd(); + +internal void +lnk_prefetch_ranges(TP_Context *tp, U64 worker_cap, U64 range_count, Rng1U64 *ranges) +{ +#if OS_WINDOWS + // resolve once (Win8+; on older OS fall through silently). Only called from + // serial phase-setup code, so the local_persist init has no race. + local_persist LNK_Win32PrefetchVirtualMemoryFunc *prefetch_proc = 0; + local_persist B32 prefetch_proc_resolved = 0; + if (!prefetch_proc_resolved) { + prefetch_proc_resolved = 1; + HMODULE kernel32 = GetModuleHandleW(L"kernel32.dll"); + if (kernel32 != 0) { + prefetch_proc = (LNK_Win32PrefetchVirtualMemoryFunc *)GetProcAddress(kernel32, "PrefetchVirtualMemory"); + } + } + if (prefetch_proc == 0 || range_count == 0) { return; } + + Temp scratch = scratch_begin(0,0); + + // Coalesce page-aligned neighbors with a single linear pass: ranges arrive + // obj-by-obj in file-offset order, so adjacent sections of the same mapped + // obj (the common case by far) fold into one entry. No sort -- the API does + // not require ordered or disjoint ranges, overlap just costs a cheap re-walk. + LNK_Win32MemoryRangeEntry *entries = push_array_no_zero(scratch.arena, LNK_Win32MemoryRangeEntry, range_count); + U64 entry_count = 0; + U64 pending_min = 0, pending_max = 0; + for EachIndex(range_idx, range_count) { + if (ranges[range_idx].min >= ranges[range_idx].max) { continue; } + U64 min = AlignDownPow2(ranges[range_idx].min, KB(4)); + U64 max = AlignPow2 (ranges[range_idx].max, KB(4)); + if (pending_max != 0 && min <= pending_max && max >= pending_min) { + pending_min = Min(pending_min, min); + pending_max = Max(pending_max, max); + continue; + } + if (pending_max != 0) { + entries[entry_count].VirtualAddress = (void *)pending_min; + entries[entry_count].NumberOfBytes = (SIZE_T)(pending_max - pending_min); + entry_count += 1; + } + pending_min = min; + pending_max = max; + } + if (pending_max != 0) { + entries[entry_count].VirtualAddress = (void *)pending_min; + entries[entry_count].NumberOfBytes = (SIZE_T)(pending_max - pending_min); + entry_count += 1; + } + + // fan the batches out over the pool: population is per-page kernel work, so + // this turns a serial ~1 s stall into a wide parallel burst. Purely advisory + // syscalls with no output -- any batch interleaving is fine. + LNK_PrefetchTask task = { .proc = prefetch_proc, .entry_count = entry_count, .entries = entries }; + U64 batch_count = CeilIntegerDiv(entry_count, LNK_PREFETCH_BATCH_SIZE); + if (tp != 0 && batch_count > 1) { + lnk_tp_for_parallel_capped(tp, 0, worker_cap, batch_count, lnk_prefetch_task, &task); + } else { + for EachIndex(batch_idx, batch_count) { lnk_prefetch_task(0, 0, batch_idx, &task, 0); } + } + + scratch_end(scratch); +#endif +} + +// PrefetchVirtualMemory was added to reduce the machine-wide page-fault storm +// when many shared-pool linker processes run concurrently. For a lone link it +// only populates pages that the parsing walks immediately touch again, adding a +// full extra pass over tens of GiB. The shared-pool process counter is advisory, +// which is exactly the precision this paging hint needs. +internal B32 +lnk_should_prefetch_mapped_input(void) +{ + U32 attached_process_count = 0; + U32 max_process_count = 0; + tp_procs_snapshot(&attached_process_count, &max_process_count); + return attached_process_count > 1; +} + internal void lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) { // discard types MemoryZeroStruct(&input->debug_t_arr[obj_idx]); - // discard symbols + // discard symbols (provenance zeroed in lockstep with the data list) String8List *symbols_ptr = cv_sub_section_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); MemoryZeroStruct(symbols_ptr); + MemoryZeroStruct(cv_sub_section_prov_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols)); // discard inline sites String8List *inlineelines_ptr = cv_sub_section_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); MemoryZeroStruct(inlineelines_ptr); + MemoryZeroStruct(cv_sub_section_prov_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines)); +} + +// Build the same lightweight C13 lists/provenance as cv_debug_s_from_data without walking the +// compressed logical section. Payload pointers are intentionally not dereferenced here; later +// consumers retain the exact original virtual addresses and fault/decode only if they truly need +// record bytes. +internal B32 +lnk_debug_s_from_compressed_index(Arena *arena, LNK_Obj *obj, U32 sect_idx, CV_DebugS *out) +{ + if (!obj->compressed_obj) { return 0; } + LNK_ObjSection section = lnk_obj_section_from_section_number(obj, sect_idx + 1); + LNK_CObjDebugSView view = {0}; + if (!lnk_compressed_obj_debug_s_index(obj->compressed_obj, section.frange, &view)) { return 0; } + for EachIndex(i, view.count) { + LNK_CObjDebugSEntry *entry = &view.v[i]; + U64 payload_min = entry->raw_payload_offset; + U64 payload_max = payload_min + entry->raw_payload_size; + if (payload_min < section.frange.min || payload_max > section.frange.max) { return 0; } + U64 idx = cv_c13_sub_section_idx_from_kind(entry->kind); + str8_list_push(arena, &out->data_list[idx], str8(obj->coff.data.str + payload_min, entry->raw_payload_size)); + cv_debug_s_prov_list_push(arena, &out->prov_list[idx], payload_min - section.frange.min, + entry->raw_payload_size, sect_idx, 0); + if (view.summaries && entry->kind == CV_C13SubSectionKind_Symbols) { + LNK_CObjDebugSSummary *summary = &view.summaries[i]; + CV_DebugSProvNode *prov = out->prov_list[idx].last; + prov->module_symbol_size = summary->module_symbol_size; + prov->gsi_candidate_count = summary->gsi_candidate_count; + prov->proc_ref_count = summary->proc_ref_count; + prov->symbol_summary_valid = 1; + } + } + return 1; } internal @@ -36,9 +301,18 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) String8List sect_list = task->debug_s_list_arr[obj_idx]; CV_DebugS *debug_s = &task->debug_s_arr [obj_idx]; - for EachNode(n, String8Node, sect_list.first) { + U32Array sect_indices = task->debug_s_sect_idx_arr[obj_idx]; + Assert(sect_indices.count == sect_list.node_count); + + U64 input_ordinal = 0; + for (String8Node *n = sect_list.first; n != 0; n = n->next, input_ordinal += 1) { // parse & merge sub sections - CV_DebugS ds = cv_debug_s_from_data(arena, n->string); + U32 sect_idx = sect_indices.v[input_ordinal]; + CV_DebugS ds = {0}; + if (!lnk_debug_s_from_compressed_index(arena, task->obj_arr[obj_idx], sect_idx, &ds)) { + ds = cv_debug_s_from_data(arena, n->string); + cv_debug_s_tag_prov_sect(&ds, sect_idx); + } cv_debug_s_concat_in_place(debug_s, &ds); // make sure there is one string table @@ -55,6 +329,35 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) lnk_error_obj(LNK_Warning_IllData, task->obj_arr[obj_idx], ".debug$S has %u file checksum sub-sections defined, picking first sub-section", checksum_data_list.node_count); } } + + // ICF-folded functions' associated .debug$S (dead-stripped, excluded from the list above): + // merge ONLY their Lines subsections. The reloc patcher patched them to the fold leader's RVA, + // so source breakpoints on folded bodies bind; symbol records stay dropped (that is the bulk + // of link.exe's size cost for the same feature). File ids in these Lines index the obj-wide + // FILECHKSMS merged above, so they stay consistent within this module. Mark 2 (fold joins a + // different source location and has locals) keeps the WHOLE record tree instead, so the watch + // window labels a folded frame with that source's own variable names. + { + LNK_Obj *obj = task->obj_arr[obj_idx]; + if (obj->icf_lines_only != 0) { + for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { + if (!obj->icf_lines_only[section_number]) { continue; } + CV_DebugS ds = {0}; + if (!lnk_debug_s_from_compressed_index(arena, obj, section_number - 1, &ds)) { + String8 raw_data = lnk_obj_section_data_from_number(obj, section_number); + ds = cv_debug_s_from_data(arena, raw_data); + cv_debug_s_tag_prov_sect(&ds, section_number - 1); // merged from a different child section + } + if (obj->icf_lines_only[section_number] == 2) { + cv_debug_s_concat_in_place(debug_s, &ds); + } else { + cv_debug_s_concat_sub_section_in_place(debug_s, &ds, CV_C13SubSectionKind_Lines); + } + } + } + } + + cv_debug_s_validate_prov(debug_s); } internal int @@ -161,7 +464,9 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_task) { ProfBeginFunction(); LNK_ParseCvTypes *task = raw_task; - if (task->raw_types[task_id].count > 0) { + if (task->out_types[task_id].offsets != 0 || task->out_types[task_id].sidecar_packed) { + // Container sidecar or giant obj already supplied the index. + } else if (task->raw_types[task_id].count > 0) { task->out_types[task_id] = cv_debug_t_from_data(arena, task->raw_types[task_id].v[0], CV_LeafAlign); } else { MemoryZeroStruct(&task->out_types[task_id]); @@ -169,6 +474,282 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_task) ProfEnd(); } +typedef struct LNK_BackgroundPrefetch +{ + U64 range_count; + Rng1U64 *ranges; +} LNK_BackgroundPrefetch; + +internal void +lnk_background_prefetch_thread(void *raw_task) +{ + LNK_BackgroundPrefetch *task = raw_task; + lnk_prefetch_ranges(0, 0, task->range_count, task->ranges); +} + +#define LNK_WINNER_PREFETCH_WORKER_MAX 32 +typedef struct LNK_WinnerPrefetchFlight +{ + Thread threads[LNK_WINNER_PREFETCH_WORKER_MAX]; + LNK_BackgroundPrefetch tasks[LNK_WINNER_PREFETCH_WORKER_MAX]; + U32 worker_count; + U64 range_count; + U64 stored_bytes; + U64 begin_us; +} LNK_WinnerPrefetchFlight; + +internal int +lnk_prefetch_range_compare(const void *a, const void *b) +{ + Rng1U64 *ra = (Rng1U64 *)a; + Rng1U64 *rb = (Rng1U64 *)b; + return ra->min < rb->min ? -1 : ra->min > rb->min; +} + +internal void +lnk_winner_prefetch_start(LNK_WinnerPrefetchFlight *flight, U32 worker_count, + U64 range_count, Rng1U64 *ranges, U64 stored_bytes) +{ + if (range_count == 0) { return; } + qsort(ranges, range_count, sizeof(*ranges), lnk_prefetch_range_compare); + flight->worker_count = Min(worker_count, range_count); + flight->range_count = range_count; + flight->stored_bytes = stored_bytes; + flight->begin_us = now_time_us(); + for EachIndex(worker_idx, flight->worker_count) { + U64 min = range_count * worker_idx / flight->worker_count; + U64 max = range_count * (worker_idx + 1) / flight->worker_count; + flight->tasks[worker_idx] = (LNK_BackgroundPrefetch){max - min, ranges + min}; + flight->threads[worker_idx] = thread_launch(lnk_background_prefetch_thread, + &flight->tasks[worker_idx]); + } +} + +internal void +lnk_winner_prefetch_join(LNK_WinnerPrefetchFlight *flight) +{ + for EachIndex(worker_idx, flight->worker_count) { + if (flight->threads[worker_idx].u64[0]) { + thread_join(flight->threads[worker_idx], max_U64); + } + } + if (flight->worker_count) { + lnk_log(LNK_Log_Timers, "[cobj prefetch] workers=%u ranges=%llu stored=%llu MiB elapsed=%.3fs", + flight->worker_count, flight->range_count, flight->stored_bytes / MB(1), + (F64)(now_time_us() - flight->begin_us) / 1e6); + } +} + +// Install the mapped leaf index carried by a compressed-object container. +// This replaces the full leaf-body chain walk with a compact kind scan; the +// body bytes remain behind the bounded decoder until a real consumer asks for +// a leaf. +internal +THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_sidecar_task) +{ + LNK_ParseCvTypes *task = raw_task; + U64 obj_idx = task_id; + if (task->raw_types[obj_idx].count == 0) { return; } + LNK_Obj *obj = task->input->obj_arr[obj_idx]; + U32 section_number = task->is_debug_p ? obj->coff.debug_p_section_number : obj->coff.debug_t_section_number; + if (!obj->compressed_obj || section_number == 0 || section_number > obj->coff.sections.count_no_null) { return; } + LNK_ObjSection sect = lnk_obj_section_from_section_number(obj, section_number); + Rng1U64 leaf_range = rng_1u64(sect.frange.min + sizeof(CV_Signature), sect.frange.max); + LNK_CObjTypeIndexView view = {0}; + if (!lnk_compressed_obj_type_index(obj->compressed_obj, leaf_range, &view)) { return; } + + CV_DebugT *out = &task->out_types[obj_idx]; + out->data = task->raw_types[obj_idx].v[0]; + out->count = view.count; + out->offsets = view.offsets; + out->sidecar_sizes = view.sizes; + out->sidecar_kinds = view.kinds; + out->sidecar_packed_v2_offset_groups = view.packed_v2_offset_groups; + out->sidecar_packed_v2_offset_payload = view.packed_v2_offset_payload; + out->sidecar_packed_kind_dictionary = view.packed_kind_dictionary; + out->sidecar_packed_kind_codes = view.packed_kind_codes; + out->sidecar_packed = view.packed_sidecar; + out->sidecar_offset_checkpoint_shift = view.offset_checkpoint_shift; + out->sidecar_raw_base = leaf_range.min; + out->sidecar_complete_udt_hashes = view.complete_udt_hashes; + out->sidecar_complete_udt_hash_count = view.complete_udt_hash_count; + for EachIndex(i, view.count) { + out->source_counts[cv_type_index_source_from_leaf_kind(cv_debug_t_get_leaf_kind(out, i))] += 1; + } + for EachElement(i, out->ti_ranges) { out->ti_ranges[i] = r1u64(CV_MinComplexTypeIndex, CV_MinComplexTypeIndex + out->count); } + if (out->count && cv_debug_t_get_leaf_kind(out, 0) == CV_LeafKind_PRECOMP) { + CV_PrecompInfo precomp_info = cv_precomp_info_from_leaf(cv_debug_t_get_leaf(out, 0)); + for EachElement(i, out->ti_ranges) { out->ti_ranges[i].max += precomp_info.leaf_count; } + } +} + +// Giant .debug$T parse: a single SharedPCH-scale .debug$T is a multi-second +// SERIAL pointer-chase (each leaf's offset depends on the previous leaf's +// size), and a handful of such objs bound the whole capped parse stage +// (measured: one 6.5 s obj on the FN editor DLL while the rest of the pool +// sat parked). Speculative mid-stream resynchronization is unsound for +// CodeView (arbitrary payload bytes chain "validly", so a wrong guess is not +// locally detectable), so instead: one cheap serial hop per giant walks the +// true chain recording a checkpoint offset every LNK_GIANT_DEBUG_T_INTERVAL +// leaves, then the full pool re-walks the intervals from those true +// boundaries, storing leaf offsets and classifying kinds. Offsets come from +// the true chain and per-source counts are reduced in interval order, so the +// result is bit-identical to the serial parse. +#define LNK_GIANT_DEBUG_T_SIZE MB(16) +#define LNK_GIANT_DEBUG_T_INTERVAL (64*1024) + +typedef U64 LNK_GiantSourceCounts[CV_TypeIndexSource_COUNT]; + +typedef struct +{ + U64 obj_idx; + String8 data; + U64 leaf_count; + U64 interval_count; + U32 *checkpoints; // [interval_count] leaf offset at each interval start + LNK_GiantSourceCounts *interval_counts; +} LNK_GiantDebugT; + +typedef struct +{ + LNK_ParseCvTypes *parse; + LNK_GiantDebugT *giants; + U32 *interval_giant; // flat interval index -> giant index + U32 *interval_local; // flat interval index -> interval within giant +} LNK_GiantDebugTTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_giant_debug_t_hop_task) +{ + ProfBeginFunction(); + LNK_GiantDebugTTask *task = raw_task; + LNK_GiantDebugT *g = &task->giants[task_id]; + String8 data = g->data; + + // bare chain walk; bounds replicate cv_read_leaf exactly (incl. its + // total-size quirks) so the leaf set matches the serial parse bit for bit. + // CV_LeafAlign == 1 => stride is sizeof(CV_LeafHeader) + (size - sizeof(CV_LeafKind)). + U64 checkpoint_cap = data.size / (sizeof(CV_LeafHeader) * LNK_GIANT_DEBUG_T_INTERVAL) + 2; + g->checkpoints = push_array_no_zero(arena, U32, checkpoint_cap); + + U64 leaf_count = 0; + if (data.size >= sizeof(CV_LeafHeader)) { + for (U64 cursor = 0; cursor < data.size; ) { + CV_LeafHeader header = { .v = memory_read32(data.str + cursor) }; + if (header.size < sizeof(CV_LeafKind)) { break; } + if (sizeof(CV_LeafSize) + (U64)header.size > data.size) { break; } + U64 stride = AlignPow2(sizeof(CV_LeafHeader) + (U64)(header.size - sizeof(CV_LeafKind)), CV_LeafAlign); + if (stride > data.size) { break; } + if ((leaf_count % LNK_GIANT_DEBUG_T_INTERVAL) == 0) { + Assert(leaf_count / LNK_GIANT_DEBUG_T_INTERVAL < checkpoint_cap); + g->checkpoints[leaf_count / LNK_GIANT_DEBUG_T_INTERVAL] = (U32)cursor; + } + leaf_count += 1; + cursor += stride; + } + } + + g->leaf_count = leaf_count; + g->interval_count = CeilIntegerDiv(leaf_count, LNK_GIANT_DEBUG_T_INTERVAL); + g->interval_counts = push_array(arena, LNK_GiantSourceCounts, g->interval_count); + + CV_DebugT *out = &task->parse->out_types[g->obj_idx]; + MemoryZeroStruct(out); + out->data = data; + out->count = leaf_count; + out->offsets = push_array_no_zero(arena, U32, leaf_count); + ProfEnd(); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_giant_debug_t_interval_task) +{ + ProfBeginFunction(); + LNK_GiantDebugTTask *task = raw_task; + LNK_GiantDebugT *g = &task->giants[task->interval_giant[task_id]]; + U64 local_idx = task->interval_local[task_id]; + String8 data = g->data; + CV_DebugT *out = &task->parse->out_types[g->obj_idx]; + + U64 leaf_lo = local_idx * LNK_GIANT_DEBUG_T_INTERVAL; + U64 leaf_hi = Min(leaf_lo + LNK_GIANT_DEBUG_T_INTERVAL, g->leaf_count); + U64 cursor = g->checkpoints[local_idx]; + U64 *counts = g->interval_counts[local_idx]; + + for (U64 leaf_idx = leaf_lo; leaf_idx < leaf_hi; leaf_idx += 1) { + CV_LeafHeader header = { .v = memory_read32(data.str + cursor) }; + out->offsets[leaf_idx] = (U32)cursor; + counts[cv_type_index_source_from_leaf_kind(header.kind)] += 1; + cursor += AlignPow2(sizeof(CV_LeafHeader) + (U64)(header.size - sizeof(CV_LeafKind)), CV_LeafAlign); + } + ProfEnd(); +} + +internal void +lnk_parse_giant_debug_t(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 obj_count, LNK_ParseCvTypes *parse) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(0,0); + + // gather giants + U64 giant_count = 0; + LNK_GiantDebugT *giants = push_array(scratch.arena, LNK_GiantDebugT, obj_count); + for EachIndex(obj_idx, obj_count) { + if (parse->out_types[obj_idx].offsets == 0 && !parse->out_types[obj_idx].sidecar_packed && + parse->raw_types[obj_idx].count > 0 && parse->raw_types[obj_idx].v[0].size >= LNK_GIANT_DEBUG_T_SIZE) { + giants[giant_count].obj_idx = obj_idx; + giants[giant_count].data = parse->raw_types[obj_idx].v[0]; + giant_count += 1; + } + } + + if (giant_count > 0) { + LNK_GiantDebugTTask task = { .parse = parse, .giants = giants }; + + // phase 1: serial chain hop per giant (capped: the hop is the fault-bound + // dependent-load chase, extra lanes only spin) + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, giant_count, lnk_giant_debug_t_hop_task, &task, "Giant .debug$T Hop"); + + // flatten intervals + U64 interval_total = 0; + for EachIndex(i, giant_count) { interval_total += giants[i].interval_count; } + task.interval_giant = push_array_no_zero(scratch.arena, U32, interval_total); + task.interval_local = push_array_no_zero(scratch.arena, U32, interval_total); + for (U64 i = 0, flat = 0; i < giant_count; i += 1) { + for EachIndex(k, giants[i].interval_count) { + task.interval_giant[flat] = (U32)i; + task.interval_local[flat] = (U32)k; + flat += 1; + } + } + + // phase 2: offsets + kind classification from true checkpoints; pages are + // hot from the hop, so this is CPU-bound -- full pool width + tp_for_parallel_prof(tp, 0, interval_total, lnk_giant_debug_t_interval_task, &task, "Giant .debug$T Intervals"); + + // phase 3: deterministic interval-order reduce + the serial-parse tail + for EachIndex(i, giant_count) { + LNK_GiantDebugT *g = &giants[i]; + CV_DebugT *out = &parse->out_types[g->obj_idx]; + for EachIndex(k, g->interval_count) { + for EachElement(s, out->source_counts) { out->source_counts[s] += g->interval_counts[k][s]; } + } +#if BUILD_DEBUG + { U64 total = 0; for EachElement(s, out->source_counts) { total += out->source_counts[s]; } Assert(total == g->leaf_count); } +#endif + for EachElement(s, out->ti_ranges) { out->ti_ranges[s] = r1u64(CV_MinComplexTypeIndex, CV_MinComplexTypeIndex + out->count); } + CV_Leaf leaf = cv_debug_t_get_leaf(out, 0); + if (leaf.kind == CV_LeafKind_PRECOMP) { + CV_PrecompInfo precomp_info = cv_precomp_info_from_leaf(leaf); + for EachElement(s, out->ti_ranges) { out->ti_ranges[s].max += precomp_info.leaf_count; } + } + } + } + + scratch_end(scratch); + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_read_type_servers_task) { @@ -656,99 +1237,959 @@ lnk_rrt_array_from_config(Arena *arena, LNK_Config *config) return rrt_arr; } -internal LNK_CodeViewInput -lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 obj_count, LNK_Obj **obj_arr, LNK_RRT_Array rrt_input) -{ - ProfBegin("Extract CodeView"); - Temp scratch = scratch_begin(0,0); - - LNK_CodeViewInput input = { .config = config, .obj_count = obj_count, .count = obj_count, .obj_arr = obj_arr, .rrt_input = rrt_input, .ts_obj_range = r1u64(0,0) }; +//////////////////////////////// +// IFC header-unit debug-record resolution - HashMap rrt_hm = {0}; - ProfScope("Make obj path -> RRT hash map") - { - for EachIndex(rrt_idx, rrt_input.count) { - for EachIndex(obj_idx, rrt_input.v[rrt_idx].obj_paths.count) { - hash_map_push_path_u64(scratch.arena, &rrt_hm, rrt_input.v[rrt_idx].obj_paths.v[obj_idx], Compose64Bit(rrt_idx, obj_idx)); +typedef struct LNK_IfcMapEntry +{ + String8 ifc_path; // absolute .ifc path + U64 blob_slot; // resolved blob slot + 1; 0 = not yet resolved (filled by the serial + // discovery replay in lnk_apply_ifc_debug_records, memoizes path lookups) +} LNK_IfcMapEntry; + +// Parse the trivial /ifcMap TOML by hand: +// [[header-unit]] +// name = ["quote", ''] +// ifc = "" +// Registers basename(header-unit-path) -> .ifc path in `hm` (hash_map of path->raw LNK_IfcMapEntry*). +internal void +lnk_parse_ifc_map_toml(Arena *arena, HashMap *hm, String8 toml_data) +{ + U64 cursor = 0; + String8 cur_name = {0}; + while (cursor < toml_data.size) { + // read a line + U64 line_end = cursor; + while (line_end < toml_data.size && toml_data.str[line_end] != '\n') { line_end += 1; } + String8 line = str8_skip_chop_whitespace(str8_substr(toml_data, r1u64(cursor, line_end))); + cursor = line_end + 1; + + if (line.size == 0 || line.str[0] == '#') { continue; } + + if (str8_match(str8_prefix(line, 4), str8_lit("name"), 0)) { + // name = ["quote", ''] -- extract the last single-quoted token + U64 q0 = str8_find_needle(line, 0, str8_lit("'"), 0); + if (q0 < line.size) { + U64 q1 = str8_find_needle(line, q0 + 1, str8_lit("'"), 0); + if (q1 < line.size) { + cur_name = str8_substr(line, r1u64(q0 + 1, q1)); + } + } + } else if (str8_match(str8_prefix(line, 3), str8_lit("ifc"), 0)) { + U64 q0 = str8_find_needle(line, 0, str8_lit("\""), 0); + if (q0 < line.size && cur_name.size) { + U64 q1 = str8_find_needle(line, q0 + 1, str8_lit("\""), 0); + if (q1 < line.size) { + String8 ifc_path = str8_substr(line, r1u64(q0 + 1, q1)); + // key by basename of the header-unit path (matches LF_IFC_RECORD header_unit_path basename) + String8 base = str8_skip_last_slash(cur_name); + // header-unit paths use backslashes; normalize to last path component + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + LNK_IfcMapEntry *e = push_array(arena, LNK_IfcMapEntry, 1); + e->ifc_path = push_str8_copy(arena, ifc_path); + hash_map_push_string_raw(arena, hm, push_str8_copy(arena, base), e); + } } + cur_name = str8_zero(); } } +} - ProfBegin("Apply RRT to Objs"); - - // hash map (obj path, obj idx) - HashMap obj_path_hm = {0}; - for EachIndex(obj_idx, obj_count) { - hash_map_push_path_u64(scratch.arena, &obj_path_hm, obj_arr[obj_idx]->path, obj_idx); +// Reads every /ifcMap toml, materializes the union of header-unit basename -> .ifc path. +internal HashMap +lnk_build_ifc_map(Arena *arena, LNK_Config *config) +{ + HashMap hm = {0}; + Temp scratch = scratch_begin(&arena, 1); + for EachNode(n, String8Node, config->ifc_map_list.first) { + B8 was_read = 0; + String8 toml = lnk_read_data_from_file_path(scratch.arena, 0, n->string, &was_read); + if ( ! was_read || toml.size == 0) { + lnk_error(LNK_Error_Cmdl, "/ifcMap: unable to read TOML '%S'", n->string); + continue; + } + lnk_parse_ifc_map_toml(arena, &hm, toml); } + scratch_end(scratch); + return hm; +} - for EachIndex(obj_idx, obj_count) { - LNK_Obj *obj = obj_arr[obj_idx]; - U64 *packed_rrt_idx = hash_map_search_path_u64(&rrt_hm, obj->path); - - // obj is not part of any input RRT - if (packed_rrt_idx == 0) { continue; } +// LF_IFC_RECORD (0x1522) body layout (header {len,kind} already stripped from leaf.data): +// u16 version (==2); u32 ifc_type_index X; u8[16] guid; u8[16] hash; char[] header_unit_path NUL +typedef struct LNK_IfcRecord +{ + U32 ifc_type_index; // X: TI into the .ifc debug-records blob (base 0x1000) + U8 guid[16]; + U8 hash[16]; + String8 header_unit_path; + B32 is_valid; +} LNK_IfcRecord; + +internal LNK_IfcRecord +lnk_parse_ifc_record(String8 leaf_data) +{ + LNK_IfcRecord rec = {0}; + if (leaf_data.size < 2 + 4 + 16 + 16) { return rec; } + U64 off = 0; + U16 version; off += str8_deserial_read_struct(leaf_data, off, &version); + off += str8_deserial_read_struct(leaf_data, off, &rec.ifc_type_index); + MemoryCopy(rec.guid, leaf_data.str + off, 16); off += 16; + MemoryCopy(rec.hash, leaf_data.str + off, 16); off += 16; + rec.header_unit_path = str8_cstring_capped(leaf_data.str + off, leaf_data.str + leaf_data.size); + rec.is_valid = 1; + return rec; +} - // unpack index - U32 rrt_idx = *packed_rrt_idx >> 32; - U32 rrt_obj_idx = *packed_rrt_idx & max_U32; - LNK_RRT *rrt = &rrt_input.v[rrt_idx]; +// Per-blob closure + NOTYPE-prune (third pass of lnk_apply_ifc_debug_records). Each blob is fully +// independent: it reads/writes only its own ref_bits[blob_i] and its own blob DebugT leaves, so the +// work parallelizes across the ~13 blobs with no shared state. The worklist scratch comes from the +// per-worker arena. Closure counts are written per-blob and summed afterward (order-independent). +// Open-addressing U64 hash set keyed by unique_name hash. Used to record which UDT unique_names +// already have a COMPLETE definition in a non-blob (consuming) obj, so the blob prune can keep a +// blob's complete definition only for names that NO normal obj completes (blob-only types). cap is +// a power of two; 0-hash is reserved as the empty sentinel (we OR in a bit so a real 0 can't occur). +typedef struct LNK_U64Set { U64 *slots; U64 cap; } LNK_U64Set; - // obj was recompiled, do not apply RRT indirection - FileProperties obj_file_props = properties_from_file_path(obj->path); - if (rrt->obj_time_stamps[rrt_obj_idx] != obj_file_props.modified) { continue; } +internal U64 +lnk_uname_hash(String8 s) +{ + U64 h = 5381; + for EachIndex(c, s.size) { h = ((h << 5) + h) ^ (U64)s.str[c]; } + return h | 1; // never 0 (0 is the empty sentinel) +} - // invalidate debug section pointers - obj->coff.debug_t_section_number = 0; - obj->coff.debug_p_section_number = 0; - obj->coff.debug_h_section_number = 0; +internal void +lnk_u64set_add(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { set->slots[i] = h; return; } + if (set->slots[i] == h) { return; } + i = (i + 1) & (set->cap - 1); + } +} - // apply type index map - obj->ti_range = rrt->obj_ti_ranges[rrt_obj_idx]; - obj->ti_map = rrt->obj_ti_maps [rrt_obj_idx]; - - // apply PCH info - U32 rrt_pch_obj_idx = rrt->obj_pch_indices[rrt_obj_idx]; - if (rrt_pch_obj_idx < rrt->obj_count) { - String8 rrt_pch_obj_path = rrt->obj_paths.v[rrt_pch_obj_idx]; - U64 pch_obj_idx = *hash_map_search_path_u64(&obj_path_hm, rrt_pch_obj_path); - obj->pch_ti_range = rrt->obj_pch_ti_ranges[rrt_obj_idx]; - obj->pch_obj_idx = pch_obj_idx; - } else { - obj->pch_ti_range = r1u64(0,0); - obj->pch_obj_idx = ~0; +// Thread-safe insert mirroring lnk_icf_map_put_atomic (lnk.c). The empty sentinel is 0 (not +// LNK_ICF_EMPTY); h is guaranteed nonzero by lnk_uname_hash (`| 1`) so a real key can never be 0. +// Claim an empty slot with an atomic CAS 0->h; the CAS winner owns it. On a lost race re-read the +// same slot (it may now hold our h via a duplicate, or another key) before advancing. Duplicate +// keys across objs are legal and idempotent (a slot already holding h returns), so any insertion +// order yields the identical final membership -- the set is read-only (lnk_u64set_has) afterward. +internal void +lnk_u64set_add_atomic(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { + if (ins_atomic_u64_eval_cond_assign(&set->slots[i], h, 0) == 0) { return; } + continue; } + if (set->slots[i] == h) { return; } + i = (i + 1) & (set->cap - 1); } - ProfEnd(); - - ProfBegin("Collect CodeView"); - input.debug_s_list_arr = lnk_collect_obj_sections(tp, tp_arena, obj_count, obj_arr, str8_lit(".debug$S"), 0); - ProfEnd(); +} - // profiler info - if (lnk_get_log_status(LNK_Log_Debug) || PROFILE_TELEMETRY) { - U64 total_debug_s_size = 0, total_debug_t_size = 0, total_debug_p_size = 0, total_debug_h_size = 0; - for EachIndex(obj_idx, obj_count) { - LNK_Obj *obj = obj_arr[obj_idx]; +internal B32 +lnk_u64set_has(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { return 0; } + if (set->slots[i] == h) { return 1; } + i = (i + 1) & (set->cap - 1); + } +} - for EachNode(n, String8Node, input.debug_s_list_arr[obj_idx].first) { total_debug_s_size += n->string.size; } +// Parallel collect of complete-definition unique_name hashes per non-blob obj. Each obj is scanned +// independently (read-only over its .debug$T leaves) and emits its hashes into a per-obj list; a +// serial pass then adds them to the shared open-addressing set. Moves the ~1.25s serial cv_get_udt_info +// scan off the main thread. Determinism: set membership is order-independent, serial-add reproduces. +typedef struct LNK_IfcCompleteScanTask +{ + LNK_CodeViewInput *input; + U64 **out_hashes; // per-obj hash array (allocated by task) + U64 *out_counts; // per-obj count +} LNK_IfcCompleteScanTask; - if (obj->coff.debug_t_section_number > 0) { - total_debug_t_size += lnk_coff_section_header_from_section_number(obj, obj->coff.debug_t_section_number)->fsize; - } - if (obj->coff.debug_p_section_number > 0) { - total_debug_p_size += lnk_coff_section_header_from_section_number(obj, obj->coff.debug_p_section_number)->fsize; - } - if (config->ghash) { - if (obj->coff.debug_h_section_number > 0) { - total_debug_h_size += lnk_coff_section_header_from_section_number(obj, obj->coff.debug_h_section_number)->fsize; - } - } - } +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_complete_scan_task) +{ + LNK_IfcCompleteScanTask *task = raw_task; + U64 obj_idx = task_id; + CV_DebugT *dt = &task->input->debug_t_arr[obj_idx]; + if (dt->sidecar_complete_udt_hashes) { + task->out_hashes[obj_idx] = dt->sidecar_complete_udt_hashes; + task->out_counts[obj_idx] = dt->sidecar_complete_udt_hash_count; + return; + } + U64 *hashes = push_array_no_zero(arena, U64, dt->count ? dt->count : 1); + U64 n = 0; + for EachIndex(leaf_idx, dt->count) { + CV_Leaf leaf = cv_debug_t_get_leaf(dt, leaf_idx); + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (!(ui.props & CV_TypeProp_HasUniqueName) || ui.unique_name.size == 0) { continue; } + if (ui.props & CV_TypeProp_FwdRef) { continue; } + hashes[n++] = lnk_uname_hash(ui.unique_name); + } + task->out_hashes[obj_idx] = hashes; + task->out_counts[obj_idx] = n; +} - U64 total_rrt_type_size = 0; - U64 total_rrt_hash_size = 0; - for EachIndex(rrt_idx, rrt_input.count) { +// Parallel merge of the per-obj complete-def hash lists into the shared set. Replaces the serial +// lnk_u64set_add loop (the ~914ms hotspot -- 801ms of it first-touch KiPageFault on a single thread +// faulting a 128MB+ set). lnk_u64set_add_atomic spreads both the random-scatter probes AND the +// page faults across the pool. Determinism: keys may legitimately duplicate across objs, but the +// insert is idempotent and the set is read-only afterward (lnk_u64set_has), so insertion order +// cannot change the final membership -- output is bit-identical to the serial merge. +typedef struct LNK_IfcSetMergeTask +{ + Rng1U64 *ranges; + U64 **out_hashes; + U64 *out_counts; + LNK_U64Set *set; + U64 nonblob_count; +} LNK_IfcSetMergeTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_set_merge_task) +{ + LNK_IfcSetMergeTask *task = raw_task; + for EachInRange(obj_idx, task->ranges[task_id]) { + U64 *h = task->out_hashes[obj_idx]; + U64 n = task->out_counts[obj_idx]; + for EachIndex(t, n) { lnk_u64set_add_atomic(task->set, h[t]); } + } +} + +// Fused discovery+redirect scan (passes 1+2 of lnk_apply_ifc_debug_records): each consuming +// obj's .debug$T is swept ONCE, in parallel, for 0x1522 (LF_IFC_RECORD) leaves. The worker +// parses each record, resolves its header-unit basename against the read-only ifc_map_hm, and +// emits one raw record per 0x1522 leaf in ascending leaf_idx order. Workers write NOTHING +// (no NOTYPE, no discovery, no redirects): every order-sensitive effect -- .ifc first-encounter +// slot assignment, NOTYPE rewrites, redirect hash-map push order, ref_bits seeding -- is +// replayed SERIALLY from these records in ascending obj_idx, then ascending leaf_idx: the exact +// order of the original serial passes, so output is byte-for-byte identical. +typedef struct LNK_IfcRawRec +{ + U64 leaf_idx; // 0x1522 leaf index inside the consuming obj + CV_TypeIndex K; // consuming obj's local placeholder TI + U32 ifc_type_index; // X: TI into the .ifc blob (base 0x1000) + U8 guid[16]; + U8 hash[16]; + LNK_IfcMapEntry *entry; // basename -> map entry (0: invalid record or no map hit) + U64 blob_i_plus1; // resolved blob slot + 1 (serial replay fills; 0 = unresolved) + U64 blob_leaf_idx; // resolved leaf inside the blob (serial replay fills) +} LNK_IfcRawRec; + +typedef struct LNK_IfcScanTask +{ + LNK_CodeViewInput *input; + HashMap *ifc_map_hm; // read-only in workers + LNK_IfcRawRec **out_recs; // per-obj ordered raw records (allocated by task) + U64 *out_counts; // per-obj record count +} LNK_IfcScanTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_scan_task) +{ + LNK_IfcScanTask *task = raw_task; + U64 obj_idx = task_id; + LNK_CodeViewInput *input = task->input; + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + + // count 0x1522 leaves first to size the per-obj record array + U64 ifc_leaf_count = 0; + for EachIndex(leaf_idx, debug_t->count) { + if (cv_debug_t_get_leaf_kind(debug_t, leaf_idx) == 0x1522) { ifc_leaf_count += 1; } + } + if (ifc_leaf_count == 0) { task->out_recs[obj_idx] = 0; task->out_counts[obj_idx] = 0; return; } + + LNK_IfcRawRec *recs = push_array_no_zero(arena, LNK_IfcRawRec, ifc_leaf_count); + U64 n = 0; + + for EachIndex(leaf_idx, debug_t->count) { + if (cv_debug_t_get_leaf_kind(debug_t, leaf_idx) != 0x1522) { continue; } + + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); + + LNK_IfcRawRec *r = &recs[n++]; + r->leaf_idx = leaf_idx; + r->K = cv_ti_from_leaf_idx(debug_t, CV_TypeIndexSource_TPI, leaf_idx); + r->ifc_type_index = rec.ifc_type_index; + r->entry = 0; + r->blob_i_plus1 = 0; + r->blob_leaf_idx = 0; + + if (rec.is_valid) { + MemoryCopy(r->guid, rec.guid, 16); + MemoryCopy(r->hash, rec.hash, 16); + String8 base = str8_skip_last_slash(rec.header_unit_path); + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + r->entry = hash_map_search_string_raw(task->ifc_map_hm, base); + } + } + + task->out_recs[obj_idx] = recs; + task->out_counts[obj_idx] = n; +} + +// Parallel per-obj record resolution + placeholder NOTYPE (runs after discovery/read/injection, +// when entry->blob_slot, ifc_files, and the injected blob debug_t entries are all frozen/read-only). +// Each worker fills its own obj's raw records in place (blob_i_plus1/blob_leaf_idx), rewrites its +// own 0x1522 leaves to NOTYPE (per-obj disjoint, constant value -- order-free), and reports the +// resolved count + K range. The serial replay below then only pushes redirects in the original +// (obj_idx, leaf_idx) order, so hash-map push order and all outputs stay bit-identical. +typedef struct LNK_IfcResolveTask +{ + LNK_CodeViewInput *input; + IFC_File *ifc_files; + LNK_IfcRawRec **recs; // per-obj raw records from the scan + U64 *counts; // per-obj record count + U64 *res_counts; // out: per-obj resolved record count + U64 *k_first; // out: first resolved K (valid when res_counts != 0) + U64 *k_last; // out: last resolved K (valid when res_counts != 0) +} LNK_IfcResolveTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_resolve_task) +{ + LNK_IfcResolveTask *task = raw_task; + U64 obj_idx = task_id; + LNK_CodeViewInput *input = task->input; + LNK_IfcRawRec *recs = task->recs[obj_idx]; + U64 n = task->counts[obj_idx]; + if (n == 0) { task->res_counts[obj_idx] = 0; return; } + + U64 res_count = 0; + U64 k_first = 0, k_last = 0; + for EachIndex(t, n) { + LNK_IfcRawRec *r = &recs[t]; + // exclude the placeholder leaf from output regardless: rewrite to NOTYPE. P4: journaled + // (KIND_ONLY -- the old write changed only the kind, size/payload stay) instead of + // CoW-dirtying the mapped view; capacity is pre-reserved serially (arena == 0 here) + lnk_notype_journal_push(0, &input->notype_journal[obj_idx], (U32)r->leaf_idx, 1, 0 /* bitmap pre-allocated */); + if (r->entry == 0 || r->entry->blob_slot == 0) { continue; } + U64 blob_i = r->entry->blob_slot - 1; + IFC_File *f = &task->ifc_files[blob_i]; + B32 hash_ok = MemoryMatch(r->guid, f->content_hash, 16) && + MemoryMatch(r->hash, f->content_hash + 16, 16); + if (!f->is_valid || !hash_ok) { continue; } + CV_DebugT *bdt = &input->debug_t_arr[input->ifc_obj_range.min + blob_i]; + U64 blob_leaf_idx = cv_leaf_idx_from_ti(bdt, CV_TypeIndexSource_TPI, r->ifc_type_index); + if (blob_leaf_idx >= bdt->count) { continue; } + r->blob_i_plus1 = blob_i + 1; + r->blob_leaf_idx = blob_leaf_idx; + if (res_count == 0) { k_first = r->K; } + k_last = r->K; + res_count += 1; + } + task->res_counts[obj_idx] = res_count; + task->k_first[obj_idx] = k_first; + task->k_last[obj_idx] = k_last; +} + +// Parallel .ifc read + `.msvc.trait.debug-records` parse into PRE-ASSIGNED slots. Slot order +// (== blob obj order == output order) is fixed by the serial discovery replay before any file +// is read, so going wide here cannot reorder anything. Workers do not call lnk_error: read +// failures are collected per slot and reported serially in slot order afterward (identical +// message order to the old serial read; LNK_Error_Cmdl stops the link either way). Worker-arena +// allocations (file bytes + leaf offsets) are long-lived, same as the parallel .debug$T parse +// (lnk_parse_debug_t_task pattern). +typedef struct LNK_IfcReadTask +{ + String8 *paths; // per-slot .ifc path + IFC_File *ifc_files; // per-slot output + CV_DebugT *blob_debug_t; // per-slot output + String8 *errors; // per-slot read error (size 0 = ok) +} LNK_IfcReadTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_read_task) +{ + LNK_IfcReadTask *task = raw_task; + U64 slot = task_id; + String8 err = {0}; + IFC_File f = ifc_file_read(arena, task->paths[slot], &err); + task->ifc_files[slot] = f; + task->errors[slot] = err; + if (f.is_valid) { + // parse the raw CV leaf stream (no signature, TI base 0x1000) + task->blob_debug_t[slot] = cv_debug_t_from_data(arena, f.debug_records, 1); + } else { + MemoryZeroStruct(&task->blob_debug_t[slot]); + } +} + +typedef struct LNK_IfcCloseTask +{ + LNK_CodeViewInput *input; + U8 **ref_bits; + U64 *closure_leaves; // per-blob output: # leaves surviving in closure + LNK_U64Set *nonblob_complete; // unique_name hashes completed by some non-blob obj +} LNK_IfcCloseTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_close_blob_task) +{ + LNK_IfcCloseTask *task = raw_task; + U64 blob_i = task_id; + LNK_CodeViewInput *input = task->input; + U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; + CV_DebugT *bdt = &input->debug_t_arr[blob_obj_idx]; + U8 *bits = task->ref_bits[blob_i]; + U64 closure = 0; + if (bdt->count == 0) { task->closure_leaves[blob_i] = 0; return; } + + Temp wtemp = temp_begin(arena); + + // Extra closure roots for forward-ref completion: in CodeView a forward-ref UDT is completed by + // ANY same-unique_name complete definition in the PDB. A consuming obj typically emits only a + // forward-ref of a header-unit type; the full-merge build incidentally kept the matching complete + // definition from the .ifc blob, so the debugger could complete it. On-demand would drop that + // definition (nothing references it by TI), leaving the type incomplete vs full-merge. To preserve + // fidelity WITHOUT dragging the whole blob, root every blob complete-def UDT whose unique_name has + // NO complete definition in any non-blob obj (i.e. blob-only types -- trait/delegate marker structs + // etc.). Common types (FString, FGuid, ...) are completed by normal objs, so their redundant blob + // copies stay pruned. Members of the kept defs are pulled by the closure walk below. + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { continue; } // already a root + CV_Leaf leaf = cv_debug_t_get_leaf(bdt, leaf_idx); + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (!(ui.props & CV_TypeProp_HasUniqueName) || ui.unique_name.size == 0) { continue; } + if (ui.props & CV_TypeProp_FwdRef) { continue; } // only complete definitions + U64 h = lnk_uname_hash(ui.unique_name); + if (lnk_u64set_has(task->nonblob_complete, h)) { continue; } // a normal obj already completes it + bits[leaf_idx >> 3] |= (U8)(1u << (leaf_idx & 7)); + } + + U64 *worklist = push_array_no_zero(wtemp.arena, U64, bdt->count); + U64 wl_count = 0; + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { worklist[wl_count++] = leaf_idx; } + } + + while (wl_count) { + U64 leaf_idx = worklist[--wl_count]; + CV_Leaf leaf = cv_debug_t_get_leaf(bdt, leaf_idx); + Temp itemp = temp_begin(wtemp.arena); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(itemp.arena, leaf.kind, leaf.data); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(leaf.data, ti_info.offset, sizeof(*ti_ptr)); + if (ti_ptr == 0) { continue; } + CV_TypeIndex sub_ti = memory_read32(ti_ptr); + if (sub_ti < bdt->ti_ranges[ti_info.source].min || + sub_ti >= bdt->ti_ranges[ti_info.source].max) { continue; } + U64 sub_leaf_idx = cv_leaf_idx_from_ti(bdt, ti_info.source, sub_ti); + if (sub_leaf_idx >= bdt->count) { continue; } + if (bits[sub_leaf_idx >> 3] & (1u << (sub_leaf_idx & 7))) { continue; } + bits[sub_leaf_idx >> 3] |= (U8)(1u << (sub_leaf_idx & 7)); + worklist[wl_count++] = sub_leaf_idx; + } + temp_end(itemp); + } + temp_end(wtemp); + + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { closure += 1; continue; } + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(bdt, leaf_idx); + if (hdr->kind == CV_LeafKind_NOTYPE) { continue; } + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, size), (U16)sizeof(CV_LeafKind)); + } + task->closure_leaves[blob_i] = closure; +} + +// Injects referenced .ifc debug-records blobs as extra "objs" in `input`, scans every +// consuming obj's .debug$T for LF_IFC_RECORD (0x1522) leaves, registers each placeholder +// local TI -> blob leaf redirect, and rewrites the 0x1522 leaf to NOTYPE so it is excluded +// from the output TPI. Must run after .debug$T is parsed and before min-type-index / symbol +// setup (which iterate input->count). +internal void +lnk_apply_ifc_debug_records(TP_Context *tp, TP_Arena *tp_arena, LNK_CodeViewInput *input, LNK_Config *config) +{ + ProfBeginFunction(); + U64 apply_begin_us = now_time_us(); + Temp scratch = scratch_begin(&tp_arena->v[0], 1); + Arena *arena = tp_arena->v[0]; + + // basename -> .ifc path + HashMap ifc_map_hm = lnk_build_ifc_map(scratch.arena, config); + U64 discover_begin_us = now_time_us(); + + // --- fused scan (old passes 1+2, parallel): ONE sweep of every consuming obj's .debug$T + // emits per-obj raw 0x1522 records in ascending leaf_idx order (record parse + basename -> + // ifc_map_hm entry resolution happen in the workers; nothing is written). Every + // order-sensitive effect is replayed serially from these records below. --- + LNK_IfcScanTask scan = {0}; + scan.input = input; + scan.ifc_map_hm = &ifc_map_hm; + scan.out_recs = push_array(scratch.arena, LNK_IfcRawRec *, input->obj_count); + scan.out_counts = push_array(scratch.arena, U64, input->obj_count); + tp_for_parallel(tp, tp_arena, input->obj_count, lnk_ifc_scan_task, &scan); + U64 scan_end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "[IFC] parallel scan in %.2f ms", (F64)(scan_end_us - discover_begin_us) / 1000.0); + + // --- serial discovery replay: assign .ifc blob slots in first-encounter order (ascending + // obj_idx, then ascending leaf_idx -- identical to the old serial pass) WITHOUT reading any + // file, so the reads can go wide below. De-dup by path; entry->blob_slot memoizes the path + // lookup. 256-slot cap semantics preserved: on overflow the entry stays unresolved and its + // records never redirect. --- + HashMap ifc_path_to_blobidx = {0}; // path -> (blob slot index + 1) + IFC_File *ifc_files = push_array(scratch.arena, IFC_File, 256); + U64 ifc_file_count = 0; + CV_DebugT blob_debug_t[256] = {0}; + String8 slot_paths[256] = {0}; + for EachIndex(obj_idx, input->obj_count) { + LNK_IfcRawRec *recs = scan.out_recs[obj_idx]; + U64 n = scan.out_counts[obj_idx]; + for EachIndex(t, n) { + LNK_IfcMapEntry *e = recs[t].entry; + if (e == 0 || e->blob_slot) { continue; } + U64 *slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); + if (slot == 0) { + if (ifc_file_count >= 256) { continue; } + hash_map_push_string_u64(scratch.arena, &ifc_path_to_blobidx, e->ifc_path, ifc_file_count + 1); + slot_paths[ifc_file_count] = e->ifc_path; + ifc_file_count += 1; + slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); + } + e->blob_slot = *slot; + } + } + + if (ifc_file_count == 0) { goto done; } + U64 discover_end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "[IFC] discover replay in %.2f ms", (F64)(discover_end_us - scan_end_us) / 1000.0); + + // --- parallel .ifc read + debug-records parse into the pre-assigned slots; report read + // errors serially in slot order (identical message order to the old serial read). --- + { + LNK_IfcReadTask read = {0}; + read.paths = slot_paths; + read.ifc_files = ifc_files; + read.blob_debug_t = blob_debug_t; + read.errors = push_array(scratch.arena, String8, ifc_file_count); + tp_for_parallel(tp, tp_arena, ifc_file_count, lnk_ifc_read_task, &read); + for EachIndex(i, ifc_file_count) { + if (!ifc_files[i].is_valid) { lnk_error(LNK_Error_Cmdl, "/ifcDebugRecords: %S", read.errors[i]); } + } + lnk_log(LNK_Log_Timers, "[IFC] read+parse %llu blob(s) in %.2f ms", ifc_file_count, (F64)(now_time_us() - discover_end_us) / 1000.0); + } + + // --- inject blob objs into the parallel arrays (like type servers, but in ifc_obj_range) --- + U64 prev_count = input->count; + U64 new_count = prev_count + ifc_file_count; + + LNK_Obj **obj_arr2 = push_array(arena, LNK_Obj *, new_count); + CV_DebugS *debug_s_arr2 = push_array(arena, CV_DebugS, new_count); + CV_DebugT *debug_t_arr2 = push_array(arena, CV_DebugT, new_count); + CV_DebugH *debug_h_arr2 = push_array(arena, CV_DebugH, new_count); + U64 *obj_to_ts2 = push_array(arena, U64, new_count); + + MemoryCopyTyped(obj_arr2, input->obj_arr, prev_count); + MemoryCopyTyped(debug_s_arr2, input->debug_s_arr, prev_count); + MemoryCopyTyped(debug_t_arr2, input->debug_t_arr, prev_count); + MemoryCopyTyped(debug_h_arr2, input->debug_h_arr, prev_count); + MemoryCopyTyped(obj_to_ts2, input->obj_to_ts, prev_count); + MemorySet(obj_to_ts2 + prev_count, 0xff, ifc_file_count * sizeof(U64)); // blobs are not type servers + + // blob obj indices + index list for hash-deep / dedup + U32Array ifc_indices = { .v = push_array(arena, U32, ifc_file_count) }; + for EachIndex(i, ifc_file_count) { + U64 blob_obj_idx = prev_count + i; + LNK_Obj *blob_obj = push_array(arena, LNK_Obj, 1); + blob_obj->path = ifc_files[i].path; + obj_arr2[blob_obj_idx] = blob_obj; + debug_t_arr2[blob_obj_idx] = blob_debug_t[i]; + ifc_indices.v[ifc_indices.count++] = (U32)blob_obj_idx; + } + + input->count = new_count; + input->obj_arr = obj_arr2; + input->debug_s_arr = debug_s_arr2; + input->debug_t_arr = debug_t_arr2; + input->debug_h_arr = debug_h_arr2; + input->obj_to_ts = obj_to_ts2; + input->ifc_obj_range = r1u64(prev_count, new_count); + input->ifc_indices = ifc_indices; // hashed + deduped before int objs (see lnk_merge_types) + + // --- on-demand pruning state: per blob, a "referenced" bitset of leaf indices that + // are reachable from some consuming obj's 0x1522 redirect (the closure roots). Only these + // + their transitive blob-internal deps get merged; the rest are rewritten to NOTYPE so the + // hash/dedup pipeline skips ~all of the ~1.5M blob leaves that nothing references. --- + U8 **ref_bits = push_array(scratch.arena, U8 *, ifc_file_count); + for EachIndex(i, ifc_file_count) { + U64 c = blob_debug_t[i].count; + ref_bits[i] = push_array(scratch.arena, U8, (c + 7) / 8); // zero-init -> nothing referenced yet + } + + // --- parallel record resolution + placeholder NOTYPE: per-obj disjoint, order-free (see + // lnk_ifc_resolve_task). All inputs (entry->blob_slot, ifc_files, blob debug_t) are frozen + // after the discovery/read/injection steps above. --- + input->has_ifc_redirects = 1; + input->ifc_redirect_bits = push_array(arena, U64 *, input->count); + input->ifc_redirect_ti_rng = push_array(arena, Rng1U64, input->count); + U64 redirect_count = 0; + U64 resolve_begin_us = now_time_us(); + LNK_IfcResolveTask resolve = {0}; + resolve.input = input; + resolve.ifc_files = ifc_files; + resolve.recs = scan.out_recs; + resolve.counts = scan.out_counts; + resolve.res_counts = push_array(scratch.arena, U64, input->obj_count); + resolve.k_first = push_array(scratch.arena, U64, input->obj_count); + resolve.k_last = push_array(scratch.arena, U64, input->obj_count); + // P4: pre-reserve NOTYPE journal capacity + bitmap serially (one entry per 0x1522 record); + // the parallel resolve task pushes with arena == 0 and must never allocate + for EachIndex(obj_idx, input->obj_count) { + U64 n = scan.out_counts[obj_idx]; + if (n == 0) { continue; } + LNK_NotypeJournal *journal = &input->notype_journal[obj_idx]; + U32 need = journal->count + (U32)n; + if (need > journal->cap) { + U32 *new_v = push_array_no_zero(arena, U32, need); + MemoryCopyTyped(new_v, journal->v, journal->count); + journal->v = new_v; + journal->cap = need; + } + if (journal->bitmap == 0) { + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + journal->bit_cap = debug_t->count; + journal->bitmap = push_array(arena, U64, (journal->bit_cap + 63) / 64); + } + } + tp_for_parallel(tp, 0, input->obj_count, lnk_ifc_resolve_task, &resolve); + lnk_log(LNK_Log_Timers, "[IFC] parallel resolve in %.2f ms", (F64)(now_time_us() - resolve_begin_us) / 1000.0); + + // --- serial redirect replay: push redirects in ascending obj_idx, then ascending leaf_idx -- + // the exact original serial order -- so the redirect hash-map push order, ref_bits seeding, + // and redirect_count are bit-for-bit identical to the serial code. + U64 merge_begin_us = now_time_us(); + for EachIndex(obj_idx, input->obj_count) { + LNK_IfcRawRec *recs = scan.out_recs[obj_idx]; + U64 n = scan.out_counts[obj_idx]; + if (n == 0 || resolve.res_counts[obj_idx] == 0) { continue; } + + // exact key filter range: records are K-ascending (the scan emits leaf_idx ascending and + // cv_ti_from_leaf_idx is monotonic), so [k_first, k_last] spans all resolved keys. + Rng1U64 krng = r1u64(resolve.k_first[obj_idx], resolve.k_last[obj_idx] + 1); + input->ifc_redirect_ti_rng[obj_idx] = krng; + input->ifc_redirect_bits[obj_idx] = push_array(arena, U64, (dim_1u64(krng) + 63) / 64); + for EachIndex(t, n) { + LNK_IfcRawRec *r = &recs[t]; + if (r->blob_i_plus1 == 0) { continue; } + U64 blob_i = r->blob_i_plus1 - 1; + U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; + hash_map_push_u64_u64(arena, &input->ifc_redirect_hm, + Compose64Bit(obj_idx, r->K), + Compose64Bit(blob_obj_idx, r->blob_leaf_idx)); + U64 rel = r->K - krng.min; + input->ifc_redirect_bits[obj_idx][rel >> 6] |= (1ull << (rel & 63)); + redirect_count += 1; + // seed closure root: this blob leaf is referenced + ref_bits[blob_i][r->blob_leaf_idx >> 3] |= (U8)(1u << (r->blob_leaf_idx & 7)); + } + } + + lnk_log(LNK_Log_Timers, "[IFC] redirect replay in %.2f ms", (F64)(now_time_us() - merge_begin_us) / 1000.0); + + // --- third pass: per blob, close the referenced set over blob-internal sub-TIs, then + // NOTYPE every leaf not in the closure. cv_leaf_idx_from_ti on a raw blob is source-agnostic + // (source_offsets are 0, all ti_ranges == [0x1000, 0x1000+count)) so a sub-TI maps directly to + // leaf_idx = ti - 0x1000 regardless of its CV_TypeIndexSource label. Walk is iterative (worklist). + U64 total_blob_leaves = 0, total_closure_leaves = 0; + U64 closure_begin_us = now_time_us(); + for EachIndex(blob_i, ifc_file_count) { + total_blob_leaves += input->debug_t_arr[input->ifc_obj_range.min + blob_i].count; + } + if (ifc_file_count) { + // Build the set of unique_names that already have a COMPLETE definition in some non-blob obj. + // The blob prune keeps a blob complete-def only when its name is absent here (blob-only type), + // so forward-refs that no normal obj can complete still get their definition (full-merge fidelity) + // while redundant blob copies of normally-defined types stay pruned. Size to ~2x the non-blob + // complete-def count, rounded up to a power of two, for low load factor. + U64 nonblob_complete_estimate = 0; + for EachIndex(obj_idx, input->ifc_obj_range.min) { + nonblob_complete_estimate += input->debug_t_arr[obj_idx].source_counts[CV_TypeIndexSource_TPI]; + } + LNK_U64Set nonblob_complete = {0}; + nonblob_complete.cap = 1; + while (nonblob_complete.cap < (nonblob_complete_estimate * 2 + 16)) { nonblob_complete.cap <<= 1; } + nonblob_complete.slots = push_array(scratch.arena, U64, nonblob_complete.cap); + // parallel scan: each non-blob obj emits its complete-def hashes; serial merge adds to the set. + U64 nonblob_count = input->ifc_obj_range.min; + if (nonblob_count) { + LNK_IfcCompleteScanTask scan = {0}; + scan.input = input; + scan.out_hashes = push_array(scratch.arena, U64 *, nonblob_count); + scan.out_counts = push_array(scratch.arena, U64, nonblob_count); + tp_for_parallel(tp, tp_arena, nonblob_count, lnk_ifc_complete_scan_task, &scan); + U64 scan_hash_count = 0; + U64 scan_sidecar_count = 0; + for EachIndex(obj_idx, nonblob_count) { + scan_hash_count += scan.out_counts[obj_idx]; + scan_sidecar_count += (input->debug_t_arr[obj_idx].sidecar_complete_udt_hashes != 0); + } + lnk_log(LNK_Log_Timers, "[IFC] complete UDT scan: hashes=%llu sidecars=%llu/%llu", + scan_hash_count, scan_sidecar_count, nonblob_count); + // parallel atomic-CAS merge (replaces the serial lnk_u64set_add loop): output-identical + // because set membership is order-independent + idempotent (see lnk_u64set_add_atomic). + LNK_IfcSetMergeTask merge = {0}; + merge.ranges = tp_divide_work(scratch.arena, nonblob_count, tp->worker_count); + merge.out_hashes = scan.out_hashes; + merge.out_counts = scan.out_counts; + merge.set = &nonblob_complete; + merge.nonblob_count = nonblob_count; + tp_for_parallel(tp, 0, tp->worker_count, lnk_ifc_set_merge_task, &merge); + } + + LNK_IfcCloseTask close_task = {0}; + close_task.input = input; + close_task.ref_bits = ref_bits; + close_task.closure_leaves = push_array(scratch.arena, U64, ifc_file_count); + close_task.nonblob_complete = &nonblob_complete; + tp_for_parallel(tp, tp_arena, ifc_file_count, lnk_ifc_close_blob_task, &close_task); + for EachIndex(blob_i, ifc_file_count) { total_closure_leaves += close_task.closure_leaves[blob_i]; } + } + (void)tp; + lnk_log(LNK_Log_Timers, "[IFC] closure pass in %.2f ms", (F64)(now_time_us() - closure_begin_us) / 1000.0); + + lnk_log(LNK_Log_Debug, "[IFC] injected %llu .ifc blob(s), %llu record redirect(s); on-demand closure %llu / %llu blob leaves (%.1f%%)", + ifc_file_count, redirect_count, total_closure_leaves, total_blob_leaves, + total_blob_leaves ? (100.0 * (F64)total_closure_leaves / (F64)total_blob_leaves) : 0.0); + +done: + scratch_end(scratch); + lnk_log(LNK_Log_Timers, "[IFC] apply total in %.2f ms", (F64)(now_time_us() - apply_begin_us) / 1000.0); + ProfEnd(); +} + +//////////////////////////////// +// parallel setup tasks for lnk_make_code_view_input + +// Loop 3 (PCH/ext/int classification). The expensive predicates -- the read-only rrt_hm lookup +// and cv_debug_t_is_type_server_ref -- run in parallel per obj. Each obj's class tag and PCH-merge +// mutation are fully independent, so this pass is data-parallel. The ordered 3-array compaction +// and the MultipleDebugTAndDebugP warning are then replayed SERIALLY in obj_idx order, so output +// (array contents/order + warning order + discarded set) is byte-identical to the serial loop. +typedef struct LNK_CvClassifyTask +{ + LNK_CodeViewInput *input; + CV_DebugT *debug_p_arr; + HashMap *rrt_hm; // read-only after build + LNK_Obj **obj_arr; + U8 *class_tag; // 0=debug_p, 1=ext, 2=int + U8 *warn_multi; +} LNK_CvClassifyTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_cv_classify_task) +{ + LNK_CvClassifyTask *t = raw_task; + U64 obj_idx = task_id; + CV_DebugT *debug_t = &t->input->debug_t_arr[obj_idx]; + CV_DebugT *debug_p = &t->debug_p_arr[obj_idx]; + + // classify (same predicate order/precedence as the serial loop) + U8 tag; + if (hash_map_search_path_u64(t->rrt_hm, t->obj_arr[obj_idx]->path)) { tag = 1; } + else if (debug_p->count > 0 && debug_t->count == 0) { tag = 0; } + else if (cv_debug_t_is_type_server_ref(debug_t)) { tag = 1; } + else { tag = 2; } + t->class_tag[obj_idx] = tag; + + // per-obj independent debug_t mutation (identical to serial) + if (debug_t->count == 0 && debug_p->count > 0) { + *debug_t = *debug_p; + } else if (debug_t->count && debug_p->count) { + t->warn_multi[obj_idx] = 1; // defer warning to serial obj-order replay + MemoryZeroStruct(debug_t); + MemoryZeroStruct(debug_p); + } +} + +// Loop 4 (Make Symbol Inputs) count pass. cv_sub_section_from_debug_s is a pure read of the +// already-parsed data_list, so caching each obj's Symbols sub-section list in parallel is safe. +typedef struct LNK_CvSymTask +{ + LNK_CodeViewInput *input; + String8List *per_obj_syms; + U64 *counts; // per-obj node_count (count pass) + U64 *offsets; // per-obj symbol_inputs offset (fill pass) +} LNK_CvSymTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_cv_sym_count_task) +{ + LNK_CvSymTask *t = raw_task; + U64 obj_idx = task_id; + t->per_obj_syms[obj_idx] = cv_sub_section_from_debug_s(t->input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); + t->counts[obj_idx] = t->per_obj_syms[obj_idx].node_count; +} + +// Loop 4 fill pass. Each obj writes a disjoint, contiguous range of symbol_inputs starting at its +// prefix-sum offset, in node order -- byte-identical to the serial append (which walked obj_idx +// ascending, each obj's nodes in list order). +internal +THREAD_POOL_TASK_FUNC(lnk_cv_sym_fill_task) +{ + LNK_CvSymTask *t = raw_task; + U64 obj_idx = task_id; + U64 cur = t->offsets[obj_idx]; + String8List s = t->per_obj_syms[obj_idx]; + for EachNode(n, String8Node, s.first) { + LNK_SymbolInput *in = &t->input->symbol_inputs[cur++]; + in->obj_idx = obj_idx; + in->raw_symbols = n->string; + } +} + +internal LNK_CodeViewInput +lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 obj_count, LNK_Obj **obj_arr, LNK_RRT_Array rrt_input) +{ + ProfBegin("Extract CodeView"); + Temp scratch = scratch_begin(0,0); + + LNK_CodeViewInput input = { .config = config, .obj_count = obj_count, .count = obj_count, .obj_arr = obj_arr, .rrt_input = rrt_input, .ts_obj_range = r1u64(0,0) }; + + // $T streaming (ring P4): per-real-obj NOTYPE journals (see LNK_NotypeJournal). Real objs + // only ([0, obj_count)); pseudo objs appended later mutate their arena-backed $T in place. + input.notype_journal = push_array(tp_arena->v[0], LNK_NotypeJournal, obj_count ? obj_count : 1); + + HashMap rrt_hm = {0}; + ProfScope("Make obj path -> RRT hash map") + { + for EachIndex(rrt_idx, rrt_input.count) { + for EachIndex(obj_idx, rrt_input.v[rrt_idx].obj_paths.count) { + hash_map_push_path_u64(scratch.arena, &rrt_hm, rrt_input.v[rrt_idx].obj_paths.v[obj_idx], Compose64Bit(rrt_idx, obj_idx)); + } + } + } + + ProfBegin("Apply RRT to Objs"); + + // hash map (obj path, obj idx). Kept SERIAL: HashMap is a 4-ary trie whose insert mutates shared + // child pointers + arena-allocates nodes -> not safe for concurrent insert. Only built (and + // consulted) when there is at least one input RRT; the monolithic Engine.dll link has none. + HashMap obj_path_hm = {0}; + if (rrt_input.count) { + for EachIndex(obj_idx, obj_count) { + hash_map_push_path_u64(scratch.arena, &obj_path_hm, obj_arr[obj_idx]->path, obj_idx); + } + + for EachIndex(obj_idx, obj_count) { + LNK_Obj *obj = obj_arr[obj_idx]; + U64 *packed_rrt_idx = hash_map_search_path_u64(&rrt_hm, obj->path); + + // obj is not part of any input RRT + if (packed_rrt_idx == 0) { continue; } + + // unpack index + U32 rrt_idx = *packed_rrt_idx >> 32; + U32 rrt_obj_idx = *packed_rrt_idx & max_U32; + LNK_RRT *rrt = &rrt_input.v[rrt_idx]; + + // obj was recompiled, do not apply RRT indirection + FileProperties obj_file_props = properties_from_file_path(obj->path); + if (rrt->obj_time_stamps[rrt_obj_idx] != obj_file_props.modified) { continue; } + + // invalidate debug section pointers + obj->coff.debug_t_section_number = 0; + obj->coff.debug_p_section_number = 0; + obj->coff.debug_h_section_number = 0; + + // apply type index map + obj->ti_range = rrt->obj_ti_ranges[rrt_obj_idx]; + obj->ti_map = rrt->obj_ti_maps [rrt_obj_idx]; + + // apply PCH info + U32 rrt_pch_obj_idx = rrt->obj_pch_indices[rrt_obj_idx]; + if (rrt_pch_obj_idx < rrt->obj_count) { + String8 rrt_pch_obj_path = rrt->obj_paths.v[rrt_pch_obj_idx]; + U64 pch_obj_idx = *hash_map_search_path_u64(&obj_path_hm, rrt_pch_obj_path); + obj->pch_ti_range = rrt->obj_pch_ti_ranges[rrt_obj_idx]; + obj->pch_obj_idx = pch_obj_idx; + } else { + obj->pch_ti_range = r1u64(0,0); + obj->pch_obj_idx = ~0; + } + } + } + ProfEnd(); + + ProfBegin("Collect CodeView"); + input.debug_s_list_arr = lnk_collect_obj_sections(tp, tp_arena, obj_count, obj_arr, str8_lit(".debug$S"), 0, &input.debug_s_sect_idx_arr); + ProfEnd(); + + // batch-populate the mapped .debug$S/$T/$P/$H input ranges before the parse + // loops below first-touch them page by page (see lnk_prefetch_ranges) + if (lnk_should_prefetch_mapped_input()) ProfScope("Prefetch CodeView") + { + Temp temp = temp_begin(scratch.arena); + + U64 range_cap = 3 * obj_count; // debug$T + debug$P + debug$H + for EachIndex(obj_idx, obj_count) { range_cap += input.debug_s_list_arr[obj_idx].node_count; } + + Rng1U64 *ranges = push_array_no_zero(temp.arena, Rng1U64, range_cap); + U64 range_count = 0; + for EachIndex(obj_idx, obj_count) { + LNK_Obj *obj = obj_arr[obj_idx]; + + for EachNode(n, String8Node, input.debug_s_list_arr[obj_idx].first) { + if (n->string.size) { ranges[range_count++] = rng_1u64((U64)n->string.str, (U64)n->string.str + n->string.size); } + } + if (obj->coff.debug_t_section_number) { + String8 data = lnk_obj_section_data_from_number(obj, obj->coff.debug_t_section_number); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + if (obj->coff.debug_p_section_number) { + String8 data = lnk_obj_section_data_from_number(obj, obj->coff.debug_p_section_number); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + if (config->ghash && obj->coff.debug_h_section_number) { + String8 data = lnk_obj_section_data_from_number(obj, obj->coff.debug_h_section_number); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + } + Assert(range_count <= range_cap); + U64 prefetch_begin_us = now_time_us(); + U64 prefetch_bytes = 0; + for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } + lnk_prefetch_ranges(tp, config->debug_worker_cap, range_count, ranges); + lnk_log(LNK_Log_Timers, "[mcvi] prefetched %llu debug section ranges (%llu MiB) in %.2f ms", + range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); + + temp_end(temp); + } + + // profiler info + if (lnk_get_log_status(LNK_Log_Debug) || PROFILE_TELEMETRY) { + U64 total_debug_s_size = 0, total_debug_t_size = 0, total_debug_p_size = 0, total_debug_h_size = 0; + for EachIndex(obj_idx, obj_count) { + LNK_Obj *obj = obj_arr[obj_idx]; + + for EachNode(n, String8Node, input.debug_s_list_arr[obj_idx].first) { total_debug_s_size += n->string.size; } + + if (obj->coff.debug_t_section_number > 0) { + total_debug_t_size += lnk_coff_section_header_from_section_number(obj, obj->coff.debug_t_section_number)->fsize; + } + if (obj->coff.debug_p_section_number > 0) { + total_debug_p_size += lnk_coff_section_header_from_section_number(obj, obj->coff.debug_p_section_number)->fsize; + } + if (config->ghash) { + if (obj->coff.debug_h_section_number > 0) { + total_debug_h_size += lnk_coff_section_header_from_section_number(obj, obj->coff.debug_h_section_number)->fsize; + } + } + } + + U64 total_rrt_type_size = 0; + U64 total_rrt_hash_size = 0; + for EachIndex(rrt_idx, rrt_input.count) { total_rrt_type_size += rrt_input.v[rrt_idx].type_data_raw.size; total_rrt_hash_size += rrt_input.v[rrt_idx].type_hashes.size; } @@ -775,7 +2216,25 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, { // parse .debug$S input.debug_s_arr = push_array(tp_arena->v[0], CV_DebugS, input.obj_count); - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_s_task, &input, "Parse .debug$S"); + lnk_compressed_obj_log_phase_stats("before parse $S"); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_s_task, &input, "Parse .debug$S"); + lnk_compressed_obj_log_phase_stats("after parse $S"); + { + U64 bytes[CV_C13SubSectionIdxKind_COUNT] = {0}; + U64 nodes[CV_C13SubSectionIdxKind_COUNT] = {0}; + for EachIndex(obj_idx, obj_count) { + for EachElement(k, input.debug_s_arr[obj_idx].data_list) { + bytes[k] += input.debug_s_arr[obj_idx].data_list[k].total_size; + nodes[k] += input.debug_s_arr[obj_idx].data_list[k].node_count; + } + } + for EachIndex(k, CV_C13SubSectionIdxKind_COUNT) { + if (bytes[k] || nodes[k]) { + lnk_log(LNK_Log_Timers, "[debugS parsed] kind=%x bytes=%llu nodes=%llu", + cv_c13_sub_section_kind_from_idx(k), bytes[k], nodes[k]); + } + } + } // collect .debug$P and .debug$T String8Array *raw_debug_p_arr = push_array(scratch.arena, String8Array, obj_count); @@ -802,44 +2261,60 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, debug_p_arr = push_array(tp_arena->v[0], CV_DebugT, obj_count); parse_types.raw_types = raw_debug_p_arr; parse_types.out_types = debug_p_arr; - tp_for_parallel_prof(tp, 0, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$P"); - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$P"); + parse_types.is_debug_p = 1; + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$P"); + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_parse_debug_t_sidecar_task, &parse_types, "Index .debug$P sidecars"); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$P"); // parse .debug$T input.debug_t_arr = push_array(tp_arena->v[0], CV_DebugT, obj_count); parse_types.raw_types = raw_debug_t_arr; parse_types.out_types = input.debug_t_arr; - tp_for_parallel_prof(tp, 0, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$T"); - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$T"); + parse_types.is_debug_p = 0; + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$T"); + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_parse_debug_t_sidecar_task, &parse_types, "Index .debug$T sidecars"); + lnk_parse_giant_debug_t(tp, tp_arena, config, obj_count, &parse_types); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$T"); // parse .debug$H input.debug_h_arr = push_array(tp_arena->v[0], CV_DebugH, input.obj_count); if (config->ghash) { - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_h_task, &input, "Parse .debug$H"); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_h_task, &input, "Parse .debug$H"); } } ProfEnd(); + lnk_compressed_obj_log_phase_stats("after parse CodeView"); // sort objs based on type: PCH, /Zi (external), /Z7 (internal) - input.debug_p_indices.v = push_array(tp_arena->v[0], U32, obj_count); + input.debug_p_indices.v = push_array(tp_arena->v[0], U32, obj_count); input.ext_obj_indices.v = push_array(tp_arena->v[0], U32, obj_count); input.int_obj_indices.v = push_array(tp_arena->v[0], U32, obj_count); - for EachIndex(obj_idx, obj_count) { - CV_DebugT *debug_t = &input.debug_t_arr[obj_idx]; - CV_DebugT *debug_p = &debug_p_arr[obj_idx]; - U32Array *arr_ptr; - if (hash_map_search_path_u64(&rrt_hm, obj_arr[obj_idx]->path)) { arr_ptr = &input.ext_obj_indices; } - else if (debug_p->count > 0 && debug_t->count == 0) { arr_ptr = &input.debug_p_indices; } - else if (cv_debug_t_is_type_server_ref(debug_t)) { arr_ptr = &input.ext_obj_indices; } - else { arr_ptr = &input.int_obj_indices; } - arr_ptr->v[arr_ptr->count++] = obj_idx; + ProfScope("Classify Objs") + { + // parallel: classify each obj + apply per-obj debug_t mutation (see lnk_cv_classify_task) + LNK_CvClassifyTask classify = {0}; + classify.input = &input; + classify.debug_p_arr = debug_p_arr; + classify.rrt_hm = &rrt_hm; + classify.obj_arr = obj_arr; + classify.class_tag = push_array(scratch.arena, U8, obj_count ? obj_count : 1); + classify.warn_multi = push_array(scratch.arena, U8, obj_count ? obj_count : 1); + tp_for_parallel_prof(tp, 0, obj_count, lnk_cv_classify_task, &classify, "Classify Objs (parallel)"); + + // serial obj-order compaction into the 3 ordered arrays + deterministic warning emission. + // Cache-linear single pass; preserves the exact element order + warning order of the old loop. + for EachIndex(obj_idx, obj_count) { + U32Array *arr_ptr; + switch (classify.class_tag[obj_idx]) { + case 0: arr_ptr = &input.debug_p_indices; break; + case 1: arr_ptr = &input.ext_obj_indices; break; + default: arr_ptr = &input.int_obj_indices; break; + } + arr_ptr->v[arr_ptr->count++] = obj_idx; - if (debug_t->count == 0 && debug_p->count > 0) { - *debug_t = *debug_p; - } else if (debug_t->count && debug_p->count) { - lnk_error_obj(LNK_Warning_MultipleDebugTAndDebugP, obj_arr[obj_idx], "multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections"); - MemoryZeroStruct(debug_t); - MemoryZeroStruct(debug_p); + if (classify.warn_multi[obj_idx]) { + lnk_error_obj(LNK_Warning_MultipleDebugTAndDebugP, obj_arr[obj_idx], "multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections"); + } } } @@ -878,6 +2353,10 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, 0); ts_kind = LNK_TypeServerKind_PDB; ts_info = cv_type_server_info_from_leaf(leaf); + // P4: ts_info.name points into the raw $T leaf 0 bytes of the mapped obj view -- copy it + // out so nothing downstream retains a raw-view pointer (ts_info is stored in ts_arr and + // read during/after the merge) + ts_info.name = push_str8_copy(tp_arena->v[0], ts_info.name); ts_path = lnk_find_first_file(scratch.arena, config->lib_dir_list, ts_info.name); } @@ -943,10 +2422,12 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, input.ts_obj_range = r1u64(prev.count, input.count); // alloc dummy objs with for each loaded type server + // (one obj per type server; this used to push a ts_arr.count-sized array + // per iteration and use only its first element -- O(T^2) arena growth) + LNK_Obj *ts_objs = push_array(tp_arena->v[0], LNK_Obj, ts_arr.count); for EachIndex(i, ts_arr.count) { - LNK_Obj *ts_obj = push_array(tp_arena->v[0], LNK_Obj, ts_arr.count); - ts_obj->path = ts_arr.v[i].ts_path; - input.obj_arr[prev.count + i] = ts_obj; + ts_objs[i].path = ts_arr.v[i].ts_path; + input.obj_arr[prev.count + i] = &ts_objs[i]; } // make type server indices @@ -1079,19 +2560,19 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, // remove CV_LeafKind_PRECOMP debug_t->count -= 1; - debug_t->offsets += 1; + debug_t->sidecar_leaf_bias += 1; } - // remove LF_ENDPRECOMP from .debug$P + // remove LF_ENDPRECOMP from .debug$P -- P4: journal the NOTYPE rewrite instead of dirtying + // the mapped $P view (the backward header scan to FIND it still reads the raw view; those + // pages are hot from the parse) for EachIndex(i, input.debug_p_indices.count) { U64 debug_p_idx = input.debug_p_indices.v[i]; CV_DebugT *debug_p = &input.debug_t_arr[debug_p_idx]; for EachIndex(i, debug_p->count) { U64 lf_idx = debug_p->count - (i + 1); - CV_LeafHeader *lf = cv_debug_t_get_leaf_header(debug_p, lf_idx); - if (lf->kind == CV_LeafKind_ENDPRECOMP) { - memory_write16(&lf->size, sizeof(lf->kind)); - memory_write16(&lf->kind, CV_LeafKind_NOTYPE); + if (cv_debug_t_get_leaf_kind(debug_p, lf_idx) == CV_LeafKind_ENDPRECOMP) { + lnk_notype_journal_push(tp_arena->v[0], &input.notype_journal[debug_p_idx], (U32)lf_idx, 0, debug_p->count); break; } } @@ -1099,6 +2580,12 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, } ProfEnd(); + // resolve MSVC header-unit IFC debug records (LF_IFC_RECORD 0x1522) -> real CodeView types. + // injects .ifc debug-records blobs as extra objs and registers placeholder-TI redirects. + if (config->ifc_debug_records == LNK_SwitchState_Yes && config->ifc_map_list.node_count) { + lnk_apply_ifc_debug_records(tp, tp_arena, &input, config); + } + // set default min type index for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { input.min_type_indices[ti_source] = CV_MinComplexTypeIndex; } @@ -1115,25 +2602,24 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfBegin("Make Symbol Inputs"); { - // count symbol blocks - for EachIndex(obj_idx, input.count) { - String8List s = cv_sub_section_from_debug_s(input.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); - input.symbol_input_count += s.node_count; - } - + // count symbol blocks (cache each obj's Symbols sub-section list so the fill pass below + // does not re-decode .debug$S a second time -- cv_sub_section_from_debug_s walks subsections). + String8List *per_obj_syms = push_array(scratch.arena, String8List, input.count ? input.count : 1); + LNK_CvSymTask sym_task = {0}; + sym_task.input = &input; + sym_task.per_obj_syms = per_obj_syms; + sym_task.counts = push_array(scratch.arena, U64, input.count ? input.count : 1); + + // parallel: cache each obj's Symbols sub-section list + count nodes + tp_for_parallel_prof(tp, 0, input.count, lnk_cv_sym_count_task, &sym_task, "Count Symbol Inputs"); + input.symbol_input_count = sum_array_u64(input.count, sym_task.counts); + sym_task.offsets = offsets_from_counts_array_u64(scratch.arena, sym_task.counts, input.count); + // alloc block pointers - input.symbol_inputs = push_array_no_zero(tp_arena->v[0], LNK_SymbolInput, input.symbol_input_count); + input.symbol_inputs = push_array_no_zero(tp_arena->v[0], LNK_SymbolInput, input.symbol_input_count ? input.symbol_input_count : 1); - U64 symbol_input_count = 0; - for EachIndex(obj_idx, input.count) { - String8List s = cv_sub_section_from_debug_s(input.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); - for EachNode(n, String8Node, s.first) { - Assert(symbol_input_count < input.symbol_input_count); - LNK_SymbolInput *in = &input.symbol_inputs[symbol_input_count++]; - in->obj_idx = obj_idx; - in->raw_symbols = n->string; - } - } + // parallel fill into disjoint per-obj ranges at prefix-sum offsets (byte-identical order) + tp_for_parallel_prof(tp, 0, input.count, lnk_cv_sym_fill_task, &sym_task, "Fill Symbol Inputs"); ProfBegin("Make Ranges"); @@ -1142,7 +2628,8 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 max_weight = CeilIntegerDiv(total_input_size, tp->worker_count); U64 cursor = 0; - input.symbol_input_ranges = push_array(tp_arena->v[0], Rng1U64, tp->worker_count); + input.symbol_input_ranges = push_array(tp_arena->v[0], Rng1U64, tp->worker_count); + input.symbol_input_range_count = tp->worker_count; for EachIndex(i, tp->worker_count) { if (cursor >= input.symbol_input_count) { break; } U64 begin = cursor; @@ -1184,6 +2671,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, } } ProfEnd(); + lnk_compressed_obj_log_phase_stats("after make symbol inputs"); scratch_end(scratch); ProfEnd(); @@ -1193,7 +2681,10 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, internal force_inline LNK_LeafRef lnk_leaf_ref_make(U64 obj_idx, U64 leaf_idx) { - LNK_LeafRef result = Compose64Bit(safe_cast_u32(obj_idx), safe_cast_u32(leaf_idx)); + // Type indices consume the low 32 bits. Realistic object counts fit comfortably in the + // remaining 29 payload bits; the top three bits stay free for the dedup-table hash tag. + Assert(obj_idx < (1ull << 29) - 1); + LNK_LeafRef result = (obj_idx << 32) | safe_cast_u32(leaf_idx); Assert(result != LNK_LEAF_REF_NULL); return result; } @@ -1201,7 +2692,7 @@ lnk_leaf_ref_make(U64 obj_idx, U64 leaf_idx) internal force_inline U32 lnk_leaf_ref_obj_idx(LNK_LeafRef ref) { - return (U32)(ref >> 32); + return (U32)((ref >> 32) & ((1ull << 29) - 1)); } internal force_inline U32 @@ -1210,9 +2701,149 @@ lnk_leaf_ref_leaf_idx(LNK_LeafRef ref) return (U32)ref; } +// $T streaming (ring P4): NOTYPE journal ops. Pushes are single-threaded per obj (input-phase +// pushes are serial or per-obj tasks; hash-phase pushes come from the obj's own hash task), so +// no atomics. Sorted insert keeps lookup a bsearch; the common shapes are 0 entries (fast path) +// or an append at the tail (input-phase entries land before hash-phase entries only for $P objs +// that are also IFC consumers -- the linear tail walk handles the interleave). +internal void +lnk_notype_journal_push(Arena *arena, LNK_NotypeJournal *journal, U32 leaf_idx, B32 kind_only, U64 bit_cap) +{ + if (journal->count == journal->cap) { + AssertAlways(arena != 0); // pseudo objs never journal; real-obj pushes always have an arena + U32 new_cap = journal->cap ? journal->cap * 2 : 8; + U32 *new_v = push_array_no_zero(arena, U32, new_cap); + MemoryCopyTyped(new_v, journal->v, journal->count); + journal->v = new_v; + journal->cap = new_cap; + } + if (journal->bitmap == 0) { + AssertAlways(arena != 0); + journal->bitmap = push_array(arena, U64, (bit_cap + 63) / 64); + journal->bit_cap = bit_cap; + } + // out-of-span indices (the pre-existing `curr_ti - min` quirk on invalid-TI error paths can + // exceed the leaf count) get a journal entry but no bit; readers only query leaf_idx < count, + // so the entry is unreachable either way (the old in-place write was equally out-of-bounds) + if (leaf_idx < journal->bit_cap) { journal->bitmap[leaf_idx >> 6] |= (1ull << (leaf_idx & 63)); } + U32 entry = leaf_idx | (kind_only ? LNK_NOTYPE_JOURNAL_KIND_ONLY : 0); + U32 i = journal->count; + for (; i > 0 && (journal->v[i-1] & ~LNK_NOTYPE_JOURNAL_KIND_ONLY) > leaf_idx; i -= 1) { + journal->v[i] = journal->v[i-1]; + } + Assert(i == 0 || (journal->v[i-1] & ~LNK_NOTYPE_JOURNAL_KIND_ONLY) != leaf_idx); + journal->v[i] = entry; + journal->count += 1; +} + +// O(1) hot-path test: bitmap == 0 for every obj without journal entries (the common case) +internal B32 +lnk_notype_journal_test(LNK_NotypeJournal *journal, U64 leaf_idx) +{ + return journal->bitmap != 0 && leaf_idx < journal->bit_cap && ((journal->bitmap[leaf_idx >> 6] >> (leaf_idx & 63)) & 1); +} + +internal B32 +lnk_notype_journal_find(LNK_NotypeJournal *journal, U32 leaf_idx, B32 *kind_only_out) +{ + if (journal->count == 0) { return 0; } + U32 lo = 0, hi = journal->count; + while (lo < hi) { + U32 mid = lo + (hi - lo) / 2; + U32 key = journal->v[mid] & ~LNK_NOTYPE_JOURNAL_KIND_ONLY; + if (key < leaf_idx) { lo = mid + 1; } + else if (key > leaf_idx) { hi = mid; } + else { + if (kind_only_out) { *kind_only_out = !!(journal->v[mid] & LNK_NOTYPE_JOURNAL_KIND_ONLY); } + return 1; + } + } + return 0; +} + +// Journal-aware leaf read: raw views hold PRE-rewrite bytes for real objs, so present the +// journaled view of the leaf to the hashers (identical to what the old in-place writes produced: +// full rewrite => { kind=LF_NOTYPE, empty payload }; KIND_ONLY => kind=LF_NOTYPE, payload kept). +internal CV_Leaf +lnk_cv_leaf_from_leaf_ref(Arena *arena, LNK_CObjDecodeWindow *decode_window, + LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx) +{ + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + CV_Leaf leaf = {0}; + LNK_Obj *obj = obj_idx < input->obj_count ? input->obj_arr[obj_idx] : 0; + if (obj && obj->compressed_obj && (debug_t->sidecar_sizes || debug_t->sidecar_packed) && decode_window) { + U64 raw_size = cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + U8 *raw_leaf = push_array_no_zero(arena, U8, raw_size); + U64 leaf_off = cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + Rng1U64 raw_range = rng_1u64(debug_t->sidecar_raw_base + leaf_off, + debug_t->sidecar_raw_base + leaf_off + raw_size); + if (lnk_compressed_obj_copy_range(obj->compressed_obj, raw_range, raw_leaf, decode_window)) { + cv_read_leaf(str8(raw_leaf, raw_size), 0, 1, &leaf); + } else { + leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + } + } else { + leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + } + // hot path: one bitmap test; journals are empty for ~all objs + if (obj_idx < input->obj_count && lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { + B32 kind_only = 0; + lnk_notype_journal_find(&input->notype_journal[obj_idx], leaf_idx, &kind_only); + leaf.kind = CV_LeafKind_NOTYPE; + if (!kind_only) { leaf.data.size = 0; } + } + return leaf; +} + +// Journal-aware raw leaf size (materialize buffer sizing + copy). Hot path = the ORIGINAL +// header read (one bitmap test on top); only a full NOTYPE rewrite changes the answer (the +// leaf shrank to its 4-byte header; KIND_ONLY keeps the size field). +#define LNK_LEAF_MATERIALIZE_FULL_NOTYPE 1 +#define LNK_LEAF_MATERIALIZE_KIND_NOTYPE 2 + +internal U64 +lnk_leaf_ref_materialize_meta(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) +{ + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + U64 raw_size = cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + U64 rewrite = 0; + if (obj_idx < input->obj_count && lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { + B32 kind_only = 0; + lnk_notype_journal_find(&input->notype_journal[obj_idx], leaf_idx, &kind_only); + if (kind_only) { + rewrite = LNK_LEAF_MATERIALIZE_KIND_NOTYPE; + } else { + raw_size = sizeof(CV_LeafHeader); + rewrite = LNK_LEAF_MATERIALIZE_FULL_NOTYPE; + } + } + return (raw_size << 2) | rewrite; +} + internal LNK_LeafRef lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource source, CV_TypeIndex ti) { + // IFC redirect: a consuming obj's local LF_IFC_RECORD placeholder TI is mapped + // to a leaf inside an injected .ifc debug-records blob obj. The blob leaves then + // dedup/hash/fixup natively through the rest of this function. + if (input->has_ifc_redirects && source == CV_TypeIndexSource_TPI) { + // exact per-obj bitset filter: bit set iff Compose64Bit(obj_idx, ti) is a key in + // ifc_redirect_hm. skips the (miss-dominated) per-call key hash + map walk; on a set + // bit the original map search runs unchanged, so behavior is bit-identical. + U64 *bits = input->ifc_redirect_bits[obj_idx]; + if (bits != 0 && contains_1u64(input->ifc_redirect_ti_rng[obj_idx], ti)) { + U64 rel = ti - input->ifc_redirect_ti_rng[obj_idx].min; + if (bits[rel >> 6] & (1ull << (rel & 63))) { + U64 *packed = hash_map_search_u64_u64(&input->ifc_redirect_hm, Compose64Bit(obj_idx, ti)); + if (packed) { + return lnk_leaf_ref_make((U32)(*packed >> 32), (U32)(*packed & max_U32)); + } + } + } + } + // ti range: external type server U64 ts_idx = input->obj_to_ts[obj_idx]; if (ts_idx != max_U64) { @@ -1276,15 +2907,35 @@ lnk_match_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b) return a_hash == b_hash; } +#define LNK_LEAF_BUCKET_TAG_MASK (7ull << 61) + +internal force_inline LNK_LeafRef +lnk_leaf_bucket_tag(LNK_LeafRef ref, U64 hash) +{ + Assert((ref & LNK_LEAF_BUCKET_TAG_MASK) == 0); + LNK_LeafRef result = ref | (hash & LNK_LEAF_BUCKET_TAG_MASK); + Assert(result != LNK_LEAF_REF_NULL); + return result; +} + +internal force_inline LNK_LeafRef +lnk_leaf_bucket_untag(LNK_LeafRef ref) +{ + return ref & ~LNK_LEAF_BUCKET_TAG_MASK; +} + +// P4: `leaf` is the caller's (journal-aware) read of the leaf -- this function no longer +// re-reads it from the raw view. `journal_arena` backs NOTYPE journal growth for real objs +// (pseudo objs keep in-place rewrites and may pass 0). internal U64 -lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles) +lnk_hash_cv_leaf(LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef leaf_ref, CV_Leaf leaf, CV_TiOffsets ti_offs, B32 discard_cycles) { U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource curr_ti_source = cv_type_index_source_from_leaf_kind(leaf.kind); CV_TypeIndex curr_ti = cv_ti_from_leaf_idx(debug_t, curr_ti_source, leaf_idx); + U64 ti_count = cv_ti_offsets_count(&ti_offs); // init hasher LNK_Hasher hasher; @@ -1293,11 +2944,12 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // hash bytes around indices { U64 last_ti_off = 0; - for EachNode(ti_info, CV_TypeIndexInfo, ti_info_list.first) { + for (U64 ti_idx = 0; ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); U8 *bytes = leaf.data.str + last_ti_off; - U64 size = ti_info->offset - last_ti_off; + U64 size = ti_info.offset - last_ti_off; lnk_hasher_update(&hasher, bytes, size); - last_ti_off = ti_info->offset + sizeof(CV_TypeIndex); + last_ti_off = ti_info.offset + sizeof(CV_TypeIndex); } Assert(leaf.data.size >= last_ti_off); @@ -1306,23 +2958,34 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf lnk_hasher_update(&hasher, bytes, size); } + // P4: set when a discard below rewrote THIS leaf's header (the final header mix-in must + // then hash the NOTYPE header, exactly like the old post-write pointer read did) + B32 self_discarded = 0; + // mix-in sub leaf hashes - for EachNode(sub_ti_n, CV_TypeIndexInfo, ti_info_list.first) { - CV_TypeIndex *sub_ti_ptr = str8_deserial_get_raw_ptr(leaf.data, sub_ti_n->offset, sizeof(*sub_ti_ptr)); + for (U64 ti_idx = 0; ti_idx < ti_count; ti_idx += 1) { + CV_TiOff sub_ti_n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *sub_ti_ptr = str8_deserial_get_raw_ptr(leaf.data, sub_ti_n.offset, sizeof(*sub_ti_ptr)); CV_TypeIndex sub_ti = memory_read32(sub_ti_ptr); - - // simple indices are stable across compile units - if (sub_ti < debug_t->ti_ranges[sub_ti_n->source].min) { + + // simple indices are stable across compile units + if (sub_ti < debug_t->ti_ranges[sub_ti_n.source].min) { lnk_hasher_update_struct(&hasher, &sub_ti); continue; } - if (sub_ti >= debug_t->ti_ranges[sub_ti_n->source].max) { - // discard type - U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; - U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + if (sub_ti >= debug_t->ti_ranges[sub_ti_n.source].max) { + // discard type: journal the NOTYPE rewrite for view-backed real objs (raw input pages + // stay clean); pseudo objs keep the in-place write on their arena-backed copy + U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; + if (obj_idx < input->obj_count) { + lnk_notype_journal_push(journal_arena, &input->notype_journal[obj_idx], leaf_idx, 0, debug_t->count); + } else { + U8 *leaf_header = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + } + if (leaf_idx == lnk_leaf_ref_leaf_idx(leaf_ref)) { self_discarded = 1; } // reset hasher lnk_hasher_init(&hasher, input->config->debug_types_hash); @@ -1330,7 +2993,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // log error Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); - String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) out of bounds type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); + String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) out of bounds type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, (U64)sub_ti_n.offset); lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[obj_idx], "%S", error_msg); scratch_end(scratch); @@ -1340,11 +3003,16 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // discard type with a cyclic-ref B32 is_type_graph_cyclic = discard_cycles && sub_ti > 0 && sub_ti > curr_ti; if (is_type_graph_cyclic) { - // discard type - U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; - U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + // discard type (journal for real objs, in-place for pseudo -- see the invalid-TI branch) + U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; + if (obj_idx < input->obj_count) { + lnk_notype_journal_push(journal_arena, &input->notype_journal[obj_idx], leaf_idx, 0, debug_t->count); + } else { + U8 *leaf_header = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + } + if (leaf_idx == lnk_leaf_ref_leaf_idx(leaf_ref)) { self_discarded = 1; } // reset hasher lnk_hasher_init(&hasher, input->config->debug_types_hash); @@ -1352,7 +3020,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // log error Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); - String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) forward refs member type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); + String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) forward refs member type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, (U64)sub_ti_n.offset); lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[obj_idx], "%S", error_msg); scratch_end(scratch); @@ -1360,16 +3028,34 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf } // type index -> hash - LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, obj_idx, sub_ti_n->source, sub_ti); + LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, obj_idx, sub_ti_n.source, sub_ti); U64 sub_hash = input->debug_h_arr[lnk_leaf_ref_obj_idx(sub_ref)].v[lnk_leaf_ref_leaf_idx(sub_ref)]; // mix-in sub-type hash lnk_hasher_update_struct(&hasher, &sub_hash); } - // hash leaf header - CV_LeafHeader *leaf_header_ptr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); - lnk_hasher_update_struct(&hasher, leaf_header_ptr); + // hash leaf header. Hot path = the ORIGINAL raw pointer read; only journaled leaves (whose + // raw header is unpatched) and self-discards (whose rewrite went to the journal for real + // objs) reconstruct the header ({ size = data.size + sizeof(kind), kind } is byte-identical + // to what the old post-in-place-write read produced) + if (self_discarded) { + CV_LeafHeader leaf_header = { .size = sizeof(CV_LeafKind), .kind = CV_LeafKind_NOTYPE }; + lnk_hasher_update_struct(&hasher, &leaf_header); + } else if (obj_idx < input->obj_count && + lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { + CV_LeafHeader leaf_header = { .size = (CV_LeafSize)(leaf.data.size + sizeof(CV_LeafKind)), .kind = leaf.kind }; + lnk_hasher_update_struct(&hasher, &leaf_header); + } else if (debug_t->sidecar_sizes || debug_t->sidecar_packed) { + // The sparse raw view deliberately leaves .debug$T bodies (including the + // four-byte leaf header) as holes. The body above came from the bounded + // decode window; reconstruct the byte-identical header from its sidecar + // instead of mixing four zero bytes from the hole into the type hash. + CV_LeafHeader leaf_header = { .size = (CV_LeafSize)(cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx) - sizeof(CV_LeafSize)), .kind = leaf.kind }; + lnk_hasher_update_struct(&hasher, &leaf_header); + } else { + lnk_hasher_update_struct(&hasher, cv_debug_t_get_leaf_header(debug_t, leaf_idx)); + } // finalize the type hash U64 hash = lnk_hasher_digest64(&hasher); @@ -1384,17 +3070,19 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf internal void lnk_hash_cv_leaf_deep(Arena *arena, + LNK_CObjDecodeWindow *decode_window, LNK_CodeViewInput *input, LNK_LeafRef root_leaf_ref, - CV_TypeIndexInfoList root_ti_info_list) + CV_TiOffsets root_ti_offs) { Temp temp = temp_begin(arena); typedef struct HashStack { struct HashStack *next; LNK_LeafRef leaf_ref; - CV_TypeIndexInfoList ti_info_list; - CV_TypeIndexInfo *ti_info; + CV_TiOffsets ti_offs; + U64 ti_next; + U64 ti_count; CV_Leaf leaf; CV_TypeIndex ti; CV_TypeIndexSource ti_source; @@ -1405,29 +3093,30 @@ lnk_hash_cv_leaf_deep(Arena *arena, CV_DebugT *root_debug_t = &input->debug_t_arr[root_obj_idx]; HashStack *root_frame = push_array(temp.arena, HashStack, 1); root_frame->leaf_ref = root_leaf_ref; - root_frame->ti_info_list = root_ti_info_list; - root_frame->ti_info = root_ti_info_list.first; - root_frame->leaf = cv_debug_t_get_leaf(root_debug_t, lnk_leaf_ref_leaf_idx(root_leaf_ref)); + root_frame->ti_offs = root_ti_offs; + root_frame->ti_next = 0; + root_frame->ti_count = cv_ti_offsets_count(&root_ti_offs); + root_frame->leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, decode_window, input, lnk_leaf_ref_obj_idx(root_leaf_ref), lnk_leaf_ref_leaf_idx(root_leaf_ref)); root_frame->ti_source = cv_type_index_source_from_leaf_kind(root_frame->leaf.kind); root_frame->ti = cv_ti_from_leaf_idx(root_debug_t, root_frame->ti_source, lnk_leaf_ref_leaf_idx(root_leaf_ref)); HashStack *stack = root_frame; while (stack) { - while (stack->ti_info) { - CV_TypeIndexInfo *ti_info = stack->ti_info; + while (stack->ti_next < stack->ti_count) { + CV_TiOff ti_info = cv_ti_offset_at(&stack->ti_offs, stack->ti_next); // advance iterator - stack->ti_info = stack->ti_info->next; + stack->ti_next += 1; // get type index info - CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(stack->leaf.data, ti_info->offset, sizeof(*ti_ptr)); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(stack->leaf.data, ti_info.offset, sizeof(*ti_ptr)); CV_TypeIndex ti = memory_read32(ti_ptr); // skip out of bounds indices - if ( ! contains_1u64(input->debug_t_arr[root_obj_idx].ti_ranges[ti_info->source], ti)) { continue; } + if ( ! contains_1u64(input->debug_t_arr[root_obj_idx].ti_ranges[ti_info.source], ti)) { continue; } // skip hashed types - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_obj_idx, ti_info->source, ti); + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_obj_idx, ti_info.source, ti); U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); if (input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } @@ -1436,18 +3125,21 @@ lnk_hash_cv_leaf_deep(Arena *arena, // recurse down to sub types HashStack *frame = push_array(temp.arena, HashStack, 1); frame->leaf_ref = leaf_ref; - frame->leaf = cv_debug_t_get_leaf(&input->debug_t_arr[obj_idx], leaf_idx); - frame->ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); - frame->ti_info = frame->ti_info_list.first; + frame->leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, decode_window, input, obj_idx, leaf_idx); + frame->ti_offs = cv_leaf_ti_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); + frame->ti_next = 0; + frame->ti_count = cv_ti_offsets_count(&frame->ti_offs); frame->ti = ti; - frame->ti_source = ti_info->source; + frame->ti_source = ti_info.source; SLLStackPush(stack, frame); break; } // no more type indices, pop frame - if ( ! stack->ti_info) { - lnk_hash_cv_leaf(input, stack->leaf_ref, stack->ti_info_list, 0); + if (stack->ti_next >= stack->ti_count) { + // deep hashing only runs on pseudo objs (type servers / .ifc blobs) and never leaves the + // root obj, so no journal arena is needed (in-place rewrite path) + lnk_hash_cv_leaf(input, 0, stack->leaf_ref, stack->leaf, stack->ti_offs, 0); SLLStackPop(stack); } } @@ -1455,17 +3147,42 @@ lnk_hash_cv_leaf_deep(Arena *arena, temp_end(temp); } +// Map a uniformly distributed 64-bit hash into [0, cap) with one multiply-high. Unlike `% cap`, +// this does not require a runtime integer division; hash tables only require a stable uniform +// mapping, not the remainder specifically. +force_inline U64 +lnk_hash_range(U64 hash, U64 cap) +{ +#if COMPILER_MSVC && ARCH_X64 + U64 high; + _umul128(hash, cap, &high); + return high; +#elif (COMPILER_CLANG || COMPILER_GCC) && ARCH_64BIT + return (U64)(((__uint128_t)hash * (__uint128_t)cap) >> 64); +#else + U64 hash_lo = (U32)hash, hash_hi = hash >> 32; + U64 cap_lo = (U32)cap, cap_hi = cap >> 32; + U64 p00 = hash_lo * cap_lo; + U64 p01 = hash_lo * cap_hi; + U64 p10 = hash_hi * cap_lo; + U64 p11 = hash_hi * cap_hi; + U64 carry = ((p00 >> 32) + (U32)p01 + (U32)p10) >> 32; + return p11 + (p01 >> 32) + (p10 >> 32) + carry; +#endif +} + internal CV_TypeIndex lnk_assigned_ti_hash_search(LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { CV_DebugH *debug_h = &input->debug_h_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; U64 hash = debug_h->v[lnk_leaf_ref_leaf_idx(leaf_ref)]; - U64 best_idx = hash % ht->cap; + U64 best_idx = lnk_hash_range(hash, ht->cap); U64 idx = best_idx; do { - CV_TypeIndex ti = ht->ti_arr[idx]; + U8 *entry = ht->v + idx * LNK_ASSIGNED_TI_ENTRY_SIZE; + CV_TypeIndex ti = memory_read32(entry + LNK_ASSIGNED_TI_TI_OFF); if (ti == 0) { break; } - if (ht->hash_arr[idx] == hash) { return ti; } + if (memory_read64(entry + LNK_ASSIGNED_TI_HASH_OFF) == hash) { return ti; } idx = (idx + 1) == ht->cap ? 0 : (idx + 1); } while (idx != best_idx); @@ -1480,10 +3197,10 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_task) U32 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; for EachIndex(leaf_idx, debug_t->count) { - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf(task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_list, 1); + Temp temp = temp_begin(task->fixed_arenas[worker_id]); + CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, &task->decode_windows[worker_id], task->input, obj_idx, leaf_idx); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_hash_cv_leaf(task->input, arena, lnk_leaf_ref_make(obj_idx, leaf_idx), leaf, ti_offs, 1); temp_end(temp); } ProfEnd(); @@ -1496,17 +3213,72 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) LNK_MergeTypes *task = raw_task; U64 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); for EachIndex(leaf_idx, debug_t->count) { if (task->input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf_deep(temp.arena, task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_list); + if (is_ifc_blob && cv_debug_t_get_leaf_kind(debug_t, leaf_idx) == CV_LeafKind_NOTYPE) { continue; } // blob $T is arena-backed + mutated in place -- raw read is post-rewrite + Temp temp = temp_begin(task->fixed_arenas[worker_id]); + CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, &task->decode_windows[worker_id], task->input, obj_idx, leaf_idx); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_hash_cv_leaf_deep(temp.arena, &task->decode_windows[worker_id], task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_offs); temp_end(temp); } ProfEnd(); } +// Deterministic sampling for the unique-leaf estimator: process every K-th leaf POSITION per obj. +// Position-based (leaf_idx % K), never value-based, so the sampled set -- and therefore the +// estimate and the table caps -- is a pure function of the input, schedule-independent. The +// dominant duplication pattern is whole-stream duplication (the same PCH/type-server leaf sequence +// repeated across objs), where a unique hash sits at the SAME position in every copy: the sampled +// distinct count then scales ~1/K, which LNK_ESTIMATE_SAMPLE_SCALE compensates for. The scale is +// calibrated (see the estimate block in lnk_merge_types); an undershoot is caught by the existing +// deterministic overflow-retry at total-based caps, an overshoot is clamped by Min(fallback cap). +// +// SCALE calibration: sampled-distinct is between distinct (fully position-scattered duplication) +// and distinct/K (whole-stream duplication or unique-heavy input), so the true ratio is in [1, K]. +// SCALE * 1.9 (the downstream safety factor) must cover the worst-case ratio K to keep the +// overflow-retry off for every duplication pattern: SCALE = 5.0 gives 5.0*1.9 = 9.5 >= K = 8 +// (1.19x margin over the bound, which also absorbs linear-counting noise). Measured on the FN +// editor-scale link: ratio 3.99 (TPI) / 6.13 (IPI); SCALE = 5.0 reproduces the unsampled +// estimator's caps exactly (64M/16M) at load factors 0.35/0.39. +#define LNK_ESTIMATE_SAMPLE_STRIDE 8 +#define LNK_ESTIMATE_SAMPLE_SCALE 5.0 + +internal +THREAD_POOL_TASK_FUNC(lnk_estimate_unique_leaves_task) +{ + ProfBeginFunction(); + LNK_MergeTypes *task = raw_task; + U64 obj_idx = task->indices.v[task_id]; + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + CV_DebugH *debug_h = &task->input->debug_h_arr[obj_idx]; + // same prune rule as lnk_leaf_dedup_task: NOTYPE'd IFC blob leaves were never hashed and are + // never inserted, so they must not contribute to the estimate either + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); + // P4: raw views hold pre-rewrite bytes for journaled real-obj leaves -- overlay LF_NOTYPE + // via the per-obj journal bitmap (0 for ~all objs; one register test per leaf) + U64 *notype_bm = 0; + U64 notype_cap = 0; + if (obj_idx < task->input->obj_count) { + notype_bm = task->input->notype_journal[obj_idx].bitmap; + notype_cap = task->input->notype_journal[obj_idx].bit_cap; + } + for (U64 leaf_idx = 0; leaf_idx < debug_t->count; leaf_idx += LNK_ESTIMATE_SAMPLE_STRIDE) { + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); + if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } + if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } + CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); + U64 bit_idx = debug_h->v[leaf_idx] & (task->estimate_bitmap_bits[leaf_source] - 1); + U32 *word = &task->estimate_bitmap[leaf_source][bit_idx / 32]; + U32 bit = 1u << (bit_idx % 32); + // atomic OR is commutative -> final bitmap contents are schedule-independent (deterministic); + // pre-check skips the interlocked op for already-set bits (the common case on dup-heavy input) + if ((ins_atomic_u32_eval(word) & bit) == 0) { ins_atomic_u32_or(word, bit); } + } + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) { @@ -1517,29 +3289,44 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) CV_DebugH *debug_h = &task->input->debug_h_arr[task->pop_obj_idx]; for EachInRange(leaf_idx, task->pop_range[task_id]) { - - LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); - CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header - CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind + // another worker overflowed an estimate-sized table -- the whole dedup result is discarded + // and retried with the total-based caps, so bail out early + if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } + + LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); + B32 is_inserted_or_updated = 1; + + // pop obj is a type-server pseudo obj: arena-backed, mutated in place, never journaled -- + // the raw header read is the post-rewrite kind (original code path) + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table - U64 best_idx = debug_h->v[leaf_idx] % leaf_ht->cap; // leaf ref -> hash -> bucket index + U64 hash = debug_h->v[leaf_idx]; // leaf ref -> hash + U64 best_idx = hash & (leaf_ht->cap - 1); // hash -> bucket index + LNK_LeafRef tagged = lnk_leaf_bucket_tag(leaf_ref, hash); U64 idx = best_idx; do { - LNK_LeafRef curr = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); - if (curr == LNK_LEAF_REF_NULL) { - LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], leaf_ref, curr); - if (cmp == curr) { + LNK_LeafRef curr_tagged = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); + if (curr_tagged == LNK_LEAF_REF_NULL) { + LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], tagged, curr_tagged); + if (cmp == curr_tagged) { goto exit; } } // advance to next bucket - idx = ((idx + 1) == leaf_ht->cap ? 0 : (idx + 1)); + idx = (idx + 1) & (leaf_ht->cap - 1); } while (idx != best_idx); - InvalidPath; + is_inserted_or_updated = 0; exit:; + if (!is_inserted_or_updated && leaf_source != CV_TypeIndexSource_NULL) { + // TPI/IPI table is full (estimate undershot) -- flag for a deterministic retry with the + // total-based caps. the NULL-source table is deliberately undersized and silently drops + // leaves that do not fit (pre-existing behavior; they are never emitted). + ins_atomic_u32_eval_assign(&task->leaf_ht_overflow, 1); + break; + } } } @@ -1550,48 +3337,73 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) U64 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; CV_DebugH *debug_h = &task->input->debug_h_arr[obj_idx]; + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); ProfBeginDynamic("dedup in obj 0x%llx (%.*s) leaf count %llu", obj_idx, str8_varg(task->input->obj_arr[obj_idx]->path), debug_t->count); - for EachIndex(leaf_idx, debug_t->count) { + // P4: raw views hold pre-rewrite bytes for journaled real-obj leaves -- overlay LF_NOTYPE via + // the per-obj journal bitmap (0 for ~all objs). Blob objs are pseudo (in-place rewrites), so + // their skip below keeps the plain raw read. + U64 *notype_bm = 0; + U64 notype_cap = 0; + if (obj_idx < task->input->obj_count) { + notype_bm = task->input->notype_journal[obj_idx].bitmap; + notype_cap = task->input->notype_journal[obj_idx].bit_cap; + } + for EachIndex(leaf_idx, debug_t->count) { + // another worker overflowed an estimate-sized table -- the whole dedup result is discarded + // and retried with the total-based caps, so bail out early + if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } B32 is_inserted_or_updated = 1; LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); - CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header - CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); + if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } + if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table - U64 best_idx = debug_h->v[leaf_idx] % leaf_ht->cap; // leaf ref -> hash -> bucket index + U64 hash = debug_h->v[leaf_idx]; // leaf ref -> hash + U64 best_idx = hash & (leaf_ht->cap - 1); // hash -> bucket index + LNK_LeafRef tagged = lnk_leaf_bucket_tag(leaf_ref, hash); U64 idx = best_idx; do { // load leaf ref - LNK_LeafRef curr = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); + LNK_LeafRef curr_tagged = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); - while (curr == LNK_LEAF_REF_NULL || lnk_match_leaf_ref(task->input, curr, leaf_ref)) { + while (curr_tagged == LNK_LEAF_REF_NULL || + ((curr_tagged & LNK_LEAF_BUCKET_TAG_MASK) == (tagged & LNK_LEAF_BUCKET_TAG_MASK) && + lnk_hash_from_leaf_ref(task->input, lnk_leaf_bucket_untag(curr_tagged)) == hash)) { + LNK_LeafRef curr = curr_tagged == LNK_LEAF_REF_NULL ? LNK_LEAF_REF_NULL : lnk_leaf_bucket_untag(curr_tagged); // exit if leaf ref is not recent if (curr != LNK_LEAF_REF_NULL && lnk_leaf_ref_compare(leaf_ref, curr) >= 0) { goto exit; } // try to update the bucket - LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], leaf_ref, curr); - if (cmp == curr) { + LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], tagged, curr_tagged); + if (cmp == curr_tagged) { goto exit; } // another thread updated the bucket -- retry - curr = cmp; + curr_tagged = cmp; } // advance to next bucket - idx = ((idx + 1) == leaf_ht->cap ? 0 : (idx + 1)); + idx = (idx + 1) & (leaf_ht->cap - 1); } while (idx != best_idx); is_inserted_or_updated = 0; exit:; - Assert(is_inserted_or_updated); + if (!is_inserted_or_updated && leaf_source != CV_TypeIndexSource_NULL) { + // TPI/IPI table is full (estimate undershot) -- flag for a deterministic retry with the + // total-based caps. the NULL-source table is deliberately undersized and silently drops + // leaves that do not fit (pre-existing behavior; they are never emitted). + ins_atomic_u32_eval_assign(&task->leaf_ht_overflow, 1); + break; + } } ProfEnd(); @@ -1627,7 +3439,35 @@ THREAD_POOL_TASK_FUNC(lnk_get_present_buckets_task) for EachInRange(bucket_idx, task->ranges[task_id]) { if (ht->bucket_arr[bucket_idx] != LNK_LEAF_REF_NULL) { - unique_leaf_refs.v[cursor++] = ht->bucket_arr[bucket_idx]; + LNK_LeafRef ref = lnk_leaf_bucket_untag(ht->bucket_arr[bucket_idx]); + unique_leaf_refs.v[cursor++] = ref; + + // TPI/IPI materialization is the next compressed-payload consumer. Claim each segment + // once while the winner is already hot in this extraction pass. A leaf can straddle a + // segment boundary, so claim the full raw range rather than only its first byte. + if (task->winner_segment_bitmap && task->ti_source != CV_TypeIndexSource_NULL && + lnk_leaf_ref_obj_idx(ref) < task->input->obj_count) { + U32 obj_idx = lnk_leaf_ref_obj_idx(ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(ref); + LNK_Obj *obj = task->input->obj_arr[obj_idx]; + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + U32 segment_size = lnk_compressed_obj_segment_size(obj->compressed_obj); + if (segment_size && (debug_t->sidecar_sizes || debug_t->sidecar_packed)) { + U64 raw_min = debug_t->sidecar_raw_base + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + U64 raw_max = raw_min + cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + U32 first = (U32)(raw_min / segment_size); + U32 opl = (U32)CeilIntegerDiv(raw_max, segment_size); + U32 count = lnk_compressed_obj_segment_count(obj->compressed_obj); + opl = Min(opl, count); + for (U32 segment_idx = first; segment_idx < opl; ++segment_idx) { + U64 global_idx = task->winner_segment_offsets[obj_idx] + segment_idx; + U64 *word = &task->winner_segment_worker_bitmaps[task_id * task->winner_segment_word_count + + (global_idx >> 6)]; + U64 bit = (U64)1 << (global_idx & 63); + *word |= bit; + } + } + } } } @@ -1650,16 +3490,18 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) CV_TypeIndex type_index = min_type_index + i; U64 hash = debug_h_arr[lnk_leaf_ref_obj_idx(leaf_ref)].v[lnk_leaf_ref_leaf_idx(leaf_ref)]; - U64 best_idx = hash % assigned->cap; + U64 best_idx = lnk_hash_range(hash, assigned->cap); U64 idx = best_idx; B32 is_inserted = 0; do { - CV_TypeIndex curr_type_index = assigned->ti_arr[idx]; + U8 *entry = assigned->v + idx * LNK_ASSIGNED_TI_ENTRY_SIZE; + CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(entry + LNK_ASSIGNED_TI_TI_OFF); + CV_TypeIndex curr_type_index = *ti_ptr; if (curr_type_index == 0) { - CV_TypeIndex cmp_type_index = ins_atomic_u32_eval_cond_assign(&assigned->ti_arr[idx], type_index, curr_type_index); + CV_TypeIndex cmp_type_index = ins_atomic_u32_eval_cond_assign(ti_ptr, type_index, curr_type_index); if (cmp_type_index == curr_type_index) { - assigned->hash_arr[idx] = hash; + memory_write64(entry + LNK_ASSIGNED_TI_HASH_OFF, hash); is_inserted = 1; break; } @@ -1672,17 +3514,18 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) } internal void -lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TypeIndexInfoList ti_info_list) +lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TiOffsets ti_offs) { - for EachNode(n, CV_TypeIndexInfo, ti_info_list.first) { - CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n->offset, sizeof(*ti_ptr)); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n.offset, sizeof(*ti_ptr)); CV_TypeIndex ti = memory_read32(ti_ptr); // skip basic types - if (ti < ctx->input->min_type_indices[n->source]) { continue; } + if (ti < ctx->input->min_type_indices[n.source]) { continue; } - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); - CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n->source], ctx->input, leaf_ref); + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n.source, ti); + CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n.source], ctx->input, leaf_ref); memory_write32(ti_ptr, final_ti); #if LNK_PARANOID @@ -1693,77 +3536,650 @@ lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_Typ } } +// Streaming-ring P2 slice A: upper-bound count of journal entries the ti_offs walk emits. +// Exact for the TI-fixup part -- an entry is emitted iff the raw TI is >= the source's min +// (the "skip basic types" test), which needs no merge-state lookups. +internal U64 +lnk_count_cv_type_index_fixups(LNK_MergeTypes *ctx, String8 data, CV_TiOffsets ti_offs) +{ + U64 count = 0; + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n.offset, sizeof(*ti_ptr)); + if (memory_read32(ti_ptr) >= ctx->input->min_type_indices[n.source]) { count += 1; } + } + return count; +} + +// 8B/16B entry emit + decode helpers. `off` is relative to the run's node base; narrow runs +// are chosen up front (node size < 2GiB guarantees every off fits off:31), so the narrow +// branch never truncates. +force_inline void +lnk_debug_s_patch_emit(LNK_DebugSPatchArray *journal, U64 off, U32 value, U32 size) +{ + if (journal->is_wide) { + ((LNK_DebugSPatchWide *)journal->v)[journal->count++] = (LNK_DebugSPatchWide){ .off = off, .value = value, .size = size }; + } else { + Assert(off < (1ull << 31)); + ((LNK_DebugSPatch *)journal->v)[journal->count++] = (LNK_DebugSPatch){ .off_w = (U32)((off << 1) | (size == 4 ? 1 : 0)), .value = value }; + } +} + +force_inline U64 +lnk_debug_s_patch_off_at(LNK_DebugSPatchArray *journal, U64 k) +{ + return journal->is_wide ? ((LNK_DebugSPatchWide *)journal->v)[k].off + : (U64)(((LNK_DebugSPatch *)journal->v)[k].off_w >> 1); +} + +force_inline U32 +lnk_debug_s_patch_value_at(LNK_DebugSPatchArray *journal, U64 k) +{ + return journal->is_wide ? ((LNK_DebugSPatchWide *)journal->v)[k].value + : ((LNK_DebugSPatch *)journal->v)[k].value; +} + +// journal-emitting twin of lnk_fixup_cv_type_indices: identical TI resolution (assigned-TI +// hash search over the merge result), but the write is RECORDED instead of applied -- the $S +// bytes stay pre-fixup until the per-obj replay at module write. `base` = the run's node base +// (entries store node-relative offsets). +internal void +lnk_journal_cv_type_index_fixups(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TiOffsets ti_offs, LNK_DebugSPatchArray *journal, U8 *base) +{ + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n.offset, sizeof(*ti_ptr)); + CV_TypeIndex ti = memory_read32(ti_ptr); + + // skip basic types + if (ti < ctx->input->min_type_indices[n.source]) { continue; } + + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n.source, ti); + CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n.source], ctx->input, leaf_ref); + lnk_debug_s_patch_emit(journal, (U64)((U8 *)ti_ptr - base), final_ti, 4); + +#if LNK_PARANOID + if (final_ti == 0) { + lnk_error_obj(LNK_Error_InvalidTypeIndex, ctx->input->obj_arr[obj_idx], "no itype 0x%x", ti); + } +#endif + } +} + +// Fuses the old lnk_cv_patcher_symbols_task (symbol-record TI fixup) and lnk_fixup_symbols_task +// (*_ID kind rewrite + itype -> FUNC_ID/MFUNC_ID itype resolve) into one journal-building walk +// per symbol input. Runs AFTER the materialize pass so the itype resolve reads the fixed-up +// merged IPI leaf copies -- exactly what the old standalone pass consumed. The old second pass +// read proc32->itype back from memory AFTER the first pass patched it; here that value is the +// resolved TI recorded for the record's itype slot (raw bytes when the slot wasn't journaled: +// basic types, MIPS/IA64 kinds cv_symbol_ti_offsets has no entry for). Emitting BOTH itype +// entries in order (resolved IPI TI, then the FUNC_ID itype) makes the sequential replay land +// on the same end-state bytes for every branch, including the early-out edge cases. +internal +THREAD_POOL_TASK_FUNC(lnk_journal_symbol_fixups_task) +{ + ProfBeginFunction(); + LNK_MergeTypes *task = raw_task; + + Arena *journal_arena = task->journal_arena->v[worker_id]; + U64 leaf_count_ipi = task->result.count [CV_TypeIndexSource_IPI]; + U8 **leaf_arr_ipi = task->result.v [CV_TypeIndexSource_IPI]; + CV_TypeIndex min_ti_ipi = task->min_type_indices[CV_TypeIndexSource_IPI]; + + Rng1U64 range = task->input->symbol_patch_task[task_id].input_range; + for EachInRange(i, range) { + LNK_SymbolInput symbols = task->input->symbol_inputs[i]; + + // The journal builder makes two immediate linear passes over each Symbols subsection. + // Keeping those reads behind the shared faulting view lets concurrently scheduled objs + // evict one another between passes. A subsection-local copy streams every compressed + // segment once through the worker's decode window, then both parser passes hit ordinary + // committed memory. The fixed arena temp is rewound for every input, so peak storage is + // bounded by one Symbols subsection per active worker rather than the corpus size. + Temp symbols_temp = temp_begin(task->fixed_arenas[worker_id]); + LNK_Obj *obj = task->input->obj_arr[symbols.obj_idx]; + if (obj->compressed_obj && symbols.raw_symbols.size) { + U8 *copy = push_array_no_zero(symbols_temp.arena, U8, symbols.raw_symbols.size); + if (lnk_compressed_obj_copy_string(obj->compressed_obj, symbols.raw_symbols, copy, + &task->decode_windows[worker_id])) { + symbols.raw_symbols = str8(copy, symbols.raw_symbols.size); + } + } + + // upper-bound pass: TI entries (exact) + kind rewrite / itype slots for *_ID records + U64 cap = 0; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + cap += lnk_count_cv_type_index_fixups(task, symbol.data, cv_symbol_ti_offsets(symbol.kind, symbol.data)); + switch (symbol.kind) { + case CV_SymKind_PROC_ID_END: cap += 1; break; + case CV_SymKind_LPROC32_ID: + case CV_SymKind_GPROC32_ID: + case CV_SymKind_LPROC32_DPC_ID: + case CV_SymKind_LPROCMIPS_ID: + case CV_SymKind_GPROCMIPS_ID: + case CV_SymKind_LPROCIA64_ID: + case CV_SymKind_GPROCIA64_ID: cap += 2; break; + default: break; + } + } + + U8 *base = symbols.raw_symbols.str; + + LNK_DebugSPatchArray *journal = &task->input->debug_s_sym_fixups[i]; + journal->is_wide = (symbols.raw_symbols.size >> 31) != 0; // narrow off:31 covers the whole node otherwise + journal->v = journal->is_wide ? (void *)push_array_no_zero(journal_arena, LNK_DebugSPatchWide, cap) + : (void *)push_array_no_zero(journal_arena, LNK_DebugSPatch, cap); + journal->count = 0; + + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + + CV_TiOffsets ti_offs = cv_symbol_ti_offsets(symbol.kind, symbol.data); + U64 rec_journal_start = journal->count; + lnk_journal_cv_type_index_fixups(task, symbols.obj_idx, symbol.data, ti_offs, journal, base); + + // convert symbol to final type + CV_SymKind *sym_kind_ptr = cv_kind_ptr_from_symbol(symbol); + CV_SymKind new_kind = CV_SymKind_END; + switch (symbol.kind) { + case CV_SymKind_PROC_ID_END: { + lnk_debug_s_patch_emit(journal, (U64)((U8 *)sym_kind_ptr - base), CV_SymKind_END, 2); + } break; + + case CV_SymKind_LPROC32_ID: new_kind = CV_SymKind_LPROC32; goto fixup_id; + case CV_SymKind_GPROC32_ID: new_kind = CV_SymKind_GPROC32; goto fixup_id; + case CV_SymKind_LPROC32_DPC_ID: new_kind = CV_SymKind_LPROC32_DPC; goto fixup_id; + case CV_SymKind_LPROCMIPS_ID: new_kind = CV_SymKind_LPROCMIPS; goto fixup_id; + case CV_SymKind_GPROCMIPS_ID: new_kind = CV_SymKind_GPROCMIPS; goto fixup_id; + case CV_SymKind_LPROCIA64_ID: new_kind = CV_SymKind_LPROCIA64; goto fixup_id; + case CV_SymKind_GPROCIA64_ID: new_kind = CV_SymKind_GPROCIA64; goto fixup_id; + fixup_id:; { + lnk_debug_s_patch_emit(journal, (U64)((U8 *)sym_kind_ptr - base), new_kind, 2); + + CV_SymProc32 *proc32 = str8_deserial_get_raw_ptr(symbol.data, 0, sizeof(*proc32)); + + // effective post-TI-fixup itype (what the old pass read back from patched memory) + U64 itype_off = (U64)((U8 *)&proc32->itype - base); + CV_TypeIndex itype = proc32->itype; + for (U64 k = rec_journal_start; k < journal->count; k += 1) { + if (lnk_debug_s_patch_off_at(journal, k) == itype_off) { itype = lnk_debug_s_patch_value_at(journal, k); break; } + } + + if (itype < min_ti_ipi) { + // TODO: in some cases destructors don't have a type, need a repro + break; + } + + if ((itype - min_ti_ipi) > leaf_count_ipi) { + Assert(0 && "TODO: error handle corrupted type index"); + break; + } + + U64 leaf_idx = itype - min_ti_ipi; + String8 leaf_data = str8(leaf_arr_ipi[leaf_idx], max_U64); + + CV_Leaf leaf; + if (cv_read_leaf(leaf_data, 0, 1, &leaf) == 0) { InvalidPath; } + + U64 min_leaf_size = cv_header_struct_size_from_leaf_kind(leaf.kind); + if (min_leaf_size > leaf.data.size) { Assert(!"TODO: error handle corrupt leaf"); break; } + + if (leaf.kind == CV_LeafKind_FUNC_ID) { + CV_LeafFuncId *func_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*func_id)); + lnk_debug_s_patch_emit(journal, itype_off, func_id->itype, 4); + } else if (leaf.kind == CV_LeafKind_MFUNC_ID) { + CV_LeafMFuncId *mfunc_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*mfunc_id)); + lnk_debug_s_patch_emit(journal, itype_off, mfunc_id->itype, 4); + } else { + Assert(!"TODO: erorr handle unexpected leaf type"); + break; + } + } break; + + default: break; + } + } + Assert(journal->count <= cap); + temp_end(symbols_temp); + } + ProfEnd(); +} + +// journal-building replacement for the old lnk_cv_patcher_inlines_task (per obj) +internal +THREAD_POOL_TASK_FUNC(lnk_journal_inline_fixups_task) +{ + ProfBeginFunction(); + LNK_MergeTypes *task = raw_task; + U64 obj_idx = task_id; + String8List inlinee_lines = cv_sub_section_from_debug_s(task->input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + Arena *fixed_arena = task->fixed_arenas[worker_id]; + Arena *journal_arena = task->journal_arena->v[worker_id]; + + // exact count; a node >= 2GiB forces the obj's whole run set to wide entries + U64 cap = 0; + B32 is_wide = 0; + for EachNode(inline_data_n, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(fixed_arena); + CV_TiOffsets ti_offs = cv_inlinee_ti_offsets(temp.arena, inline_data_n->string); + cap += lnk_count_cv_type_index_fixups(task, inline_data_n->string, ti_offs); + is_wide |= (inline_data_n->string.size >> 31) != 0; + temp_end(temp); + } + + LNK_DebugSInlineJournal *journal = &task->input->debug_s_inline_fixups[obj_idx]; + journal->patches.is_wide = is_wide; + journal->patches.v = is_wide ? (void *)push_array_no_zero(journal_arena, LNK_DebugSPatchWide, cap) + : (void *)push_array_no_zero(journal_arena, LNK_DebugSPatch, cap); + journal->patches.count = 0; + journal->node_counts = push_array_no_zero(journal_arena, U32, inlinee_lines.node_count ? inlinee_lines.node_count : 1); + + U64 node_idx = 0; + for EachNode(inline_data_n, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(fixed_arena); + U64 run_start = journal->patches.count; + CV_TiOffsets ti_offs = cv_inlinee_ti_offsets(temp.arena, inline_data_n->string); + lnk_journal_cv_type_index_fixups(task, obj_idx, inline_data_n->string, ti_offs, &journal->patches, inline_data_n->string.str); + journal->node_counts[node_idx++] = (U32)(journal->patches.count - run_start); + temp_end(temp); + } + Assert(journal->patches.count == cap); + ProfEnd(); +} + +internal void +lnk_apply_debug_s_patch_run(U8 *base, LNK_DebugSPatchArray *arr, U64 lo, U64 opl) +{ + if (arr->is_wide) { + LNK_DebugSPatchWide *v = arr->v; + for (U64 k = lo; k < opl; k += 1) { + if (v[k].size == 4) { memory_write32(base + v[k].off, v[k].value); } + else { memory_write16(base + v[k].off, (U16)v[k].value); } + } + } else { + LNK_DebugSPatch *v = arr->v; + for (U64 k = lo; k < opl; k += 1) { + U8 *ptr = base + (v[k].off_w >> 1); + if (v[k].off_w & 1) { memory_write32(ptr, v[k].value); } + else { memory_write16(ptr, (U16)v[k].value); } + } + } +} + +// Replays the obj's deferred $S TI/kind fixups (journaled in lnk_merge_types). Entries write +// only into the obj's own $S backing bytes (its patched section copies / raw-mapped sections; +// $S is never shared across objs, unlike $T PCH refs), so this is safe inside any per-obj +// parallel loop and produces the same bytes regardless of schedule. Write order == journal +// emission order: symbol inputs in index order (= Symbols data_list node order), then the +// InlineeLines node runs in data_list order. +internal void +lnk_apply_debug_s_fixups_for_obj(LNK_CodeViewInput *cv, U64 obj_idx) +{ + if (!cv->has_debug_s_fixup_journal) { return; } + + for (U64 i = cv->debug_s_sym_fixup_offsets[obj_idx], opl = cv->debug_s_sym_fixup_offsets[obj_idx+1]; i < opl; i += 1) { + Assert(cv->symbol_inputs[i].obj_idx == obj_idx); + lnk_apply_debug_s_patch_run(cv->symbol_inputs[i].raw_symbols.str, &cv->debug_s_sym_fixups[i], 0, cv->debug_s_sym_fixups[i].count); + } + + LNK_DebugSInlineJournal *inline_journal = &cv->debug_s_inline_fixups[obj_idx]; + if (inline_journal->patches.count > 0) { + String8List inlinee_lines = cv_sub_section_from_debug_s(cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + U64 node_idx = 0, cursor = 0; + for EachNode(inline_data_n, String8Node, inlinee_lines.first) { + U64 run = inline_journal->node_counts[node_idx++]; + lnk_apply_debug_s_patch_run(inline_data_n->string.str, &inline_journal->patches, cursor, cursor + run); + cursor += run; + } + Assert(cursor == inline_journal->patches.count); + } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_apply_debug_s_fixups_task) +{ + LNK_CodeViewInput *cv = raw_task; + lnk_apply_debug_s_fixups_for_obj(cv, task_id); +} + +// Eager whole-input replay for configs that consume fixed-up $S bytes before (or without) +// the module-write pass: /OPT:GCTYPES reads + rewrites $S type indices right after the merge, +// and a /PDBSTRIPPED-only build re-walks $S without ever writing modules. Consumes the +// journal -- the module-write replay is skipped afterwards. +// Consumes + releases the journal: drops every reference (entry arrays, per-input/per-obj +// headers, offsets table all live inside the DEBUG_S_FIXUP_JOURNAL arenas), then hands the +// arena set to the background reaper. Idempotent; no-op when the journal was never built +// (SkipSymbolTypeFixup / stripped cv) or already consumed. +internal void +lnk_release_debug_s_fixup_journal(LNK_CodeViewInput *cv) +{ + if (cv->debug_s_fixup_journal_arenas == 0) { return; } + cv->debug_s_sym_fixups = 0; + cv->debug_s_inline_fixups = 0; + cv->debug_s_sym_fixup_offsets = 0; + cv->has_debug_s_fixup_journal = 0; + if (g_arena_reaper_thread.u64[0] != 0) { thread_join(g_arena_reaper_thread, max_U64); } + g_arena_reaper_thread = thread_launch(lnk_tp_arena_release_thread, cv->debug_s_fixup_journal_arenas); + cv->debug_s_fixup_journal_arenas = 0; +} + +internal void +lnk_apply_debug_s_fixups_eager(TP_Context *tp, LNK_CodeViewInput *cv) +{ + if (!cv->has_debug_s_fixup_journal) { return; } + ProfBegin("Apply $S Fixups (eager)"); + tp_for_parallel(tp, 0, cv->obj_count, lnk_apply_debug_s_fixups_task, cv); + lnk_release_debug_s_fixup_journal(cv); // consumed -- module write skips replay AND release + ProfEnd(); +} + +// high-water mark of a single obj's windowed $S bytes (telemetry: bounds the per-worker +// window arena growth; reported under /RAD_LOG:Debug at the end of Write Modules) +global U64 g_debug_s_window_hwm = 0; + +// ===== Streaming-ring P3.3: the window ========================================================== +// Materializes an obj's parsed .debug$S subsections as CONSUMABLE bytes in `arena` (a per-worker +// scratch reset per obj -- the "window"), without ever writing to the raw mapped input views and +// without any persistent patched copy: +// - every source section a provenance record references is either aliased RAW (no relocs, no +// in-window mutation needed) or COPIED into the window and reloc-patched there via +// lnk_obj_apply_relocs_to_buffer (the exact routine the image-build patcher uses -- relocs, +// symbol tables and the image section table are immutable by module-write time, so the bytes +// are identical to the retired image-time patch-on-copy); +// - Symbols/InlineeLines sections are always windowed while the $S fixup journal is alive +// (replay writes into the window, never a mapped view), and the obj's journal runs replay +// here in build order -- entries are absolute writes, so every re-fill of the window replays +// to identical bytes (relocs are RMW but always start from the immutable raw addend); +// - synthetic / untracked nodes alias their existing linker-made bytes. +// The returned CV_DebugS carries data_list only (no provenance -- parity walks run against the +// original struct). `symbols_only` limits the fill to the Symbols subsection (epilogue dedup +// re-reads, /PDBSTRIPPED pre-pass). Callers own the arena lifetime; nothing in the result may +// outlive it except aliased raw/synthetic node bytes. +internal CV_DebugS +lnk_obj_window_debug_s(Arena *arena, LNK_CodeViewInput *cv, U64 obj_idx, U64 image_base, COFF_SectionHeader **image_section_table, B32 symbols_only) +{ + LNK_Obj *obj = cv->obj_arr[obj_idx]; + CV_DebugS *src = &cv->debug_s_arr[obj_idx]; + CV_DebugS out = {0}; + LNK_CObjDecodeWindow decode_window = {0}; + + U64 idx_symbols = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_Symbols); + U64 idx_inlinees = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_InlineeLines); + + // decide per-section representation from the sections the prov records reference + // (0 = unused, 1 = alias raw view, 2 = window copy + relocs) + U64 sect_count = obj->coff.sections.count_no_null; + U8 *sect_state = push_array(arena, U8, sect_count); + U8 **sect_base = push_array(arena, U8 *, sect_count); + for EachElement(k, src->data_list) { + if (symbols_only && k != idx_symbols) { continue; } + if (src->prov_list[k].count == 0) { continue; } // untracked: nodes alias as-is below + B32 is_journal_kind = (k == idx_symbols || k == idx_inlinees) && cv->has_debug_s_fixup_journal; + for (CV_DebugSProvNode *prov = src->prov_list[k].first; prov != 0; prov = prov->next) { + if (prov->is_synthetic || prov->sect_idx == CV_DebugSProvSect_Nil) { continue; } + U8 state = 1; + if (is_journal_kind) { + state = 2; + } else { + U64 section_number = (U64)prov->sect_idx + 1; + COFF_SectionHeader *hdr = &obj->coff.sections.headers[section_number]; + if (lnk_coff_relocs_from_section_header(obj, hdr).count > 0) { state = 2; } + } + if (state > sect_state[prov->sect_idx]) { sect_state[prov->sect_idx] = state; } + } + } + + // materialize window sections: raw memcpy + reloc application + COFF_SectionHeader *raw_section_table = (COFF_SectionHeader *)str8_substr(obj->coff.data, obj->coff.header.section_table_range).str; + U64 window_size = 0; + for EachIndex(sect_idx, sect_count) { + if (sect_state[sect_idx] == 0) { continue; } + U64 section_number = sect_idx + 1; + LNK_ObjSection section = lnk_obj_section_from_section_number(obj, section_number); + String8 raw = lnk_compressed_obj_direct_range(obj->compressed_obj, section.frange); + B32 explicitly_decoded = 0; + if (raw.size == 0 && obj->compressed_obj != 0) { + U8 *copy = push_array_no_zero(arena, U8, dim_1u64(section.frange)); + if (lnk_compressed_obj_copy_range(obj->compressed_obj, section.frange, copy, &decode_window)) { + raw = str8(copy, dim_1u64(section.frange)); + explicitly_decoded = 1; + } + } + if (raw.size == 0) { raw = str8_substr(obj->coff.data, section.frange); } + if (sect_state[sect_idx] == 1) { + sect_base[sect_idx] = raw.str; + if (explicitly_decoded) { window_size += raw.size; } + } else { + U8 *copy = raw.str; + if (!explicitly_decoded) { + copy = push_array_no_zero(arena, U8, raw.size); + MemoryCopy(copy, raw.str, raw.size); + } + lnk_obj_apply_relocs_to_buffer(obj, section_number, section.header, str8(copy, raw.size), image_base, image_section_table); + sect_base[sect_idx] = copy; + window_size += raw.size; + } + } + if (window_size > 0) { + for (U64 hwm = g_debug_s_window_hwm; window_size > hwm; hwm = g_debug_s_window_hwm) { + ins_atomic_u64_eval_cond_assign(&g_debug_s_window_hwm, window_size, hwm); + } + } + + // build the remapped CV_DebugS: tracked nodes -> section base + prov offset, + // synthetic/untracked nodes alias their existing bytes + for EachElement(k, src->data_list) { + if (symbols_only && k != idx_symbols) { continue; } + CV_DebugSProvNode *prov = src->prov_list[k].count ? src->prov_list[k].first : 0; + for (String8Node *n = src->data_list[k].first; n != 0; n = n->next) { + String8 s = n->string; + if (prov != 0 && !prov->is_synthetic && prov->sect_idx != CV_DebugSProvSect_Nil) { + Assert(prov->size == n->string.size); + s = str8(sect_base[prov->sect_idx] + prov->off, prov->size); + } + str8_list_push(arena, &out.data_list[k], s); + if (prov != 0) { prov = prov->next; } + } + } + +#if BUILD_DEBUG + // the string table is consumed from the RAW view by cv_dedup_string_tables and the + // module-write checksum/source-file pass -- prove relocs never alter it (journal entries + // cannot: they only target Symbols/InlineeLines runs by construction) + if (!symbols_only) { + U64 idx_strtab = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_StringTable); + CV_DebugSProvNode *prov = src->prov_list[idx_strtab].count ? src->prov_list[idx_strtab].first : 0; + for (String8Node *n = src->data_list[idx_strtab].first; n != 0 && prov != 0; n = n->next, prov = prov->next) { + if (prov->is_synthetic || prov->sect_idx == CV_DebugSProvSect_Nil) { continue; } + if (sect_state[prov->sect_idx] == 2) { + COFF_SectionHeader *raw_hdr = &raw_section_table[prov->sect_idx]; + String8 raw = str8_substr(obj->coff.data, r1u64s(raw_hdr->foff, raw_hdr->fsize)); + Assert(MemoryMatch(sect_base[prov->sect_idx] + prov->off, raw.str + prov->off, prov->size)); + } + } + } +#endif + + // replay the obj's deferred TI/kind fixup journal into the window: same runs, same order as + // lnk_apply_debug_s_fixups_for_obj -- only the destination base differs (the i-th symbol + // input IS the i-th Symbols data_list node, asserted below) + if (cv->has_debug_s_fixup_journal) { + { + String8Node *n = out.data_list[idx_symbols].first; + for (U64 i = cv->debug_s_sym_fixup_offsets[obj_idx], opl = cv->debug_s_sym_fixup_offsets[obj_idx+1]; i < opl; i += 1, n = n->next) { + AssertAlways(n != 0); + Assert(cv->symbol_inputs[i].obj_idx == obj_idx); + Assert(cv->symbol_inputs[i].raw_symbols.size == n->string.size); + lnk_apply_debug_s_patch_run(n->string.str, &cv->debug_s_sym_fixups[i], 0, cv->debug_s_sym_fixups[i].count); + } + } + if (!symbols_only) { + LNK_DebugSInlineJournal *inline_journal = &cv->debug_s_inline_fixups[obj_idx]; + if (inline_journal->patches.count > 0) { + U64 node_idx = 0, cursor = 0; + for (String8Node *n = out.data_list[idx_inlinees].first; n != 0; n = n->next) { + U64 run = inline_journal->node_counts[node_idx++]; + lnk_apply_debug_s_patch_run(n->string.str, &inline_journal->patches, cursor, cursor + run); + cursor += run; + } + Assert(cursor == inline_journal->patches.count); + } + } + } + + lnk_compressed_obj_release_window(&decode_window); + return out; +} + internal -THREAD_POOL_TASK_FUNC(lnk_cv_patcher_symbols_task) +THREAD_POOL_TASK_FUNC(lnk_count_unique_leaf_sizes_task) { - ProfBeginFunction(); LNK_MergeTypes *task = raw_task; - Rng1U64 range = task->input->symbol_patch_task[task_id].input_range; + Rng1U64 range = task->ranges[task_id]; + U64 size = 0; for EachInRange(i, range) { - LNK_SymbolInput symbols = task->input->symbol_inputs[i]; - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - - CV_TypeIndexInfoList ti_info_list = cv_get_symbol_type_index_offsets(temp.arena, symbol.kind, symbol.data); - lnk_fixup_cv_type_indices(task, symbols.obj_idx, symbol.data, ti_info_list); - - temp_end(temp); - } + // Cache size + NOTYPE disposition in result.v's otherwise-unused slots. Materialization + // consumes and replaces each value with the final pointer, avoiding a second journal lookup + // and raw-leaf header read without allocating a side array. + U64 meta = lnk_leaf_ref_materialize_meta(task->input, task->unique_leaf_refs_arr[task->ti_source].v[i]); + task->result.v[task->ti_source][i] = (U8 *)meta; + size += meta >> 2; } - ProfEnd(); + task->leaf_buffer_offsets[task_id] = size; // exclusive-scanned into offsets on the main thread } +// Materialize unique leaves: copy each unique raw leaf (existing sorted order) into one contiguous +// private buffer and apply the type-index fixup to the COPY. This fuses the old +// lnk_cv_patcher_leaves_task (which patched TIs in-place into the mapped input, dirtying one +// copy-on-write page per touched .debug$T page) with the old lnk_unbucket_raw_leaves_task (which +// pointed result.v into the input). result.v now points into the copy: identical bytes, identical +// order, clean input pages. internal -THREAD_POOL_TASK_FUNC(lnk_cv_patcher_inlines_task) +THREAD_POOL_TASK_FUNC(lnk_assign_unique_leaf_destinations_task) { - ProfBeginFunction(); - LNK_MergeTypes *task = raw_task; - U64 obj_idx = task_id; - String8List inlinee_lines = cv_sub_section_from_debug_s(task->input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); - Arena *fixed_arena = task->fixed_arenas[worker_id]; - for EachNode(inline_data_n, String8Node, inlinee_lines.first) { + LNK_MergeTypes *task = raw_task; + Rng1U64 range = task->ranges[task_id]; + U8 *cursor = task->leaf_buffer + task->leaf_buffer_offsets[task_id]; + for EachInRange(i, range) { + U64 meta = (U64)task->result.v[task->ti_source][i]; + U64 raw_size = meta >> 2; + U64 rewrite = meta & 3; + Assert(((U64)cursor & 3) == 0); + task->result.v[task->ti_source][i] = (U8 *)((U64)cursor | rewrite); + cursor += raw_size; + } +} + +internal void +lnk_materialize_unique_leaf_to(LNK_MergeTypes *task, CV_TypeIndexSource source, U64 i, + U8 *cursor, U64 raw_size, U64 rewrite, + Arena *fixed_arena, LNK_CObjDecodeWindow *decode_window) +{ + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[source].v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + + // copy raw leaf into the private buffer. Hot path = the ORIGINAL read+memcpy (one bitmap + // test on top); journaled leaves replay the NOTYPE rewrite into the COPY (full rewrite => + // bare { size=sizeof(CV_LeafKind), kind=LF_NOTYPE } header; KIND_ONLY (0x1522) => copy then + // patch the kind). Byte-identical to the old post-in-place-write copy. + if (rewrite == LNK_LEAF_MATERIALIZE_FULL_NOTYPE) { + memory_write16(cursor + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + } else { + LNK_Obj *obj = obj_idx < task->input->obj_count ? task->input->obj_arr[obj_idx] : 0; + B32 copied = 0; + if (obj && obj->compressed_obj && (debug_t->sidecar_sizes || debug_t->sidecar_packed)) { + U64 leaf_off = cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + Rng1U64 raw_range = rng_1u64(debug_t->sidecar_raw_base + leaf_off, + debug_t->sidecar_raw_base + leaf_off + raw_size); + copied = lnk_compressed_obj_copy_range(obj->compressed_obj, raw_range, cursor, decode_window); + } + if (!copied) { + U8 *raw_leaf = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + MemoryCopy(cursor, raw_leaf, raw_size); + } + if (rewrite == LNK_LEAF_MATERIALIZE_KIND_NOTYPE) { + memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + } + } + task->result.v[source][i] = cursor; + + // fixup type indices on the copy (same math the in-place leaf patcher applied) Temp temp = temp_begin(fixed_arena); - CV_TypeIndexInfoList ti_info_list = cv_get_inlinee_type_index_offsets(temp.arena, inline_data_n->string); - lnk_fixup_cv_type_indices(task, obj_idx, inline_data_n->string, ti_info_list); + CV_Leaf leaf = { + .kind = memory_read16(task->result.v[source][i] + OffsetOf(CV_LeafHeader, kind)), + .data = str8(task->result.v[source][i] + sizeof(CV_LeafHeader), raw_size - sizeof(CV_LeafHeader)), + }; + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_offs); temp_end(temp); - } - ProfEnd(); +} + +internal void +lnk_materialize_tagged_unique_leaf(LNK_MergeTypes *task, CV_TypeIndexSource source, U64 i, + Arena *fixed_arena, LNK_CObjDecodeWindow *decode_window) +{ + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[source].v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + U64 tagged = (U64)task->result.v[source][i]; + U64 rewrite = tagged & 3; + U8 *cursor = (U8 *)(tagged & ~(U64)3); + U64 raw_size = rewrite == LNK_LEAF_MATERIALIZE_FULL_NOTYPE ? sizeof(CV_LeafHeader) : + cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + lnk_materialize_unique_leaf_to(task, source, i, cursor, raw_size, rewrite, fixed_arena, decode_window); } internal -THREAD_POOL_TASK_FUNC(lnk_cv_patcher_leaves_task) +THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) { ProfBeginFunction(); - LNK_MergeTypes *task = raw_task; - Rng1U64 range = task->ranges[task_id]; - Arena *fixed_arena = task->fixed_arenas[task_id]; - for EachInRange(leaf_ref_idx, range) { - Temp temp = temp_begin(fixed_arena); - LNK_LeafRef patch = task->unique_leaf_refs_arr[task->ti_source].v[leaf_ref_idx]; - U32 obj_idx = lnk_leaf_ref_obj_idx(patch); - CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, lnk_leaf_ref_leaf_idx(patch)); - CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_info_list); - temp_end(temp); + LNK_MergeTypes *task = raw_task; + Rng1U64 range = task->ranges[task_id]; + U8 *cursor = task->leaf_buffer + task->leaf_buffer_offsets[task_id]; + for EachInRange(i, range) { + U64 meta = (U64)task->result.v[task->ti_source][i]; + U64 raw_size = meta >> 2; + U64 rewrite = meta & 3; + lnk_materialize_unique_leaf_to(task, task->ti_source, i, cursor, raw_size, rewrite, + task->fixed_arenas[worker_id], &task->decode_windows[worker_id]); + cursor += raw_size; } ProfEnd(); } internal -THREAD_POOL_TASK_FUNC(lnk_unbucket_raw_leaves_task) +THREAD_POOL_TASK_FUNC(lnk_materialize_fused_tpi_ipi_task) { + ProfBeginFunction(); LNK_MergeTypes *task = raw_task; - Rng1U64 range = task->ranges[task_id]; - for EachInRange(i, range) { - LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; - CV_DebugT *debug_t = &task->input->debug_t_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, lnk_leaf_ref_leaf_idx(leaf_ref)); - task->result.v[task->ti_source][i] = raw_leaf.str; + U64 tpi_i = task->materialize_obj_offsets[CV_TypeIndexSource_TPI][task_id]; + U64 tpi_opl = task->materialize_obj_offsets[CV_TypeIndexSource_TPI][task_id + 1]; + U64 ipi_i = task->materialize_obj_offsets[CV_TypeIndexSource_IPI][task_id]; + U64 ipi_opl = task->materialize_obj_offsets[CV_TypeIndexSource_IPI][task_id + 1]; + while (tpi_i < tpi_opl || ipi_i < ipi_opl) { + CV_TypeIndexSource source; + U64 i; + if (ipi_i >= ipi_opl || + (tpi_i < tpi_opl && + lnk_leaf_ref_leaf_idx(task->unique_leaf_refs_arr[CV_TypeIndexSource_TPI].v[tpi_i]) < + lnk_leaf_ref_leaf_idx(task->unique_leaf_refs_arr[CV_TypeIndexSource_IPI].v[ipi_i]))) { + source = CV_TypeIndexSource_TPI; + i = tpi_i++; + } else { + source = CV_TypeIndexSource_IPI; + i = ipi_i++; + } + lnk_materialize_tagged_unique_leaf(task, source, i, task->fixed_arenas[worker_id], + &task->decode_windows[worker_id]); } + ProfEnd(); } internal @@ -1772,73 +4188,59 @@ THREAD_POOL_TASK_FUNC(lnk_unbucket_hashes_task) LNK_MergeTypes *task = raw_task; Rng1U64 range = task->ranges[task_id]; for EachInRange(i, range) { - LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; - U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); - U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); - CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); task->result.hashes[task->ti_source][i] = task->input->debug_h_arr[obj_idx].v[leaf_idx]; } } -internal -THREAD_POOL_TASK_FUNC(lnk_fixup_symbols_task) +internal void +lnk_profile_cobj_winner_segments(LNK_MergeTypes *task) { - LNK_MergeTypes *task = raw_task; - - LNK_SymbolInput symbols = task->input->symbol_inputs[task_id]; - U64 leaf_count_ipi = task->result.count [CV_TypeIndexSource_IPI]; - U8 **leaf_arr_ipi = task->result.v [CV_TypeIndexSource_IPI]; - CV_TypeIndex min_ti_ipi = task->min_type_indices[CV_TypeIndexSource_IPI]; - - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - - // convert symbol to final type - CV_SymKind *sym_kind_ptr = cv_kind_ptr_from_symbol(symbol); - switch (*sym_kind_ptr) { - case CV_SymKind_PROC_ID_END: *sym_kind_ptr = CV_SymKind_END; break; - - case CV_SymKind_LPROC32_ID: *sym_kind_ptr = CV_SymKind_LPROC32; goto fixup_id; - case CV_SymKind_GPROC32_ID: *sym_kind_ptr = CV_SymKind_GPROC32; goto fixup_id; - case CV_SymKind_LPROC32_DPC_ID: *sym_kind_ptr = CV_SymKind_LPROC32_DPC; goto fixup_id; - case CV_SymKind_LPROCMIPS_ID: *sym_kind_ptr = CV_SymKind_LPROCMIPS; goto fixup_id; - case CV_SymKind_GPROCMIPS_ID: *sym_kind_ptr = CV_SymKind_GPROCMIPS; goto fixup_id; - case CV_SymKind_LPROCIA64_ID: *sym_kind_ptr = CV_SymKind_LPROCIA64; goto fixup_id; - case CV_SymKind_GPROCIA64_ID: *sym_kind_ptr = CV_SymKind_GPROCIA64; goto fixup_id; - fixup_id:; { - CV_SymProc32 *proc32 = str8_deserial_get_raw_ptr(symbol.data, 0, sizeof(*proc32)); - if (proc32->itype < min_ti_ipi) { - // TODO: in some cases destructors don't have a type, need a repro - break; - } - - if ((proc32->itype - min_ti_ipi) > leaf_count_ipi) { - Assert(0 && "TODO: error handle corrupted type index"); - break; - } - - U64 leaf_idx = proc32->itype - min_ti_ipi; - String8 leaf_data = str8(leaf_arr_ipi[leaf_idx], max_U64); - - CV_Leaf leaf; - if (cv_read_leaf(leaf_data, 0, 1, &leaf) == 0) { InvalidPath; } - - U64 min_leaf_size = cv_header_struct_size_from_leaf_kind(leaf.kind); - if (min_leaf_size > leaf.data.size) { Assert(!"TODO: error handle corrupt leaf"); break; } - - if (leaf.kind == CV_LeafKind_FUNC_ID) { - CV_LeafFuncId *func_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*func_id)); - proc32->itype = func_id->itype; - } else if (leaf.kind == CV_LeafKind_MFUNC_ID) { - CV_LeafMFuncId *mfunc_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*mfunc_id)); - proc32->itype = mfunc_id->itype; - } else { - Assert(!"TODO: erorr handle unexpected leaf type"); - break; + char *env = getenv("RAD_COBJ_PROFILE_WINNERS"); + if (env == 0 || env[0] == '0') { return; } + enum { BucketCount = 16, SizeCount = 4 }; + U64 segment_sizes[SizeCount] = { KB(128), KB(256), KB(512), MB(1) }; + U64 touched[CV_TypeIndexSource_COUNT][BucketCount][SizeCount] = {0}; + U64 leaf_counts[CV_TypeIndexSource_COUNT][BucketCount] = {0}; + U64 leaf_bytes[CV_TypeIndexSource_COUNT][BucketCount] = {0}; + + for (U32 source = CV_TypeIndexSource_TPI; source <= CV_TypeIndexSource_IPI; ++source) { + U64 last_key[SizeCount] = { max_U64, max_U64, max_U64, max_U64 }; + LNK_LeafRefArray refs = task->unique_leaf_refs_arr[source]; + for EachIndex(i, refs.count) { + LNK_LeafRef ref = refs.v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(ref); + if (obj_idx >= task->input->obj_count) { continue; } + LNK_Obj *obj = task->input->obj_arr[obj_idx]; + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + if (!obj->compressed_obj || (!debug_t->sidecar_sizes && !debug_t->sidecar_packed)) { continue; } + U64 bucket = (U64)obj_idx * BucketCount / task->input->obj_count; + U64 raw_off = debug_t->sidecar_raw_base + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + leaf_counts[source][bucket] += 1; + leaf_bytes[source][bucket] += cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + for (U32 s = 0; s < SizeCount; ++s) { + U64 key = ((U64)obj_idx << 32) | (raw_off / segment_sizes[s]); + if (key != last_key[s]) { + touched[source][bucket][s] += 1; + last_key[s] = key; + } } - } break; + } + } + + for (U32 source = CV_TypeIndexSource_TPI; source <= CV_TypeIndexSource_IPI; ++source) { + for (U32 bucket = 0; bucket < BucketCount; ++bucket) { + lnk_log(LNK_Log_Timers, + "[cobj winners] src=%u bucket=%u objs=%u-%u leaves=%llu leafMiB=%llu seg128=%llu seg256=%llu seg512=%llu seg1024=%llu", + source, bucket, + (task->input->obj_count * bucket) / BucketCount, + (task->input->obj_count * (bucket + 1)) / BucketCount - 1, + leaf_counts[source][bucket], leaf_bytes[source][bucket] / MB(1), + touched[source][bucket][0], touched[source][bucket][1], + touched[source][bucket][2], touched[source][bucket][3]); } } } @@ -1853,9 +4255,14 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; CV_TypeIndex *obj_ti_map = task->obj_ti_batch + task->obj_ti_map_offsets[obj_idx]; + // P4: journal bitmap overlays LF_NOTYPE on journaled real-obj leaves (raw bytes are pre-rewrite) + U64 *notype_bm = input->notype_journal[obj_idx].bitmap; + U64 notype_cap = input->notype_journal[obj_idx].bit_cap; + for EachIndex(leaf_idx, debug_t->count) { - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(leaf.kind); + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); + if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } + CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(kind); LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); LNK_AssignedTiHash *assigned = &task->assigned_ti_arr[source]; obj_ti_map[leaf_idx] = lnk_assigned_ti_hash_search(assigned, input, leaf_ref); @@ -1871,8 +4278,12 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK Temp scratch = temp_begin(lnk_get_huge_arena()); LNK_MergeTypes task = { .input = input }; - U64 max_ti_list_size = sizeof(CV_TypeIndexInfo) * (max_U16 / sizeof(CV_TypeIndex)); + // scratch bound: CV_TiOff arrays for member-walk leaves are built with doubling growth + // (sum of caps <= ~4x entry count, entries <= max_U16/8 per leaf => <= ~512KB), plus + // deep-hash stack frames; 2x the legacy per-node list bound keeps comfortable headroom + U64 max_ti_list_size = 2 * sizeof(CV_TypeIndexInfo) * (max_U16 / sizeof(CV_TypeIndex)); task.fixed_arenas = alloc_fixed_size_arena_array(scratch.arena, tp->worker_count, max_ti_list_size, max_ti_list_size); + task.decode_windows = push_array(scratch.arena, LNK_CObjDecodeWindow, tp->worker_count); ProfBegin("Produce Hashes"); { @@ -1882,6 +4293,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK U32Array indices; U32Array hash_indices; } hash_targets[] = { + { lnk_hash_debug_t_deep_task, input->ifc_indices }, // hash .ifc blobs first: int-obj leaves redirect into them { lnk_hash_debug_t_task, input->debug_p_indices }, // hash .debug$P first so we can mix in hashes for precompiled sub leaves when hashing leaves in .debug$T { lnk_hash_debug_t_task, input->int_obj_indices }, { lnk_hash_debug_t_deep_task, input->type_server_indices }, @@ -1912,10 +4324,40 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); + // batch-populate the .debug$T/$P leaf data the hashers below walk leaf by + // leaf; under farm-wide memory pressure these mapped pages were trimmed + // since the parse phase touched them (see lnk_prefetch_ranges) + if (lnk_should_prefetch_mapped_input()) ProfScope("Prefetch Type Data") + { + Temp temp = temp_begin(scratch.arena); + + U64 range_cap = 0; + for EachElement(i, hash_targets) { range_cap += hash_targets[i].hash_indices.count; } + + Rng1U64 *ranges = push_array_no_zero(temp.arena, Rng1U64, range_cap); + U64 range_count = 0; + for EachElement(i, hash_targets) { + for EachIndex(k, hash_targets[i].hash_indices.count) { + String8 data = input->debug_t_arr[hash_targets[i].hash_indices.v[k]].data; + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + } + U64 prefetch_begin_us = now_time_us(); + U64 prefetch_bytes = 0; + for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } + lnk_prefetch_ranges(tp, input->config->debug_worker_cap, range_count, ranges); + lnk_log(LNK_Log_Timers, "[merge] prefetched %llu type data ranges (%llu MiB) in %.2f ms", + range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); + + temp_end(temp); + } + for EachElement(i, hash_targets) { task.indices = hash_targets[i].hash_indices; ProfBegin("Hash [Count: %.*s]", str8_varg(str8_from_count(scratch.arena, task.indices.count))); - tp_for_parallel(tp, 0, task.indices.count, hash_targets[i].hasher_task, &task); + // P4: pass real worker arenas -- the shallow hasher's invalid-TI/cyclic discards push + // NOTYPE journal entries (rare error paths, bytes are negligible on tp_temp) + tp_for_parallel(tp, tp_temp, task.indices.count, hash_targets[i].hasher_task, &task); ProfEnd(); } @@ -1938,27 +4380,110 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); + // bucket_arr (the ~1.3x-total-leaf-count probe tables) is only live through the dedup + extract + // phases: its last read is in lnk_get_present_buckets_task ("Copy present buckets") which copies + // bucket pointers into unique_leaf_refs. Allocate it in a dedicated arena so we can release that + // multi-GB working set immediately after the extract loop, before the merge-types/PDB-build peak. + // (A temp_begin on scratch.arena would not work: many surviving allocations -- unique_leaf_refs, + // assigned_ti, radix scratch -- land in scratch.arena after bucket_arr.) + Arena *bucket_arena = arena_alloc(.name = "LEAF_BUCKETS"); + ProfBegin("Leaf Hash Table Init"); - for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { - U64 total_count = 0; - for EachIndex(obj_idx, input->count) { total_count += input->debug_t_arr[obj_idx].source_counts[ti_source]; } + // fallback caps derived from TOTAL (pre-dedup, pre-prune) leaf counts. total >= unique always, so + // these caps can never overflow; they are also the caps the estimate-based sizing clamps against + // and retries with. NOTE: the NULL-source cap is deliberately derived from the pre-prune + // source_counts (see the IFC pruning comment in lnk_leaf_dedup_task) -- pruned NOTYPE leaves are + // never inserted, so pre-prune totals always cover the insert set with slack. + U64 leaf_ht_cap_fallback[CV_TypeIndexSource_COUNT] = {0}; + { + U64 total_counts[CV_TypeIndexSource_COUNT] = {0}; + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + for EachIndex(obj_idx, input->count) { total_counts[ti_source] += input->debug_t_arr[obj_idx].source_counts[ti_source]; } + // pow2 cap so bucket index is hash & (cap-1) (mask) instead of hash % cap (a 64-bit DIV in the + // densest dedup probe loop). u64_up_to_pow2(1.3*count) keeps load factor <= ~0.65. + leaf_ht_cap_fallback[ti_source] = u64_up_to_pow2(1 + ((total_counts[ti_source] * 13) / 10)); // * 1.3, pow2 + } + + // On dup-heavy input (PCH/type-server fan-out) unique count is a small fraction of total, and a + // total-sized probe table wastes multi-GB of demand-zero page faults on 64B-apart random probes. + // Estimate the distinct-hash count from the already-produced debug_h hashes (Produce Hashes + // completes above) with a per-source presence bitmap + linear counting, and size the tables from + // that instead. Everything here is a pure function of the input hashes, so the caps -- and the + // overflow/retry decision below -- are identical run to run. + ProfBegin("Estimate Unique Leaves"); + U64 estimate_begin_us = now_time_us(); + { + // sweep exactly the objs whose leaves get inserted: prepopulate + the four dedup passes + U32Array sweep_arrs[] = { input->debug_p_indices, input->int_obj_indices, input->type_server_indices, input->ifc_indices }; + U32Array sweep_indices = {0}; + for EachElement(i, sweep_arrs) { sweep_indices.count += sweep_arrs[i].count; } + sweep_indices.v = push_array_no_zero(scratch.arena, U32, sweep_indices.count); + sweep_indices.count = 0; + for EachElement(i, sweep_arrs) { + MemoryCopy(sweep_indices.v + sweep_indices.count, sweep_arrs[i].v, sizeof(U32) * sweep_arrs[i].count); + sweep_indices.count += sweep_arrs[i].count; + } + + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + // at most ceil(total/K) hashes are inserted under K-th-position sampling, so bits >= + // total/K >= sampled-unique keeps the bitmap load < 1, where linear counting is accurate; + // clamp keeps the transient bitmap allocation bounded (16MB per source at the top end) + U64 sampled_total = (total_counts[ti_source] + LNK_ESTIMATE_SAMPLE_STRIDE - 1) / LNK_ESTIMATE_SAMPLE_STRIDE; + task.estimate_bitmap_bits[ti_source] = u64_up_to_pow2(Clamp(1ull << 16, sampled_total, 1ull << 27)); + task.estimate_bitmap [ti_source] = push_array(scratch.arena, U32, task.estimate_bitmap_bits[ti_source] / 32); + } + + task.indices = sweep_indices; + tp_for_parallel(tp, 0, task.indices.count, lnk_estimate_unique_leaves_task, &task); + + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + U64 bit_count = task.estimate_bitmap_bits[ti_source]; + U64 word_count = bit_count / 32; + U64 set_count = 0; + for EachIndex(word_idx, word_count) { set_count += count_bits_set32(task.estimate_bitmap[ti_source][word_idx]); } + + U64 cap = leaf_ht_cap_fallback[ti_source]; + U64 zero_count = bit_count - set_count; + // the NULL-source table keeps the historic total-based cap: objs synthesized outside the + // parse path never populate source_counts, so its cap can be far below the number of + // NULL-source leaves thrown at it. that has always been tolerated -- NULL-source leaves are + // never emitted, the table just drops what does not fit (see the fall-through handling in + // lnk_leaf_dedup_task) -- so it must not participate in estimate sizing or overflow retry. + if (ti_source != CV_TypeIndexSource_NULL && set_count > 0 && zero_count > 0) { + // linear counting: distinct ~= m * ln(m / zeros) + F64 estimate = (F64)bit_count * log((F64)bit_count / (F64)zero_count); + if (estimate < (F64)set_count) { estimate = (F64)set_count; } + // scale the sampled distinct count back up to a full-population estimate (see the + // sampling comment above lnk_estimate_unique_leaves_task) + estimate *= LNK_ESTIMATE_SAMPLE_SCALE; + // 1.9x safety keeps the load factor <= ~0.55 even before pow2 rounding; overflow (only + // possible if the estimate undershoots by >1.8x) is caught and retried deterministically + U64 target = (U64)(estimate * 1.9) + 4096; + cap = Min(u64_up_to_pow2(target), leaf_ht_cap_fallback[ti_source]); + lnk_log(LNK_Log_Timers, "[typededup] estimate src=%llu: set=%llu est=%.0f cap=%llu (fallback %llu)", + ti_source, set_count, estimate, cap, leaf_ht_cap_fallback[ti_source]); + } + task.leaf_ht_arr[ti_source].cap = cap; + } + } + lnk_log(LNK_Log_Timers, "[typededup] unique-leaf estimate in %.2f ms", (F64)(now_time_us() - estimate_begin_us) / 1000.0); + ProfEnd(); - task.leaf_ht_arr[ti_source].cap = total_count; - task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 - task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(scratch.arena, LNK_LeafRef, task.leaf_ht_arr[ti_source].cap); - MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * task.leaf_ht_arr[ti_source].cap); + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(bucket_arena, LNK_LeafRef, task.leaf_ht_arr[ti_source].cap); + MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY - tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); + tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); #endif + } } ProfEnd(); U32Array dedup_type_server_indices = input->type_server_indices; - ProfBegin("Prepopulate hash table with largest type-set"); + LNK_TypeServer *largest_ts = 0; { - LNK_TypeServer *largest_ts = 0; for EachIndex(i, input->ts_arr.count) { LNK_TypeServer *ts = &input->ts_arr.v[i]; if (ts->rrt == 0) { continue; } @@ -1970,7 +4495,6 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK if (largest_ts) { task.pop_obj_idx = input->ts_obj_range.min + largest_ts->ts_idx; task.pop_range = tp_divide_work(scratch.arena, task.input->debug_t_arr[task.pop_obj_idx].count, tp->worker_count); - tp_for_parallel(tp, tp_temp, tp->worker_count, lnk_populate_leaf_ht, &task); U32Array new_dedup_type_server_indices = { .v = push_array(scratch.arena, U32, input->type_server_indices.count) }; for EachIndex(i, input->type_server_indices.count) { @@ -1980,21 +4504,75 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK dedup_type_server_indices = new_dedup_type_server_indices; } } - ProfEnd(); - ProfBegin("Leaf Dedup"); - task.indices = input->debug_p_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$P"); + for (U64 attempt = 0; ; attempt += 1) { + ProfBegin("Prepopulate hash table with largest type-set"); + if (largest_ts) { + tp_for_parallel(tp, tp_temp, tp->worker_count, lnk_populate_leaf_ht, &task); + } + ProfEnd(); - task.indices = input->int_obj_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$T"); + ProfBegin("Leaf Dedup"); + task.indices = input->debug_p_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$P"); - task.indices = dedup_type_server_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "Type Servers"); - ProfEnd(); + task.indices = input->int_obj_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$T"); + + task.indices = dedup_type_server_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "Type Servers"); + + task.indices = input->ifc_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "IFC Blobs"); + ProfEnd(); + + if (ins_atomic_u32_eval(&task.leaf_ht_overflow) == 0) { break; } + + // an estimate-sized table overflowed: rebuild every probe table at the always-sufficient + // total-based caps and redo the passes. the input hashes are deterministic, so this branch is + // taken (or not) identically every run, and the retried result is what the total-based sizing + // would have produced in the first place. the fallback caps cannot overflow, so at most one retry. + AssertAlways(attempt == 0); + lnk_log(LNK_Log_Debug, "leaf dedup: unique-count estimate overflowed, retrying with total-based table caps"); + ins_atomic_u32_eval_assign(&task.leaf_ht_overflow, 0); + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + task.leaf_ht_arr[ti_source].cap = leaf_ht_cap_fallback[ti_source]; + task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(bucket_arena, LNK_LeafRef, leaf_ht_cap_fallback[ti_source]); + MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * leaf_ht_cap_fallback[ti_source]); + } + } ProfBegin("Extract present buckets from the leaf hash tables"); + B32 winner_prefetch_enabled = 0; + { + char *env = getenv("RAD_COBJ_PREFETCH_WINNERS"); + winner_prefetch_enabled = (env == 0 || env[0] != '0'); + if (winner_prefetch_enabled) { + task.winner_segment_offsets = push_array(scratch.arena, U64, input->obj_count + 1); + for EachIndex(obj_idx, input->obj_count) { + task.winner_segment_offsets[obj_idx + 1] = task.winner_segment_offsets[obj_idx] + + lnk_compressed_obj_segment_count(input->obj_arr[obj_idx]->compressed_obj); + } + U64 total_segments = task.winner_segment_offsets[input->obj_count]; + if (total_segments) { + task.winner_segment_word_count = CeilIntegerDiv(total_segments, 64); + task.winner_segment_bitmap = push_array(scratch.arena, U64, task.winner_segment_word_count); + task.winner_segment_worker_bitmaps = push_array(scratch.arena, U64, + task.winner_segment_word_count * tp->worker_count); + task.winner_segment_keys = push_array_no_zero(scratch.arena, U64, total_segments); + } + } + } + + U32 winner_prefetch_worker_count = 8; + { + char *env = getenv("RAD_COBJ_PREFETCH_WORKERS"); + if (env) { winner_prefetch_worker_count = Clamp(1, atoi(env), LNK_WINNER_PREFETCH_WORKER_MAX); } + } + LNK_WinnerPrefetchFlight winner_prefetch_flights[CV_TypeIndexSource_COUNT] = {0}; + U64 winner_prefetch_key_cursor = 0; + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.ti_source = ti_source; task.counts[ti_source] = push_array(scratch.arena, U64, tp->worker_count); @@ -2006,12 +4584,81 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.offsets[ti_source] = offsets_from_counts_array_u64(scratch.arena, task.counts[ti_source], tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_get_present_buckets_task, &task, "Copy present buckets"); + lnk_log(LNK_Log_Timers, "[typededup] src=%llu: unique=%llu cap=%llu load=%.3f", + ti_source, task.unique_leaf_refs_arr[ti_source].count, task.leaf_ht_arr[ti_source].cap, + task.leaf_ht_arr[ti_source].cap ? (F64)task.unique_leaf_refs_arr[ti_source].count / (F64)task.leaf_ht_arr[ti_source].cap : 0.0); + + // Each type source gets an independent flight. TPI remains in flight while IPI is extracted + // and sorted, and IPI may join it; both are joined only immediately before materialization. + // Reduce lane-private bitmaps after extraction. The bitmap is only ~180k bits for UEFN; + // scanning it is negligible and avoids a contended atomic operation for every winning leaf. + if (task.winner_segment_bitmap && ti_source != CV_TypeIndexSource_NULL) { + U64 obj_idx = 0; + for EachIndex(word_idx, task.winner_segment_word_count) { + U64 merged = 0; + for EachIndex(lane, tp->worker_count) { + merged |= task.winner_segment_worker_bitmaps[lane * task.winner_segment_word_count + word_idx]; + } + U64 new_bits = merged & ~task.winner_segment_bitmap[word_idx]; + task.winner_segment_bitmap[word_idx] |= merged; + while (new_bits) { + U64 bit_idx = ctz64(new_bits); + U64 global_idx = word_idx * 64 + bit_idx; + while (obj_idx + 1 < input->obj_count && + global_idx >= task.winner_segment_offsets[obj_idx + 1]) { + obj_idx += 1; + } + U64 segment_idx = global_idx - task.winner_segment_offsets[obj_idx]; + task.winner_segment_keys[task.winner_segment_key_count++] = (obj_idx << 32) | segment_idx; + new_bits &= new_bits - 1; + } + } + } + U64 key_opl = task.winner_segment_key_count; + if (key_opl > winner_prefetch_key_cursor) { + U64 count = key_opl - winner_prefetch_key_cursor; + Rng1U64 *ranges = push_array_no_zero(scratch.arena, Rng1U64, count); + U64 stored_bytes = 0; + for EachIndex(i, count) { + U64 key = task.winner_segment_keys[winner_prefetch_key_cursor + i]; + U32 obj_idx = (U32)(key >> 32); + U32 segment_idx = (U32)key; + ranges[i] = lnk_compressed_obj_stored_segment_range(input->obj_arr[obj_idx]->compressed_obj, segment_idx); + stored_bytes += dim_1u64(ranges[i]); + } + winner_prefetch_key_cursor = key_opl; + lnk_winner_prefetch_start(&winner_prefetch_flights[ti_source], winner_prefetch_worker_count, + count, ranges, stored_bytes); + } + // sort output leaves based on { location index, leaf index } to guarantee determinism ProfScope("Radix Sort") { u64_array_sort_radix_parallel(tp, task.unique_leaf_refs_arr[ti_source].count, task.unique_leaf_refs_arr[ti_source].v); +#if BUILD_DEBUG + LNK_LeafRefArray arr = task.unique_leaf_refs_arr[ti_source]; + for (U64 i = 1; i < arr.count; ++i) { + AssertAlways(lnk_leaf_ref_compare(arr.v[i-1], arr.v[i]) <= 0); + } +#endif } } + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + lnk_winner_prefetch_join(&winner_prefetch_flights[ti_source]); + } + + lnk_profile_cobj_winner_segments(&task); + + // bucket_arr is fully consumed (copied into unique_leaf_refs / sorted) -- release the probe tables + // now so this multi-GB working set is gone before the merge-types/PDB-build peak. Handed to a + // background reaper thread: a serial VirtualFree(MEM_RELEASE) of these multi-GB committed blocks + // costs ~350ms+ of main-thread kernel time (MiDeleteVaDirect/MiDecommitFreePage), which otherwise + // sits on the critical path between dedup and the type-index fixup passes. All pointers into the + // arena are dropped below before the launch returns ownership to the reaper. + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.leaf_ht_arr[ti_source].bucket_arr = 0; } + if (g_arena_reaper_thread.u64[0] != 0) { thread_join(g_arena_reaper_thread, max_U64); } + g_arena_reaper_thread = thread_launch(lnk_arena_release_thread, bucket_arena); + #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "TPI Count: %.*s", str8_varg(str8_from_count(scratch.arena, task.unique_leaf_refs_arr[CV_TypeIndexSource_TPI].count))); tmMessage(0, TMMF_ICON_NOTE, "IPI Count: %.*s", str8_varg(str8_from_count(scratch.arena, task.unique_leaf_refs_arr[CV_TypeIndexSource_IPI].count))); @@ -2025,12 +4672,10 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.ti_source = ti_source; task.assigned_ti_arr[ti_source].cap = ((task.unique_leaf_refs_arr[ti_source].count * 13) / 10); - task.assigned_ti_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.assigned_ti_arr[ti_source].cap); - - // unique extraction is complete, so the dedup bucket slots can back the - // direct hash table without increasing peak memory - Assert(task.assigned_ti_arr[ti_source].cap <= task.leaf_ht_arr[ti_source].cap); - task.assigned_ti_arr[ti_source].hash_arr = task.leaf_ht_arr[ti_source].bucket_arr; + // Keep hash + assigned TI adjacent: the fixup passes probe this table hundreds of + // millions of times and usually need both fields. Packed 12-byte entries preserve the + // exact former 8+4 byte footprint while normally requiring one cache line instead of two. + task.assigned_ti_arr[ti_source].v = push_array(scratch.arena, U8, task.assigned_ti_arr[ti_source].cap * LNK_ASSIGNED_TI_ENTRY_SIZE); task.min_type_indices[ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); @@ -2038,19 +4683,12 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); - if (~merge_flags & LNK_MergeTypeFlag_SkipSymbolTypeFixup) { - tp_for_parallel_prof(tp, 0, input->symbol_patch_task_count, lnk_cv_patcher_symbols_task, &task, "Fixup Symbol Type Indices"); - - task.ranges = 0; - task.debug_s_arr = input->debug_s_arr; - tp_for_parallel_prof(tp, 0, input->count, lnk_cv_patcher_inlines_task, &task, "Fixup Inlines Type Indices"); - } + // NOTE: the $S symbol/inlinee TI fixups are journaled below (after the materialize pass); + // the bytes are patched per obj at the start of the module-write visit. - for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { - task.ti_source = ti_source; - task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); - tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_cv_patcher_leaves_task, &task, "Fixup Types Type Indices"); - } + // NOTE: the leaf TI-fixup is fused into the unbucket/materialize pass below -- it copies each + // unique leaf into a private buffer and patches the copy, instead of patching the mapped input + // (which copy-on-writes one page per touched .debug$T page). } ProfEnd(); @@ -2070,6 +4708,15 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK tp_for_parallel_prof(tp, 0, input->obj_count, lnk_build_obj_ti_map, &task, "Build TI Map"); } + lnk_compressed_obj_log_phase_stats("before type materialize"); + U64 materialize_begin_us = now_time_us(); + B32 has_compressed_inputs = 0; + for EachIndex(obj_idx, input->obj_count) { + if (input->obj_arr[obj_idx]->compressed_obj) { has_compressed_inputs = 1; break; } + } + char *fuse_materialize_env = getenv("RAD_COBJ_FUSE_TYPE_MATERIALIZE"); + B32 fuse_materialize = has_compressed_inputs && + (fuse_materialize_env == 0 || fuse_materialize_env[0] != '0'); for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { LNK_LeafRefArray unique_leaf_refs = task.unique_leaf_refs_arr[ti_source]; @@ -2077,18 +4724,146 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.result.count[ti_source] = unique_leaf_refs.count; task.result.v [ti_source] = push_array(tp_temp->v[0], U8 *, unique_leaf_refs.count); task.ranges = tp_divide_work(scratch.arena, unique_leaf_refs.count, tp->worker_count); - tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_unbucket_raw_leaves_task, &task, "Unbucket Leaves"); - if (merge_flags & LNK_MergeTypeFlag_ExportHashes) { + // per-lane byte totals for the materialize buffer, exclusive-scanned into offsets + task.leaf_buffer_offsets = push_array_no_zero(scratch.arena, U64, tp->worker_count + 1); + tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_count_unique_leaf_sizes_task, &task, "Count Leaf Sizes"); + { + U64 acc = 0; + for EachIndex(lane, tp->worker_count) { + U64 lane_size = task.leaf_buffer_offsets[lane]; + task.leaf_buffer_offsets[lane] = acc; + acc += lane_size; + } + task.leaf_buffer_offsets[tp->worker_count] = acc; + // standalone allocation (not an arena push): pdb_build_types copies these bytes + // into MSF pages and nothing reads cv_types.v afterwards, so the linker path + // releases the buffer right after -- multi-GB on AutoRTFM-scale inputs. An arena + // push could not be handed back without poisoning later pushes into the range. + if (acc > 0) { + task.leaf_buffer = reserve_memory(acc); + if (task.leaf_buffer == 0 || !commit_memory(task.leaf_buffer, acc)) { + lnk_error(LNK_Error_Boot, "failed to allocate %M for merged type leaves", acc); + } + task.result.leaf_buffers[ti_source] = str8(task.leaf_buffer, acc); + } else { + task.leaf_buffer = push_array_no_zero(tp_temp->v[0], U8, 1); + } + } + if (fuse_materialize && ti_source != CV_TypeIndexSource_NULL) { + tp_for_parallel(tp, 0, tp->worker_count, lnk_assign_unique_leaf_destinations_task, &task); + } else { + // Exact original raw-OBJ path: count/meta and destination copy remain a + // single lane-balanced traversal with no tagged-pointer setup pass. + tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_materialize_unique_leaves_task, &task, "Materialize NULL Leaves"); + } + } + + // TPI and IPI leaf kinds are interleaved in each .debug$T stream. Build an + // object range index for each sorted winner array, then merge the two ranges + // by leaf index inside one task. A compressed object's decode window now sees + // each physical segment once instead of once in the TPI pass and again in the + // IPI pass. Destination pointers were assigned above, so output order and + // bytes remain exactly the same as the separate passes. + if (fuse_materialize) { + for (U32 source = CV_TypeIndexSource_TPI; source <= CV_TypeIndexSource_IPI; ++source) { + LNK_LeafRefArray refs = task.unique_leaf_refs_arr[source]; + U64 *offsets = task.materialize_obj_offsets[source] = push_array_no_zero(scratch.arena, U64, input->count + 1); + U64 cursor = 0; + for EachIndex(obj_idx, input->count) { + offsets[obj_idx] = cursor; + while (cursor < refs.count && lnk_leaf_ref_obj_idx(refs.v[cursor]) == obj_idx) { cursor += 1; } + } + offsets[input->count] = cursor; + Assert(cursor == refs.count); + } + + tp_for_parallel_prof(tp, 0, input->count, lnk_materialize_fused_tpi_ipi_task, &task, "Materialize + Fixup TPI/IPI"); + } + lnk_log(LNK_Log_Timers, "[cobj materialize] fused=%u time=%.3fs", fuse_materialize, + (F64)(now_time_us() - materialize_begin_us) / 1e6); + lnk_compressed_obj_log_phase_stats("after type materialize"); + + if (merge_flags & LNK_MergeTypeFlag_ExportHashes) { + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + LNK_LeafRefArray unique_leaf_refs = task.unique_leaf_refs_arr[ti_source]; + task.ti_source = ti_source; + task.ranges = tp_divide_work(scratch.arena, unique_leaf_refs.count, tp->worker_count); task.result.hashes[ti_source] = push_array_no_zero(tp_temp->v[0], U64, unique_leaf_refs.count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_unbucket_hashes_task, &task, "Export Hashes"); } } + // Streaming-ring P2 slice A: the $S TI/kind fixups no longer patch bytes here. While the + // merge state they consume is still alive -- the assigned-TI hash tables (merge scratch, + // dead at temp_end below) and the materialized IPI leaf copies (released after + // pdb_build_types, i.e. BEFORE the module-write pass) -- record every write into per-input / + // per-obj journals. lnk_write_pdb_modules replays an obj's journal at the start of its + // module-write visit; configs that read fixed-up $S earlier (or never write modules) run + // lnk_apply_debug_s_fixups_eager instead (see lnk.c). if (~merge_flags & LNK_MergeTypeFlag_SkipSymbolTypeFixup) { - tp_for_parallel_prof(tp, 0, input->symbol_input_count, lnk_fixup_symbols_task, &task, "Fixup ID Symbols"); + ProfBegin("Journal $S Fixups"); + // The journal is dead after the last per-obj replay (module write / eager apply) but is + // GB-class at FN scale -- park it on dedicated per-worker arenas (TYPE_MERGE_SCRATCH + // pattern: TP_Arena header + v[] live inside v[0], released last by tp_arena_release) so + // lnk_release_debug_s_fixup_journal can hand the whole set to the background reaper + // instead of the commit riding the link-lifetime TP arena to process exit. + { + Temp temp = temp_begin(scratch.arena); + Arena **arr = push_array(temp.arena, Arena *, tp->worker_count); + for EachIndex(i, tp->worker_count) { arr[i] = arena_alloc(.commit_size = MB(2), .name = "DEBUG_S_FIXUP_JOURNAL"); } + TP_Arena *journal_arenas = push_array(arr[0], TP_Arena, 1); + journal_arenas->count = tp->worker_count; + journal_arenas->v = push_array(arr[0], Arena *, tp->worker_count); + MemoryCopyTyped(journal_arenas->v, arr, tp->worker_count); + input->debug_s_fixup_journal_arenas = journal_arenas; + temp_end(temp); + } + + // NOTE: all per-obj journal arrays span input->count, NOT input->obj_count -- injected + // type-server / .ifc blob pseudo objs live at indices [obj_count, count) in the parallel + // arrays (their $S is empty, so their journals stay empty, but the inline task dispatches + // over input->count and must have a slot to write). + task.journal_arena = input->debug_s_fixup_journal_arenas; + task.debug_s_arr = input->debug_s_arr; + input->debug_s_sym_fixups = push_array(task.journal_arena->v[0], LNK_DebugSPatchArray, input->symbol_input_count ? input->symbol_input_count : 1); + input->debug_s_inline_fixups = push_array(task.journal_arena->v[0], LNK_DebugSInlineJournal, input->count ? input->count : 1); + input->debug_s_sym_fixup_offsets = push_array(task.journal_arena->v[0], U64, input->count + 1); + { + // symbol_inputs are filled per obj at prefix-sum offsets => obj-contiguous, ascending + U64 *counts = push_array(scratch.arena, U64, input->count ? input->count : 1); + for EachIndex(i, input->symbol_input_count) { counts[input->symbol_inputs[i].obj_idx] += 1; } + U64 acc = 0; + for EachIndex(obj_idx, input->count) { input->debug_s_sym_fixup_offsets[obj_idx] = acc; acc += counts[obj_idx]; } + input->debug_s_sym_fixup_offsets[input->count] = acc; + Assert(acc == input->symbol_input_count); + } + tp_for_parallel_prof(tp, 0, input->symbol_patch_task_count, lnk_journal_symbol_fixups_task, &task, "Journal Symbol Fixups"); + tp_for_parallel_prof(tp, 0, input->count, lnk_journal_inline_fixups_task, &task, "Journal Inline Fixups"); + input->has_debug_s_fixup_journal = 1; + lnk_compressed_obj_log_phase_stats("after journal $S fixups"); + + if (lnk_get_log_status(LNK_Log_Debug)) { + U64 entry_count = 0, entry_bytes = 0, wide_runs = 0; + for EachIndex(i, input->symbol_input_count) { + LNK_DebugSPatchArray *a = &input->debug_s_sym_fixups[i]; + entry_count += a->count; entry_bytes += a->count * (a->is_wide ? sizeof(LNK_DebugSPatchWide) : sizeof(LNK_DebugSPatch)); wide_runs += !!a->is_wide; + } + for EachIndex(i, input->count) { + LNK_DebugSPatchArray *a = &input->debug_s_inline_fixups[i].patches; + entry_count += a->count; entry_bytes += a->count * (a->is_wide ? sizeof(LNK_DebugSPatchWide) : sizeof(LNK_DebugSPatch)); wide_runs += !!a->is_wide; + } + U64 assigned_bytes = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { assigned_bytes += task.assigned_ti_arr[s].cap * LNK_ASSIGNED_TI_ENTRY_SIZE; } + U64 leaf_buffer_bytes = task.result.leaf_buffers[CV_TypeIndexSource_TPI].size + task.result.leaf_buffers[CV_TypeIndexSource_IPI].size; + lnk_log(LNK_Log_Debug, "$S fixup journal: %llu entries / %llu bytes (%llu wide runs) (state it decouples from module write: assigned-TI tables %llu bytes, merged leaf buffers %llu bytes)", + entry_count, entry_bytes, wide_runs, assigned_bytes, leaf_buffer_bytes); + } + ProfEnd(); } + for EachIndex(i, tp->worker_count) { lnk_compressed_obj_release_window(&task.decode_windows[i]); } + MemoryCopyTyped(task.result.min_type_indices, input->min_type_indices, CV_TypeIndexSource_COUNT); temp_end(scratch); @@ -2304,76 +5079,34 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) { Temp scratch = scratch_begin(&arena, 1); - LNK_BuildPdb *task = raw_task; - PDB_GsiContext *gsi = task->pdb->gsi; - PDB_PsiContext *psi = task->pdb->psi; - U32Array obj_indices = task->obj_indices[task_id]; + LNK_BuildPdb *task = raw_task; + PDB_GsiContext *gsi = task->pdb->gsi; + PDB_PsiContext *psi = task->pdb->psi; + Assert(gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); + Assert(psi->gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); ProfBegin("Global Symbols"); { - VoidList global_symbols = {0}; - for EachInRange(i, task->cv->symbol_input_ranges[task_id]) { - LNK_SymbolInput symbols = task->cv->symbol_inputs[i]; - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { - void *ptr = cv_ptr_from_symbol(symbol); - void_list_push(scratch.arena, &global_symbols, ptr); - } - - if (cv_is_scope_symbol(symbol.kind)) { - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } - depth -= 1; - } - } - } - - // collect global data and global typedefs - U64 global_symbol_count = tp_sum_u64(tp, task_id, global_symbols.count); - - U64 bucket_cap; - void **buckets; - if (task_id == 0) { - bucket_cap = global_symbol_count * 13 / 10; - buckets = push_array(scratch.arena, void *, bucket_cap); - } - tp_broadcast(&bucket_cap); - tp_broadcast(&buckets); - - // insert symbols into hash table - for EachNode(n, VoidNode, global_symbols.first) { - String8 raw = cv_raw_from_symbol(n->v); - U64 hash = u64_hash_from_str8(raw); - cv_symbol_deduper_insert_or_update(buckets, bucket_cap, hash, n->v); - } - barrier_wait(tp->barrier); - - U64 symbol_count = 0; - void **symbol_arr = 0; // [symbol_count] - Rng1U64 *symbol_ranges = 0; // [worker_count] - U32 *symbol_hashes = 0; // [symbol_count] + // Exact dedup + compaction ran inside the module-write phase while each transformed $S + // window was alive; section backing is already released by the time this pass runs. + // Consume only materialized winner records (pointer array in hash-table slot order, + // payload on surviving per-worker arenas): hash names, size buckets, sharded fill. This + // pass never touches $S. Winner-array order is schedule-dependent, but every GSI bucket + // chain is content-sorted at serialization (gsi_symbol_is_before), so output is stable. + U64 symbol_count = task->gsi_winner_count; + void **symbol_arr = task->gsi_winner_ptrs; // [symbol_count] materialized copies + + Rng1U64 *symbol_ranges = 0; // [worker_count] + U32 *symbol_hashes = 0; // [symbol_count] if (task_id == 0) { - ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); - for EachIndex(src, bucket_cap) { - buckets[symbol_count] = buckets[src]; - symbol_count += buckets[src] != 0; - } - ProfEnd(); - - symbol_arr = buckets; symbol_ranges = tp_divide_work(scratch.arena, symbol_count, tp->worker_count); - symbol_hashes = push_array_no_zero(scratch.arena, U32, symbol_count); + symbol_hashes = push_array_no_zero(scratch.arena, U32, symbol_count ? symbol_count : 1); } - tp_broadcast(&symbol_count); - tp_broadcast(&symbol_arr); tp_broadcast(&symbol_ranges); tp_broadcast(&symbol_hashes); - // hash symbols + // hash symbols (reads the materialized winner bytes -- byte-identical to the $S records + // they were copied from, so every hash value matches the pre-direct-dedup flow) Rng1U64 symbol_range = symbol_ranges[task_id]; for EachInRange(i, symbol_range) { CV_Symbol symbol = cv_symbol_from_ptr(symbol_arr[i]); @@ -2382,222 +5115,318 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); - // push global symbols + // size buckets up front so each one reallocs at most once for this wave (arena pushes are + // single-threaded on task 0; sizing has no determinism impact) if (task_id == 0) { - CV_SymbolNode *nodes = push_array_no_zero(gsi->arena, CV_SymbolNode, symbol_count); + U64 *bucket_adds = push_array(scratch.arena, U64, gsi->bucket_count); + for EachIndex(i, symbol_count) { bucket_adds[symbol_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1)] += 1; } + for EachIndex(bucket_idx, gsi->bucket_count) { + if (bucket_adds[bucket_idx]) { gsi_reserve(gsi, bucket_idx, bucket_adds[bucket_idx]); } + } + } + barrier_wait(tp->barrier); + + // push global symbols, sharded by bucket range: worker i owns buckets [i*B/W, (i+1)*B/W) and + // walks the FULL symbol sequence in global order, inserting only symbols whose bucket lands in + // its range. each bucket has a single owner and receives its inserts in global sequence order, + // so per-bucket order (which is serialized into the PDB) is byte-identical to a serial loop, + // for any worker count -- no locks, no atomics. + { + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, symbol_count) { - CV_SymbolNode *n = &nodes[i]; - n->prev = n->next = 0; - n->data = cv_symbol_from_ptr(symbol_arr[i]); - n->data.offset = i; - gsi_push_(gsi, symbol_hashes[i], n); + U64 bucket_idx = symbol_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1); + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + CV_Symbol *dst = &bucket->v[bucket->count]; + // the bucket value points at the winner's MATERIALIZED copy (made at the end of Write + // Modules, byte-identical to the original record, alive through GSI serialization) -- + // no copy and no $S read here + *dst = cv_symbol_from_ptr(symbol_arr[i]); + // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task + // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content + // hash of the full raw record, never a slot/array position. The winner ARRAY order is a + // dedup-implementation detail (currently compacted open-addressing slot order) and + // per-bucket insert order follows it -- the content-keyed sort is what makes the + // serialized bytes invariant to it. On collision the comparator's kind/data-bytes + // fallback stays content-deterministic, and byte-identical records cannot reach the + // sort (dedup folds them), so the pointer tiebreaker stays unreachable. Hashing the + // materialized copy yields the exact pre-dedup-restructure value (bytes identical). + dst->offset = u64_hash_from_str8(cv_raw_from_symbol(symbol_arr[i])); + bucket->count += 1; } } + barrier_wait(tp->barrier); + if (task_id == 0) { gsi->symbol_count += symbol_count; } } ProfEnd(); ProfBegin("Proc Refs"); { - U64 *proc_ref_sizes = 0; - U64 *proc_ref_counts = 0; + // P2b: proc-refs were pre-built per obj at module write (payloads on the surviving + // procref_payload_arenas); flatten the per-obj segments POSITIONALLY in ascending + // obj-index order -- deterministic for any cohort width or schedule, never completion + // order. (The old flat order was the obj_indices lane concatenation, which was already + // cohort-dependent and relied on the content sort at serialization; obj-index order is + // strictly more deterministic.) Zero $S reads. + U64 total_proc_ref_count = 0; + U64 *procref_offsets = 0; // [obj_count+1] prefix sums in obj-index order + U32 *proc_ref_hashes = 0; // [total_proc_ref_count] + CV_Symbol *proc_ref_symbols = 0; // [total_proc_ref_count] if (task_id == 0) { - proc_ref_sizes = push_array(scratch.arena, U64, tp->worker_count); - proc_ref_counts = push_array(scratch.arena, U64, tp->worker_count); + procref_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); + U64 acc = 0; + for EachIndex(obj_idx, task->cv->obj_count) { + procref_offsets[obj_idx] = acc; + acc += task->preext[obj_idx].procref_count; + } + procref_offsets[task->cv->obj_count] = acc; + total_proc_ref_count = acc; + proc_ref_hashes = push_array_no_zero(scratch.arena, U32, total_proc_ref_count ? total_proc_ref_count : 1); + proc_ref_symbols = push_array_no_zero(scratch.arena, CV_Symbol, total_proc_ref_count ? total_proc_ref_count : 1); } - tp_broadcast(&proc_ref_sizes); - tp_broadcast(&proc_ref_counts); - - U64 proc_ref_size = 0; - U64 proc_ref_count = 0; - for EachIndex(i, obj_indices.count) { - U64 obj_idx = obj_indices.v[i]; - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; - String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + tp_broadcast(&total_proc_ref_count); + tp_broadcast(&procref_offsets); + tp_broadcast(&proc_ref_hashes); + tp_broadcast(&proc_ref_symbols); - if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { - String8 name = cv_name_from_symbol(symbol.kind, symbol.data); - proc_ref_size += AlignPow2(sizeof(CV_SymRef2) + name.size + 1, sizeof(void *)); - proc_ref_count += 1; - } - } - } + for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + if (pre->procref_count == 0) { continue; } + MemoryCopyTyped(&proc_ref_symbols[procref_offsets[obj_idx]], pre->procref_syms, pre->procref_count); + MemoryCopyTyped(&proc_ref_hashes [procref_offsets[obj_idx]], pre->procref_hashes, pre->procref_count); } - proc_ref_sizes[task_id] = proc_ref_size; - proc_ref_counts[task_id] = proc_ref_count; barrier_wait(tp->barrier); - U64 total_proc_ref_size = tp_sum_u64(tp, task_id, proc_ref_size); - U64 total_proc_ref_count = tp_sum_u64(tp, task_id, proc_ref_count); - - U64 *proc_ref_hashes = 0; - U64 *proc_ref_indices = 0; - Arena **proc_ref_arenas = 0; - CV_SymbolNode *proc_ref_nodes = 0; + // size buckets up front so each one reallocs at most once for this wave (arena pushes are + // single-threaded on task 0; sizing has no determinism impact) if (task_id == 0) { - proc_ref_hashes = push_array(scratch.arena, U64, total_proc_ref_count); - proc_ref_indices = offsets_from_counts_array_u64(scratch.arena, proc_ref_counts, tp->worker_count); - proc_ref_arenas = alloc_arena_many(gsi->arena, tp->worker_count, proc_ref_sizes); - proc_ref_nodes = push_array(gsi->arena, CV_SymbolNode, total_proc_ref_count); - } - tp_broadcast(&proc_ref_hashes); - tp_broadcast(&proc_ref_indices); - tp_broadcast(&proc_ref_arenas); - tp_broadcast(&proc_ref_nodes); - - Arena *proc_ref_arena = proc_ref_arenas[task_id]; - U64 proc_ref_idx = proc_ref_indices[task_id]; - for EachIndex(i, obj_indices.count) { - U64 obj_idx = obj_indices.v[i]; - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; - String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - CV_ModIndex imod = task->mod_arr[obj_idx]->imod; - U64 symbol_cursor = sizeof(CV_Signature); - U64 scope_depth = 0; - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - - if (symbol.kind == CV_SymKind_SKIP) { continue; } - else if (cv_is_global_symbol(symbol.kind)) { continue; } - else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } - else if (symbol.kind == 0x1176) { continue; } - - if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } - else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } - - if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { - String8 name = cv_name_from_symbol(symbol.kind, symbol.data); - proc_ref_nodes [proc_ref_idx].data = cv_make_proc_ref(proc_ref_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); - proc_ref_nodes [proc_ref_idx].data.offset = symbol_cursor; - proc_ref_hashes[proc_ref_idx] = gsi_hash(gsi, name); - proc_ref_idx += 1; - } - - symbol_cursor += cv_write_symbol_buf(0, 0, &symbol, PDB_SYMBOL_ALIGN); - } + U64 *bucket_adds = push_array(scratch.arena, U64, gsi->bucket_count); + for EachIndex(i, total_proc_ref_count) { bucket_adds[proc_ref_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1)] += 1; } + for EachIndex(bucket_idx, gsi->bucket_count) { + if (bucket_adds[bucket_idx]) { gsi_reserve(gsi, bucket_idx, bucket_adds[bucket_idx]); } } } barrier_wait(tp->barrier); - // push proc refs - if (task_id == 0) { - U64 total_proc_ref_count = sum_array_u64(tp->worker_count, proc_ref_counts); - for EachIndex(i, total_proc_ref_count) { gsi_push_(gsi, proc_ref_hashes[i], &proc_ref_nodes[i]); } + // push proc refs, sharded by bucket range (single owner per bucket, inserts in global + // order -> per-bucket order identical to a serial loop for any worker count) + { + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; + for EachIndex(i, total_proc_ref_count) { + U64 bucket_idx = proc_ref_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1); + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + bucket->v[bucket->count] = proc_ref_symbols[i]; + bucket->count += 1; + } } barrier_wait(tp->barrier); + if (task_id == 0) { gsi->symbol_count += total_proc_ref_count; } } ProfEnd(); + // Patched debug-section copies were released at the end of the module-write phase; nothing + // here reads $S. Global winner bytes and the procref value/hash arrays + payloads consumed + // above live on the surviving procref_payload_arenas through GSI serialization. + ProfBegin("Public Symbols"); { - U64 *public_symbol_sizes = 0; // [worker_count] - U64 *public_symbol_node_counts = 0; // [worker_count] + // FAIR-SHARE: task->symtab->chunks is a FIXED [symtab->arena->count] partition built at + // full pool width, but this barrier pass runs at the pinned cohort C == tp->worker_count + // (C <= fixed). Walk the fixed lanes strided by the cohort so every fixed lane is + // processed exactly once for any C, and keep every per-lane array in FIXED-lane order so + // the flattened global order below (which feeds the sharded PSI insert) is byte-identical + // to a full-width run. At C == fixed the strided loops degenerate to lane == task_id. + U64 fixed_lane_count = task->symtab->arena->count; + + U64 *public_symbol_sizes = 0; // [fixed_lane_count] + U64 *public_symbol_node_counts = 0; // [fixed_lane_count] if (task_id == 0) { - public_symbol_sizes = push_array(scratch.arena, U64, tp->worker_count); - public_symbol_node_counts = push_array(scratch.arena, U64, tp->worker_count); + public_symbol_sizes = push_array(scratch.arena, U64, fixed_lane_count); + public_symbol_node_counts = push_array(scratch.arena, U64, fixed_lane_count); } tp_broadcast(&public_symbol_sizes); tp_broadcast(&public_symbol_node_counts); // compute buffer size for CV public symbols - LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[task_id]; - U64 public_symbol_size = 0; - U64 public_symbol_count = 0; - for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { - for EachIndex(i, chunk->count) { - LNK_Symbol *symbol = chunk->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); - - if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); - if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } - - public_symbol_size += AlignPow2(sizeof(CV_SymPub32) + symbol->name.size + 1, sizeof(void *)); - public_symbol_count += 1; - public_symbol_node_counts[task_id] += 1; - } - } - public_symbol_sizes [task_id] += public_symbol_size; - public_symbol_node_counts[task_id] += public_symbol_count; + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; + U64 public_symbol_size = 0; + U64 public_symbol_count = 0; + for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { + for EachIndex(i, chunk->count) { + LNK_Symbol *symbol = chunk->v[i].symbol; + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); + + if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } + + public_symbol_size += AlignPow2(sizeof(CV_SymPub32) + symbol->name.size + 1, sizeof(void *)); + public_symbol_count += 1; + public_symbol_node_counts[lane] += 1; + } + } + public_symbol_sizes [lane] += public_symbol_size; + public_symbol_node_counts[lane] += public_symbol_count; + } barrier_wait(tp->barrier); Arena **public_symbol_arenas = 0; - Arena **public_symbol_node_arenas = 0; - CV_SymbolList *public_symbols = 0; // [worker_count] - U32 **public_symbol_hashes = 0; // [worker_count][public_symbol.count] + CV_Symbol **public_symbol_vals = 0; // [fixed_lane_count][public_symbol_lane_counts[lane]] + U64 *public_symbol_lane_counts = 0; // [fixed_lane_count] + U32 **public_symbol_hashes = 0; // [fixed_lane_count][public_symbol_lane_counts[lane]] if (task_id == 0) { - U64 public_symbol_total_count = sum_array_u64(tp->worker_count, public_symbol_node_counts); - public_symbol_arenas = alloc_arena_many(psi->gsi->arena, tp->worker_count, public_symbol_sizes); - public_symbol_node_arenas = alloc_arena_array(psi->gsi->arena, tp->worker_count, public_symbol_node_counts, CV_SymbolNode); - public_symbols = push_array(scratch.arena, CV_SymbolList, tp->worker_count); - public_symbol_hashes = push_array(scratch.arena, U32 *, tp->worker_count); + public_symbol_arenas = alloc_arena_many(psi->gsi->arena, fixed_lane_count, public_symbol_sizes); + public_symbol_vals = push_array(scratch.arena, CV_Symbol *, fixed_lane_count); + public_symbol_lane_counts = push_array(scratch.arena, U64, fixed_lane_count); + public_symbol_hashes = push_array(scratch.arena, U32 *, fixed_lane_count); } tp_broadcast(&public_symbol_arenas); - tp_broadcast(&public_symbol_node_arenas); - tp_broadcast(&public_symbols); + tp_broadcast(&public_symbol_vals); + tp_broadcast(&public_symbol_lane_counts); tp_broadcast(&public_symbol_hashes); - // make CV public symbols - Arena *public_symbol_arena = public_symbol_arenas [task_id]; - Arena *public_symbol_node_arena = public_symbol_node_arenas[task_id]; - CV_SymbolList *public_symbol_list = &public_symbols [task_id]; - for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { - for EachIndex(i, chunk->count) { - LNK_Symbol *symbol = chunk->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + // make CV public symbols (per-lane CV_Symbol value arrays on scratch; payload bytes stay on + // public_symbol_arenas). lane arrays live on the owning worker's scratch and are only read by + // the same lane->worker striding below, then copied into the flat array. + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; + Arena *public_symbol_arena = public_symbol_arenas[lane]; + CV_Symbol *vals = push_array_no_zero(scratch.arena, CV_Symbol, public_symbol_node_counts[lane]); + U64 val_count = 0; + for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { + for EachIndex(i, chunk->count) { + LNK_Symbol *symbol = chunk->v[i].symbol; + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); + + // discard removed and non-section symbols + if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } + + CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; + ISectOff sc = lnk_sc_from_symbol(symbol); + Assert(val_count < public_symbol_node_counts[lane]); + vals[val_count++] = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); + } + } + public_symbol_vals [lane] = vals; + public_symbol_lane_counts[lane] = val_count; + } + barrier_wait(tp->barrier); - // discard removed and non-section symbols - if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); - if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } + // hash public symbols + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + U64 lane_count = public_symbol_lane_counts[lane]; + CV_Symbol *vals = public_symbol_vals[lane]; + U32 *hashes = push_array(scratch.arena, U32, lane_count); + for EachIndex(k, lane_count) { + String8 name = cv_name_from_symbol(vals[k].kind, vals[k].data); + hashes[k] = gsi_hash(gsi, name); + } + public_symbol_hashes[lane] = hashes; + } + barrier_wait(tp->barrier); - CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; - ISectOff sc = lnk_sc_from_symbol(symbol); - CV_Symbol pub_symbol = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); - cv_symbol_list_push(public_symbol_node_arena, public_symbol_list, pub_symbol); + // flatten the per-worker symbol arrays (in worker order, matching the old serial walk) into + // one global-order value/hash array, so the sharded insert below can walk it + U64 public_symbol_total_count = 0; + U64 *public_symbol_offsets = 0; // [fixed_lane_count] + CV_Symbol *public_symbol_flat_vals = 0; // [public_symbol_total_count] + U32 *public_symbol_flat_hashes = 0; // [public_symbol_total_count] + if (task_id == 0) { + U64 *list_counts = push_array_no_zero(scratch.arena, U64, fixed_lane_count); + for EachIndex(i, fixed_lane_count) { list_counts[i] = public_symbol_lane_counts[i]; } + public_symbol_offsets = offsets_from_counts_array_u64(scratch.arena, list_counts, fixed_lane_count); + public_symbol_total_count = sum_array_u64(fixed_lane_count, list_counts); + public_symbol_flat_vals = push_array_no_zero(scratch.arena, CV_Symbol, public_symbol_total_count); + public_symbol_flat_hashes = push_array_no_zero(scratch.arena, U32, public_symbol_total_count); + } + tp_broadcast(&public_symbol_total_count); + tp_broadcast(&public_symbol_offsets); + tp_broadcast(&public_symbol_flat_vals); + tp_broadcast(&public_symbol_flat_hashes); + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + U64 cursor = public_symbol_offsets[lane]; + U64 lane_count = public_symbol_lane_counts[lane]; + for EachIndex(k, lane_count) { + public_symbol_flat_vals [cursor] = public_symbol_vals [lane][k]; + public_symbol_flat_hashes[cursor] = public_symbol_hashes[lane][k]; + cursor += 1; } } barrier_wait(tp->barrier); - // hash public symbols - { - U64 hash_idx = 0; - U32 *hashes = push_array(scratch.arena, U32, public_symbols[task_id].count); - for EachNode(n, CV_SymbolNode, public_symbols[task_id].first) { - String8 name = cv_name_from_symbol(n->data.kind, n->data.data); - hashes[hash_idx++] = gsi_hash(gsi, name); + // size buckets up front so each one reallocs at most once for this wave (arena pushes are + // single-threaded on task 0; sizing has no determinism impact) + if (task_id == 0) { + PDB_GsiContext *pub_gsi = psi->gsi; + U64 *bucket_adds = push_array(scratch.arena, U64, pub_gsi->bucket_count); + for EachIndex(i, public_symbol_total_count) { bucket_adds[public_symbol_flat_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1)] += 1; } + for EachIndex(bucket_idx, pub_gsi->bucket_count) { + if (bucket_adds[bucket_idx]) { gsi_reserve(pub_gsi, bucket_idx, bucket_adds[bucket_idx]); } } - public_symbol_hashes[task_id] = hashes; } barrier_wait(tp->barrier); - // insert public symbols into PSI - if (task_id == 0) { - for EachIndex(i, tp->worker_count) { - U64 k = 0; - for (CV_SymbolNode *curr = public_symbols[i].first, *next = 0; curr != 0; curr = next, k += 1) { - next = curr->next; - curr->next = 0; - gsi_push_(psi->gsi, public_symbol_hashes[i][k], curr); - } + // insert public symbols into PSI, sharded by bucket range (single owner per bucket, inserts in + // global order -> per-bucket order identical to a serial loop for any worker count) + { + PDB_GsiContext *pub_gsi = psi->gsi; + U64 shard_min = (task_id * pub_gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * pub_gsi->bucket_count) / tp->worker_count; + for EachIndex(i, public_symbol_total_count) { + U64 bucket_idx = public_symbol_flat_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1); + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + PDB_GsiSymbolBucket *bucket = &pub_gsi->bucket_arr[bucket_idx]; + bucket->v[bucket->count] = public_symbol_flat_vals[i]; + bucket->count += 1; } } barrier_wait(tp->barrier); + if (task_id == 0) { psi->gsi->symbol_count += public_symbol_total_count; } } ProfEnd(); scratch_end(scratch); } + +// Streaming-ring P1.2 parity walk (debug builds only -- it doubles the reads): prove the +// dormant provenance recorded at parse time is authoritative by re-resolving every tracked +// subsection node through lnk_resolve_debug_s_node and comparing CONTENT against the node's +// String8. For reloc-PATCHED sections both the node slice and the resolver point into the +// same section_data_copies bytes (pointers may even be equal), so the assert is on content, +// which also holds across the in-place $S TI/kind fixups (they mutate the shared bytes). +// Skips untracked lists (prov count == 0: wholesale synthetic constructions) and synthetic +// nodes (no backing section). Valid only while lnk_obj_section_data_from_number still returns what the +// parse consumed, i.e. before the sect-data copies release in lnk_move_global_symbols_to_gsi. +internal void +lnk_assert_debug_s_prov_parity(LNK_Obj *obj, CV_DebugS *debug_s) +{ +#if BUILD_DEBUG + for EachElement(k, debug_s->data_list) { + if (debug_s->prov_list[k].count == 0) { continue; } // untracked construction + Assert(debug_s->prov_list[k].count == debug_s->data_list[k].node_count); + CV_DebugSProvNode *prov = debug_s->prov_list[k].first; + for (String8Node *data_n = debug_s->data_list[k].first; data_n != 0; data_n = data_n->next, prov = prov->next) { + String8 resolved = lnk_resolve_debug_s_node(obj, prov); + if (prov->is_synthetic) { Assert(resolved.size == 0); continue; } + Assert(resolved.size == data_n->string.size); + Assert(str8_match(resolved, data_n->string, 0)); + } + Assert(prov == 0); + } +#endif +} + internal U64 -lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8Node *buf, U64 *buf_pos) +lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8Node *buf, U64 *buf_pos, LNK_GsiPreExtractObj *pre) { U64 mod_cursor = 0; @@ -2614,24 +5443,63 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No mod->sym_data_size += sig_size; mod_cursor += sig_size; - // write symbols - U64 scope_depth = 0; - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + U64 symbols_idx = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_Symbols); + CV_DebugSProvList *symbol_prov = &debug_s.prov_list[symbols_idx]; + B32 use_summary = (buf == 0 && pre != 0 && symbol_prov->count == symbols.node_count); + if (use_summary) { + for (CV_DebugSProvNode *prov = symbol_prov->first; prov != 0; prov = prov->next) { + if (!prov->symbol_summary_valid) { use_summary = 0; break; } + } + } + if (use_summary) { + for (CV_DebugSProvNode *prov = symbol_prov->first; prov != 0; prov = prov->next) { + mod_cursor += prov->module_symbol_size; + mod->sym_data_size += prov->module_symbol_size; + pre->cand_count += prov->gsi_candidate_count; + pre->procref_count += prov->proc_ref_count; + } + } else { + // write symbols + U64 scope_depth = 0; + for EachNode(n, String8Node, symbols.first) { + U64 cand_depth = 0; + B32 cand_active = 1; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + + if (pre != 0) { + if (cand_active && (cv_is_global_symbol(symbol.kind) || (cand_depth == 0 && cv_is_typedef(symbol.kind)))) { + pre->cand_count += 1; + } + if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32 || + symbol.kind == CV_SymKind_GPROC32_ID || symbol.kind == CV_SymKind_LPROC32_ID) { + pre->procref_count += 1; + } + if (cand_active) { + if (cv_is_scope_symbol(symbol.kind)) { + cand_depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (cand_depth == 0) { Assert(0 && "malformed symbol stream"); cand_active = 0; } + else { cand_depth -= 1; } + } + } + } - if (symbol.kind == CV_SymKind_SKIP) { continue; } - else if (cv_is_global_symbol(symbol.kind)) { continue; } - else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } - else if (symbol.kind == 0x1176) { continue; } + if (symbol.kind == CV_SymKind_SKIP) { continue; } + else if (cv_is_global_symbol(symbol.kind)) { continue; } + else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } + else if (symbol.kind == 0x1176) { continue; } - if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } - else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } + if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } + else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } - U64 symbol_size = cv_write_symbol_buf(buf, buf_pos, &symbol, PDB_SYMBOL_ALIGN); - mod_cursor += symbol_size; - mod->sym_data_size += symbol_size; + String8 raw_symbol = cv_raw_from_symbol(cv_ptr_from_symbol(symbol)); + U64 symbol_size = (raw_symbol.size & (PDB_SYMBOL_ALIGN - 1)) == 0 ? str8_buffer_write(buf, buf_pos, raw_symbol) + : cv_write_symbol_buf(buf, buf_pos, &symbol, PDB_SYMBOL_ALIGN); + mod_cursor += symbol_size; + mod->sym_data_size += symbol_size; + } } } } @@ -2685,6 +5553,128 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No return mod_cursor; } +typedef struct LNK_PdbOutput LNK_PdbOutput; +internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn); + +#define LNK_GSI_DEDUP_RESERVED ((void *)(U64)1) + +// Exact concurrent content set. Claiming an empty slot with a sentinel before allocating is +// important: only the thread that adds a distinct record copies bytes, so the common duplicate +// case consumes neither transient metadata nor discarded arena space. Published pointers own +// immutable bytes on a per-worker surviving arena and are safe to compare immediately. +internal void +lnk_gsi_deduper_insert_copy(void **buckets, U64 bucket_cap, U64 hash, void *symbol_ptr, Arena *dst_arena) +{ + String8 raw = cv_raw_from_symbol(symbol_ptr); + U64 best_idx = lnk_hash_range(hash, bucket_cap); + U64 idx = best_idx; + for (;;) { + void *curr = ins_atomic_ptr_eval(&buckets[idx]); + if (curr == 0) { + void *cmp = ins_atomic_ptr_eval_cond_assign(&buckets[idx], LNK_GSI_DEDUP_RESERVED, 0); + if (cmp == 0) { + U8 *copy = push_array_no_zero(dst_arena, U8, raw.size); + MemoryCopy(copy, raw.str, raw.size); + ins_atomic_ptr_eval_assign(&buckets[idx], copy); + return; + } + curr = cmp; + } + while (curr == LNK_GSI_DEDUP_RESERVED) { curr = ins_atomic_ptr_eval(&buckets[idx]); } + String8 seen = cv_raw_from_symbol(curr); + if (seen.size == raw.size && MemoryMatch(seen.str, raw.str, raw.size)) { return; } + idx = (idx + 1 == bucket_cap) ? 0 : idx + 1; + Assert(idx != best_idx); + } +} + +// Streaming-ring P2b: extract the obj's GSI inputs -- direct global-record winners and +// proc-refs -- inside the module-write per-obj visit, so lnk_move_global_symbols_to_gsi does +// no $S record-decode walks (its only remaining $S touch is the bucket-fill materialize of +// the dedup winners' bytes through the refs). Runs right after the obj's deferred $S fixup +// replay, so records are post-fixup (identical bytes to what the old post-modules collect +// walk saw). +// +// Faithful to the two walks it replaces: +// - candidate walk: per-NODE scope depth reset + malformed-end break (the old collect ran per +// symbol input, and symbol inputs are exactly the Symbols data_list nodes); +// - proc-ref walk: obj-continuous scope depth + module-stream cursor (starts at +// sizeof(CV_Signature); records the module write drops -- SKIP, globals, top-level typedefs, +// 0x1176 -- do not advance it). +// Exact global winners plus proc-ref value/hash arrays and payloads go on the surviving +// procref_payload_arenas (referenced until GSI serialization), same lifetime as the old +// proc_ref_arenas. +internal void +lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_DebugS *debug_s_ptr) +{ + CV_DebugS debug_s = *debug_s_ptr; // window copy (g_debug_s_window) or the patched backing + String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + if (symbols.total_size == 0) { return; } + + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + PDB_GsiContext *gsi = task->pdb->gsi; + Arena *payload_arena = task->procref_payload_arenas[task_id]; + + // Dedup globals and build proc refs in one decode walk over the post-reloc/post-fixup + // window. The size-only pass counted both record sets from invariant raw headers, so all + // arrays and the shared table are exact-sized before this visit. + U64 procref_count = pre->procref_count; + if (procref_count) { + pre->procref_syms = push_array_no_zero(payload_arena, CV_Symbol, procref_count); + pre->procref_hashes = push_array_no_zero(payload_arena, U32, procref_count); + } + + CV_ModIndex imod = task->mod_arr[obj_idx]->imod; + U64 symbol_cursor = sizeof(CV_Signature); + U64 scope_depth = 0; + U64 cand_count = 0; + U64 procref_idx = 0; + for EachNode(n, String8Node, symbols.first) { + U64 cand_depth = 0; + B32 cand_active = 1; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + + if (cand_active && (cv_is_global_symbol(symbol.kind) || (cand_depth == 0 && cv_is_typedef(symbol.kind)))) { + U8 *ptr = cv_ptr_from_symbol(symbol); + lnk_gsi_deduper_insert_copy(task->gsi_dedup_buckets, task->gsi_dedup_bucket_cap, + u64_hash_from_str8(cv_raw_from_symbol(ptr)), ptr, payload_arena); + cand_count += 1; + } + if (cand_active) { + if (cv_is_scope_symbol(symbol.kind)) { + cand_depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (cand_depth == 0) { Assert(0 && "malformed symbol stream"); cand_active = 0; } + else { cand_depth -= 1; } + } + } + + B32 is_module_symbol = (symbol.kind != CV_SymKind_SKIP && + !cv_is_global_symbol(symbol.kind) && + !(cv_is_typedef(symbol.kind) && scope_depth == 0) && + symbol.kind != 0x1176); + if (is_module_symbol) { + if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } + else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } + + if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { + String8 name = cv_name_from_symbol(symbol.kind, symbol.data); + pre->procref_syms[procref_idx] = cv_make_proc_ref(payload_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); + pre->procref_syms[procref_idx].offset = symbol_cursor; + pre->procref_hashes[procref_idx] = gsi_hash(gsi, name); + procref_idx += 1; + } + + symbol_cursor += cv_write_symbol_buf(0, 0, &symbol, PDB_SYMBOL_ALIGN); + } + } + } + Assert(cand_count == pre->cand_count); + Assert(procref_idx == procref_count); +} + internal THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) { @@ -2696,12 +5686,29 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // compute sizes for module streams for EachIndex(i, obj_indices.count) { U64 obj_idx = obj_indices.v[i]; - lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); + // P3.4: sizing needs NO window fill. Every term of the module stream size is either pure + // subsection metadata (C13 layout kinds come from the data_list INDEX, sizes/alignment + // from node sizes) or -- for sym_data_size -- a read-only walk of the Symbols records' + // size prefixes + kinds. Both are invariant under the window transforms: relocs and TI + // fixups never touch record size/kind fields, and the journal's kind REWRITES are + // classification-invariant for every predicate the sizing walk uses + // (cv_is_scope_symbol treats GPROC32_ID/LPROC32_ID like their non-ID rewrites, + // cv_is_end_symbol treats PROC_ID_END like END, and no rewrite produces or removes a + // global/typedef/SKIP/0x1176 kind). So size straight off the raw mapped view under + // g_debug_s_window (the patched backing otherwise) -- the write pass below asserts the + // re-accumulated sizes match. Fixup replay, parity, and the GSI extraction all move to + // the write pass's single fill. + lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0, &task->preext[obj_idx]); } barrier_wait(tp->barrier); - // alloc module streams + // Allocate module streams and the exact global-symbol content set. Candidate count depends + // only on record headers, so this table is ready before the one transformed window visit. if (task_id == 0) { + U64 cand_count = 0; + for EachIndex(obj_idx, task->cv->obj_count) { cand_count += task->preext[obj_idx].cand_count; } + task->gsi_dedup_bucket_cap = Max(cand_count * 13 / 10, 1); + task->gsi_dedup_buckets = push_array(scratch.arena, void *, task->gsi_dedup_bucket_cap); for EachIndex(obj_idx, task->cv->obj_count) { PDB_DbiModule *mod = task->mod_arr[obj_idx]; U64 mod_size = mod->sym_data_size + mod->c11_data_size + mod->c13_data_size + mod->globrefs_size; @@ -2719,15 +5726,42 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) U64 obj_idx = obj_indices.v[i]; PDB_DbiModule *mod = task->mod_arr[obj_idx]; - if (mod->sn == MSF_INVALID_STREAM_NUMBER) { continue; } + if (mod->sn == MSF_INVALID_STREAM_NUMBER) { +#if BUILD_DEBUG + // an obj skipped here must have NO Symbols payload -- otherwise sym_data_size >= sig + // would have allocated a stream -- so skipping the extraction below loses nothing + Assert(cv_sub_section_from_debug_s(task->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols).total_size == 0); +#endif + temp_end(temp); + continue; + } + + // P3.4: THE single window fill per obj in module write (sizing above reads raw metadata/ + // headers only). Copy mode ( /OPT:GCTYPES ) consumes the patched backing; its journal was + // consumed eagerly, so the replay call is a structural no-op. + CV_DebugS debug_s; + if (g_debug_s_window) { + debug_s = lnk_obj_window_debug_s(temp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 0); + } else { + lnk_apply_debug_s_fixups_for_obj(task->cv, obj_idx); + debug_s = task->cv->debug_s_arr[obj_idx]; + } + lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity + // P2b: globals-candidate + proc-ref extraction fused into this (now only) fill + lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id, &debug_s); - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; - String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); + String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); if (mod_data.node_count) { + // raw-header sizing vs post-transform write must agree byte-for-byte + U64 size_check[4] = { mod->sym_data_size, mod->c11_data_size, mod->c13_data_size, mod->globrefs_size }; String8Node buf = *mod_data.first; U64 pos = 0; - lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos); + lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos, 0); + AssertAlways(mod->sym_data_size == size_check[0]); + AssertAlways(mod->c11_data_size == size_check[1]); + AssertAlways(mod->c13_data_size == size_check[2]); + AssertAlways(mod->globrefs_size == size_check[3]); // sub range symbol data pages and patch symbol tree offsets if (mod->sym_data_size) { @@ -2828,12 +5862,95 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // collect mod source files String8List source_file_list = str8_split_by_string_chars(string_arenas[task_id], string_table, str8_lit("\0"), 0); + { + // the split nodes alias the string table inside the obj's $S backing (str8_split + // does not copy) and DBI file-info hashes these bytes after the backing dies -- + // whether that backing is a debug-section COPY (released at the end of this phase) + // or a reloc-free RAW-MAPPED view (alive today, dies in P5). P3.1: repoint + // UNCONDITIONALLY. Every piece is present in string_ht -- the dedup task split the + // very same table -- and its bucket bytes were rehomed to a surviving blob before + // the strtab add, so repoint at the bucket's copy (byte-identical). + for EachNode(n, String8Node, source_file_list.first) { + CV_StringBucket *bucket = cv_string_hash_table_lookup(task->string_ht, n->string); + Assert(bucket != 0); + if (bucket != 0) { n->string = bucket->string; } + } + } str8_list_concat_in_place(&mod->source_file_list, &source_file_list); temp_end(temp); + + // the module stream is final (symbols + C13 written, checksum name + // offsets patched): hand it to the background writer NOW so its pages + // flush + decommit while the remaining modules are still being written + // -- enqueueing all modules after the pass kept the entire module + // payload (GB-class) committed through the globals phase + if (task->output != 0) { + lnk_pdb_output_enqueue_stream(task->output, task->pdb->msf, mod->sn); + } + + } + barrier_wait(tp->barrier); + } + + // Compact the exact winners published during the one module-write window visit. Bucket + // order is deterministic for a fixed content set; downstream GSI buckets content-sort + // records before serialization, so the pointer-array order is byte-invisible. + ProfBegin("Compact Global Symbol Winners"); + { + U64 *compact_counts = 0; + U64 winner_count = 0; + void **winner_ptrs = 0; + if (task_id == 0) { compact_counts = push_array(scratch.arena, U64, tp->worker_count); } + tp_broadcast(&compact_counts); + + U64 slot_lo = (task_id * task->gsi_dedup_bucket_cap) / tp->worker_count; + U64 slot_hi = ((task_id + 1) * task->gsi_dedup_bucket_cap) / tp->worker_count; + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { + void *p = task->gsi_dedup_buckets[slot]; + Assert(p != LNK_GSI_DEDUP_RESERVED); + compact_counts[task_id] += (p != 0); + } + barrier_wait(tp->barrier); + + for EachIndex(w, tp->worker_count) { winner_count += compact_counts[w]; } + if (task_id == 0) { + winner_ptrs = push_array_no_zero(task->pdb->gsi->arena, void *, winner_count ? winner_count : 1); + } + tp_broadcast(&winner_ptrs); + + U64 dst = 0; + for (U64 w = 0; w < task_id; w += 1) { dst += compact_counts[w]; } + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { + void *p = task->gsi_dedup_buckets[slot]; + if (p != 0) { winner_ptrs[dst++] = p; } } barrier_wait(tp->barrier); + if (task_id == 0) { + task->gsi_winner_count = winner_count; + task->gsi_winner_ptrs = winner_ptrs; + } } + ProfEnd(); + + // Copies release: moved here from mid-globals. Direct winner materialization above was + // the LAST $S reader on this path -- release the patched debug-section copies before the + // globals/publics staging begins instead of overlapping it. Same /PDBSTRIPPED gate as + // before (its second build re-walks cv->debug_s_arr Symbols after lnk_build_pdb returns; + // the materialize still ran -- winner ptrs never dangle either way). The barrier above + // guarantees every worker is done reading before task 0 releases section copies. + if (task_id == 0) { + if (task->free_sect_copies) { + ProfScope("Release Sect Data Copies") { + for EachIndex(obj_idx, task->cv->obj_count) { + lnk_obj_drop_section_data_copies(task->cv->obj_arr[obj_idx]); + } + for EachIndex(i, g_sect_copy_arena_count) { + if (g_sect_copy_arenas[i] != 0) { arena_release(g_sect_copy_arenas[i]); g_sect_copy_arenas[i] = 0; } + } + } + } + } scratch_end(scratch); } @@ -2905,14 +6022,15 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task) } } -typedef struct +struct LNK_PdbOutput { LNK_BackgroundFileWriter *writer; LNK_BackgroundFile *file; MSF_StreamNumber *sealed_streams; U64 sealed_stream_count; U64 sealed_stream_cap; -} LNK_PdbOutput; + B32 decommit_flushed; // off when /RAD_DEBUG re-reads the PDB page memory for RDI conversion +}; typedef struct LNK_MsfPageCursor { @@ -2941,8 +6059,13 @@ internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn) { if (sn == MSF_INVALID_STREAM_NUMBER) { return; } - Assert(output->sealed_stream_count < output->sealed_stream_cap); - output->sealed_streams[output->sealed_stream_count++] = sn; + // atomic slot: module streams enqueue from pool workers as each module + // finishes; order in sealed_streams is irrelevant (set semantics for the + // remaining-pages bitmap), file writes are positional, so output bytes are + // unaffected by completion order + U64 slot = ins_atomic_u64_add_eval(&output->sealed_stream_count, 1) - 1; + Assert(slot < output->sealed_stream_cap); + output->sealed_streams[slot] = sn; MSF_Stream *stream = msf_find_stream(msf, sn); Assert(stream != 0); @@ -2963,7 +6086,7 @@ lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_Strea run_size += msf->page_size; } else { if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size), output->decommit_flushed); } run_first_pn = run_last_pn = page->pn; run_data = page_data; @@ -2971,7 +6094,7 @@ lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_Strea } } if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size), output->decommit_flushed); } } @@ -2981,6 +6104,17 @@ lnk_pdb_output_finalize_stream(void *user_data, MSF_Context *msf, MSF_StreamNumb lnk_pdb_output_enqueue_stream(user_data, msf, sn); } +// Timers telemetry: cumulative enqueued/completed bytes at a named point in the build +internal void +lnk_pdb_output_log_mark(LNK_PdbOutput *output, char *tag) +{ + if (output == 0) { return; } + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs %s: enq=%.2f GiB done=%.2f GiB", + (F64)(now_time_us() - output->writer->begin_time_us) / 1e6, tag, + (F64)ins_atomic_u64_eval(&output->writer->bytes_enqueued) / GB(1), + (F64)ins_atomic_u64_eval(&output->writer->bytes_completed) / GB(1)); +} + internal void lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf) { @@ -3013,26 +6147,411 @@ lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf) run_size += page_size; } else { if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size), /*decommit*/ 0); // MSF metadata (FPM/header/stream table) is read after this enqueue } run_first_pn = pn; run_data = page_data; run_size = page_size; } } else if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size), /*decommit*/ 0); // MSF metadata (FPM/header/stream table) is read after this enqueue run_data = 0; run_size = 0; } } if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size), /*decommit*/ 0); // MSF metadata (FPM/header/stream table) is read after this enqueue + } + scratch_end(scratch); +} + +//////////////////////////////// +// Type Garbage Collection +// +// After type merging, prune merged TPI/IPI leaves that are not reachable from any surviving +// symbol record (the GC roots), compact them, and remap all type indices. link.exe keeps a +// large unreferenced-type set; pruning it is a transparent PDB-size win (debug-info only -- the +// image is untouched). Runs on the final post-fixup type indices in place. + +typedef struct LNK_GCTypes +{ + LNK_CodeViewInput *cv; + U64 min [CV_TypeIndexSource_COUNT]; // first type index per source + U64 orig_n[CV_TypeIndexSource_COUNT]; // pre-GC leaf count per source + U8 *mark [CV_TypeIndexSource_COUNT]; // reachable bitmap, indexed by (ti - min) + CV_TypeIndex *remap [CV_TypeIndexSource_COUNT]; // old leaf idx -> new type index + U8 **leaf_v [CV_TypeIndexSource_COUNT]; // original leaf pointer arrays + U32 *udt_next; // TPI fwdref<->definition unique_name ring + Rng1U64 *sym_ranges; + B32 do_rewrite; // 0 = mark roots, 1 = rewrite to compacted indices + // transitive-closure frontier: indices marked but not yet expanded. Each leaf is appended once + // (the atomic mark gates it), so frontier[s] is sized orig_n[s] and fcount[s] is its atomic tail. + U32 *frontier[CV_TypeIndexSource_COUNT]; + U32 *fcount [CV_TypeIndexSource_COUNT]; // atomic append cursor per source + // per-source scratch (set before dispatch) + CV_TypeIndexSource cur_source; + U8 **cur_leaf_v; + Rng1U64 *cur_ranges; + U64 round_begin, round_end; // frontier slice processed this round +} LNK_GCTypes; + +typedef struct LNK_GCNamePair { U64 hash; U32 idx; } LNK_GCNamePair; + +internal int +lnk_gc_name_pair_is_before(void *raw_a, void *raw_b) +{ + LNK_GCNamePair *a = raw_a, *b = raw_b; + return a->hash != b->hash ? (a->hash < b->hash) : (a->idx < b->idx); +} + +internal void +lnk_gc_mark_ti(LNK_GCTypes *g, CV_TypeIndexSource s, CV_TypeIndex ti) +{ + U64 lo = g->min[s]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[s]) { g->mark[s][idx] = 1; } } +} + +// walk a record's type-index sites; mark roots (do_rewrite==0) or rewrite to compacted indices (==1) +internal void +lnk_gc_visit_offsets(LNK_GCTypes *g, String8 data, CV_TiOffsets ti_offs) +{ + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + U8 *p = data.str + n.offset; + CV_TypeIndex ti = memory_read32(p); + if (g->do_rewrite) { + U64 lo = g->min[n.source]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[n.source]) { memory_write32(p, g->remap[n.source][idx]); } } + } else { + lnk_gc_mark_ti(g, n.source, ti); + } + } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_syms_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->sym_ranges[task_id]) { + LNK_SymbolInput symbols = g->cv->symbol_inputs[i]; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + lnk_gc_visit_offsets(g, symbol.data, cv_symbol_ti_offsets(symbol.kind, symbol.data)); + } + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_inlines_task) +{ + LNK_GCTypes *g = raw_task; + U64 obj_idx = task_id; + Temp scratch = scratch_begin(0, 0); + String8List inlinee_lines = cv_sub_section_from_debug_s(g->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + for EachNode(dn, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(scratch.arena); + CV_TiOffsets l = cv_inlinee_ti_offsets(temp.arena, dn->string); + lnk_gc_visit_offsets(g, dn->string, l); + temp_end(temp); + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_rewrite_leaves_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->cur_ranges[task_id]) { + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->cur_leaf_v[i]); + CV_TiOffsets l = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_gc_visit_offsets(g, leaf.data, l); + temp_end(temp); + } + scratch_end(scratch); +} + +typedef struct LNK_GCRingTask +{ + U8 **leaf_v; + Rng1U64 *ranges; + U64 *counts; + U64 *offsets; + LNK_GCNamePair *pairs; +} LNK_GCRingTask; + +// parallel: count UDT leaves with a unique_name per range (pass 0) / emit (hash,idx) pairs (pass 1) +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_count_task) +{ + LNK_GCRingTask *t = raw_task; + U64 n = 0; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { n += 1; } + } + } + t->counts[task_id] = n; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) +{ + LNK_GCRingTask *t = raw_task; + U64 cur = t->offsets[task_id]; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { + U64 h = 14695981039346656037ull; + for EachIndex(c, ui.unique_name.size) { h = (h ^ ui.unique_name.str[c]) * 0x100000001b3ull; } + t->pairs[cur].hash = h; t->pairs[cur].idx = (U32)i; cur += 1; + } + } + } +} + +// mark a leaf reachable and, if this is its first mark, append it to its source's frontier so a +// later round expands it. Atomic mark gates the append, so each leaf lands on the frontier once. +internal void +lnk_gc_mark_enqueue(LNK_GCTypes *g, CV_TypeIndexSource ns, U64 ci) +{ + if (ci >= g->orig_n[ns]) { return; } + if (g->mark[ns][ci]) { return; } // fast non-atomic skip: already reachable (the common edge) + // only the worker that wins the 0->1 transition appends, so the atomic runs once per leaf (not + // once per reference edge). + if (!ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { + U32 pos = ins_atomic_u32_inc_eval(g->fcount[ns]) - 1; + g->frontier[ns][pos] = (U32)ci; + } +} + +// one bulk-synchronous round of transitive closure: expand the frontier slice [round_begin, +// round_end) of cur_source -- visit each leaf and enqueue the leaves it references (and its +// unique_name UDT counterparts). Frontier-driven, so total work is O(reachable leaves), not +// O(rounds * total leaves). +internal +THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) +{ + LNK_GCTypes *g = raw_task; + CV_TypeIndexSource s = g->cur_source; + Temp scratch = scratch_begin(0, 0); + for EachInRange(local, g->cur_ranges[task_id]) { + U32 i = g->frontier[s][g->round_begin + local]; + + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); + CV_TiOffsets l = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&l); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&l, ti_idx); + CV_TypeIndex ti = memory_read32(leaf.data.str + n.offset); + U64 lo = g->min[n.source]; + if (ti >= lo) { lnk_gc_mark_enqueue(g, n.source, ti - lo); } + } + temp_end(temp); + + if (s == CV_TypeIndexSource_TPI) { + for (U32 j = g->udt_next[i]; j != i; j = g->udt_next[j]) { + lnk_gc_mark_enqueue(g, CV_TypeIndexSource_TPI, j); + } + } + } + scratch_end(scratch); +} + +internal void +lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&arena, 1); + + LNK_GCTypes g = {0}; + g.cv = cv; + U64 total_leaves = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.min[s] = types->min_type_indices[s]; + g.orig_n[s] = types->count[s]; + g.leaf_v[s] = types->v[s]; + g.mark[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); // zeroed + total_leaves += g.orig_n[s]; + } + + // mark roots: every type index referenced by a surviving symbol / inlinee record + g.do_rewrite = 0; + g.sym_ranges = tp_divide_work(scratch.arena, cv->symbol_input_count, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + + // link UDT leaves that share a unique_name into rings, so that marking any one (e.g. a + // forward ref reached as a member-pointer target) also keeps its full definition -- needed + // for the debugger to complete types referenced only by name. TPI only (IPI has no UDTs). + U64 n_tpi = g.orig_n[CV_TypeIndexSource_TPI]; + g.udt_next = push_array_no_zero(scratch.arena, U32, n_tpi ? n_tpi : 1); + for EachIndex(i, n_tpi) { g.udt_next[i] = (U32)i; } + { + LNK_GCRingTask rt = {0}; + rt.leaf_v = g.leaf_v[CV_TypeIndexSource_TPI]; + rt.ranges = tp_divide_work(scratch.arena, n_tpi, tp->worker_count); + rt.counts = push_array(scratch.arena, U64, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_count_task, &rt); + rt.offsets = offsets_from_counts_array_u64(scratch.arena, rt.counts, tp->worker_count); + U64 np = sum_array_u64(tp->worker_count, rt.counts); + rt.pairs = push_array_no_zero(scratch.arena, LNK_GCNamePair, np ? np : 1); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_fill_task, &rt); + + radsort(rt.pairs, np, lnk_gc_name_pair_is_before); + for (U64 a = 0; a < np; ) { + U64 b = a + 1; + while (b < np && rt.pairs[b].hash == rt.pairs[a].hash) { b += 1; } + for (U64 k = a; k < b; k += 1) { g.udt_next[rt.pairs[k].idx] = rt.pairs[(k + 1 < b) ? (k + 1) : a].idx; } + a = b; + } } + + // transitive closure (parallel, bulk-synchronous): repeat rounds that visit each + // marked-but-unexpanded leaf and mark what it references, until a round marks nothing new + // seed the frontier with the root-marked leaves (one O(total leaves) scan), then expand + // frontier slices until both sources drain. Each leaf is expanded exactly once. + U32 fcount[CV_TypeIndexSource_COUNT] = {0}; + U64 start [CV_TypeIndexSource_COUNT] = {0}; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.frontier[s] = push_array_no_zero(scratch.arena, U32, g.orig_n[s] ? g.orig_n[s] : 1); + g.fcount[s] = &fcount[s]; + for EachIndex(i, g.orig_n[s]) { if (g.mark[s][i]) { g.frontier[s][fcount[s]++] = (U32)i; } } + } + for (;;) { + B32 any = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + U64 begin = start[s], end = fcount[s]; // fcount may grow during the round (cross-source enqueues) + if (begin < end) { + any = 1; + g.cur_source = (CV_TypeIndexSource)s; + g.round_begin = begin; + g.round_end = end; + g.cur_ranges = tp_divide_work(scratch.arena, end - begin, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + start[s] = end; + } + } + if (!any) { break; } + } + + // compact each source: assign new contiguous type indices to the kept leaves. The leaf + // pointer array is compacted IN PLACE (kept count only shrinks, so the write cursor never + // passes the read cursor) and remap lives in scratch -- so the GC adds nothing to the arena + // that survives into the (peak) PDB build. + U64 kept_total = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.remap[s] = push_array_no_zero(scratch.arena, CV_TypeIndex, g.orig_n[s] ? g.orig_n[s] : 1); + U8 **v = g.leaf_v[s]; + U64 new_n = 0; + for EachIndex(idx, g.orig_n[s]) { + if (g.mark[s][idx]) { g.remap[s][idx] = (CV_TypeIndex)(g.min[s] + new_n); v[new_n++] = v[idx]; } + else { g.remap[s][idx] = 0; /* T_NOTYPE; never referenced by a kept record */ } + } + types->count[s] = new_n; + kept_total += new_n; + } + + // rewrite all type-index references to the compacted indices + g.do_rewrite = 1; + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.cur_source = (CV_TypeIndexSource)s; + g.cur_leaf_v = types->v[s]; + g.cur_ranges = tp_divide_work(scratch.arena, types->count[s], tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_rewrite_leaves_task, &g); + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + lnk_log(LNK_Log_Debug, "type GC: kept %llu of %llu leaves (pruned %llu)", kept_total, total_leaves, total_leaves - kept_total); + } + scratch_end(scratch); + ProfEnd(); +} + +typedef struct +{ + U64 weight; + U32 obj_idx; +} LNK_ObjDistWeight; + +force_inline int +lnk_obj_dist_weight_is_before(void *raw_a, void *raw_b) +{ + LNK_ObjDistWeight *a = raw_a, *b = raw_b; + if (a->weight != b->weight) { return a->weight > b->weight; } + return a->obj_idx < b->obj_idx; // deterministic total order +} + +// FAIR-SHARE: distribute cv->obj_count objs across `worker_count` lane buckets. +// Rebuilt per barrier pass so the distribution matches the cohort C that pass +// runs at (lnk_move_global_symbols_to_gsi / lnk_write_pdb_modules read +// task->obj_indices[task_id] for lanes [0,C)). Output is width- and +// assignment-independent -- per-obj results land in per-obj slots (module +// streams) or in GSI bucket chains that are content-sorted at serialization +// (gsi_symbol_is_before radsorts every chain) -- so any deterministic partition +// produces byte-identical PDB bytes; only the per-lane balance changes. +// +// `weights` (optional, [obj_count]) upgrades the round-robin to a greedy LPT +// (longest-processing-time) assignment: objs are taken in weight-descending +// order (obj_idx tie-break -> deterministic) and each goes to the least-loaded +// lane. Round-robin ignores per-obj symbol-stream size, so a lane that draws +// several giant objs holds the whole barrier pass at the final barrier while +// the other lanes idle. +internal void +lnk_build_pdb_distribute_obj_indices(Arena *arena, LNK_BuildPdb *task, U64 obj_count, U32 worker_count, U64 *weights) +{ + task->obj_indices = push_array(arena, U32Array, worker_count); + if (weights == 0) { + U64 objs_per_worker = CeilIntegerDiv(obj_count, worker_count); + for EachIndex(i, worker_count) { task->obj_indices[i].v = push_array(arena, U32, objs_per_worker ? objs_per_worker : 1); } + for EachIndex(obj_idx, obj_count) { + U32Array *obj_indices = &task->obj_indices[obj_idx % worker_count]; + obj_indices->v[obj_indices->count++] = (U32)obj_idx; + } + } else { + Temp scratch = scratch_begin(&arena, 1); + + LNK_ObjDistWeight *order = push_array_no_zero(scratch.arena, LNK_ObjDistWeight, obj_count); + for EachIndex(obj_idx, obj_count) { order[obj_idx] = (LNK_ObjDistWeight){ .weight = weights[obj_idx], .obj_idx = (U32)obj_idx }; } + radsort(order, obj_count, lnk_obj_dist_weight_is_before); + + U64 *loads = push_array(scratch.arena, U64, worker_count); + U32 *assign = push_array_no_zero(scratch.arena, U32, obj_count); + for EachIndex(i, obj_count) { + U32 min_lane = 0; + for (U32 lane = 1; lane < worker_count; lane += 1) { if (loads[lane] < loads[min_lane]) { min_lane = lane; } } + assign[order[i].obj_idx] = min_lane; + loads[min_lane] += order[i].weight + 1; // +1 spreads zero-weight objs too + task->obj_indices[min_lane].count += 1; + } + + for EachIndex(lane, worker_count) { + task->obj_indices[lane].v = push_array_no_zero(arena, U32, task->obj_indices[lane].count); + task->obj_indices[lane].count = 0; + } + // fill in ascending obj order per lane (deterministic, cache-friendly iteration) + for EachIndex(obj_idx, obj_count) { + U32Array *obj_indices = &task->obj_indices[assign[obj_idx]]; + obj_indices->v[obj_indices->count++] = (U32)obj_idx; + } + + scratch_end(scratch); + } } internal LNK_FileArtifact -lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags) +lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags, struct LNK_Inputer *inputer) { ProfBeginFunction(); Temp scratch = scratch_begin(tp_arena->v, tp_arena->count); @@ -3041,6 +6560,11 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config builder_flags = ~0; } + // ini= bucket: pdb_alloc_ commits the MSF + type-server tables (fresh pages, + // ~132K faults on the editor link) -- under a storm every fresh commit pays + // the page-repurpose path, so this span needs its own attribution + lnk_summary_phase_begin(LNK_SummaryPhase_PdbIni); + LNK_BuildPdb task = { .image_data = image_data, .symtab = symtab, @@ -3060,6 +6584,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config LNK_PdbOutput *output_ptr = 0; if (writer.output_path.size > 0) { output.writer = writer.file_writer; + output.decommit_flushed = (config->rad_debug != LNK_SwitchState_Yes); output.file = lnk_background_file_writer_begin_file(output.writer, writer.output_path, writer.temp_output_path); if (output.file != 0) { output.sealed_stream_cap = cv->obj_count + 128; @@ -3068,6 +6593,17 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config } } + task.output = output_ptr; + + // patched debug-section copies (obj->section_data_copies) release at the END of the + // module-write phase. P3.3: with g_debug_s_window set (default) $S never has copies -- + // only the (essentially nonexistent) reloc-bearing non-$S debug sections do -- and the + // /PDBSTRIPPED pre-pass re-reads through lnk_obj_window_debug_s, so the gate below only + // still matters for the /OPT:GCTYPES copy path, where the stripping loop re-walks + // cv->debug_s_arr Symbols aliasing the copies after this build returns. The RDI converter + // is safe (it reads the PDB artifact pages, not obj debug sections). + task.free_sect_copies = (config->pdb_stripped_name.size == 0); + PDB_BuildHooks build_hooks = {0}; if (output_ptr != 0) { build_hooks.stream_finalize = lnk_pdb_output_finalize_stream; @@ -3077,35 +6613,79 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config // set min type indices for EachElement(ti_source, cv_types.min_type_indices) { task.pdb->type_servers[ti_source]->ti_lo = cv_types.min_type_indices[ti_source]; } - // per worker obj indices - { - U64 objs_per_worker = CeilIntegerDiv(cv->obj_count, tp->worker_count); - task.obj_indices = push_array(scratch.arena, U32Array, tp->worker_count); - for EachIndex(i, tp->worker_count) { task.obj_indices[i].v = push_array(scratch.arena, U32, objs_per_worker); } - for EachIndex(obj_idx, cv->obj_count) { - U32Array *obj_indices = &task.obj_indices[obj_idx % tp->worker_count]; - obj_indices->v[obj_indices->count++] = obj_idx; - } - } + // per-worker obj indices are (re)distributed per barrier pass to the cohort + // that pass actually runs at (FAIR-SHARE: tp->worker_count is pinned to the + // cohort C inside each tp_barrier_begin/end bracket, and the + // lnk_write_pdb_modules task reads task.obj_indices[task_id] for lanes [0,C); + // P2b made lnk_move_global_symbols_to_gsi obj_indices-free -- it strides the + // pre-extracted per-obj tables directly). Distributing to the full + // worker_count up front would leave objs in buckets [C,worker_count) + // unprocessed when Ctype_servers[CV_TypeIndexSource_IPI], cv_types.count[CV_TypeIndexSource_IPI], cv_types.v[CV_TypeIndexSource_IPI]); } if (builder_flags & LNK_PDB_BuilderFlag_Tpi) { pdb_type_server_push_parallel(tp, task.pdb->type_servers[CV_TypeIndexSource_TPI], cv_types.count[CV_TypeIndexSource_TPI], cv_types.v[CV_TypeIndexSource_TPI]); } + lnk_summary_phase_end(LNK_SummaryPhase_PdbTpi); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbStr); ProfBegin("Merge String Tables"); task.string_ht = cv_dedup_string_tables(tp_arena, tp, cv->obj_count, cv->debug_s_arr); cv_string_hash_table_assign_buffer_offsets(tp, task.string_ht); + + // the deduped buckets alias the objs' $S string tables in place -- patched debug-section + // COPIES for reloc-carrying $S, RAW-MAPPED view bytes for reloc-free $S -- and both /names + // (pdb_strtab_build memcpys bucket bytes at serialize time in pdb_build_dbi_info) and DBI + // file-info hash these bytes AFTER the backing dies (copies release at the end of Write + // Modules; views die in P5) -- rehome the winning buckets into their own blob first + // (total = the deduped /names payload, tiny next to the backing being released). P3.1: + // UNCONDITIONAL, and the bucket walk below is backing-agnostic (every non-null winning + // bucket is copied, whatever its bytes alias), so raw-mapped tables are covered too. + // Offsets are already assigned; bytes are identical, so the /names stream and every + // recorded offset are unchanged -- /PDBSTRIPPED is unaffected (its second build makes its + // own string_ht from its own debug_s_arr; strtab serialize reads content-identical bytes). + if (task.string_ht.total_string_size > 0) { + ProfBegin("Materialize String Table Bytes"); + U8 *blob = push_array_no_zero(tp_arena->v[0], U8, task.string_ht.total_string_size); + U64 cursor = 0; + for EachIndex(bucket_idx, task.string_ht.bucket_cap) { + CV_StringBucket *bucket = task.string_ht.buckets[bucket_idx]; + if (bucket == 0) { continue; } + Assert(cursor + bucket->string.size <= task.string_ht.total_string_size); + MemoryCopy(blob + cursor, bucket->string.str, bucket->string.size); + bucket->string.str = blob + cursor; + cursor += bucket->string.size; + } + ProfEnd(); + } ProfEnd(); + lnk_summary_phase_end(LNK_SummaryPhase_PdbStr); task.string_table_base_offset = task.pdb->info->strtab.size; ProfBegin("Add string tables"); pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht); ProfEnd(); pdb_build_types(tp, task.pdb, &build_hooks); + lnk_compressed_obj_log_phase_stats("before write modules"); + lnk_pdb_output_log_mark(output_ptr, "after pdb_build_types (TPI/IPI sealed)"); + + // merged leaf bytes are now in MSF pages (and on their way to disk); no consumer of + // cv_types.v remains on this path (RDI converts from the PDB artifact, the RRT export + // is a separate boot mode), so hand the multi-GB materialize buffers back to the OS + for EachElement(ti_source, cv_types.leaf_buffers) { + if (cv_types.leaf_buffers[ti_source].size > 0) { + release_memory(cv_types.leaf_buffers[ti_source].str, cv_types.leaf_buffers[ti_source].size); + cv_types.leaf_buffers[ti_source] = str8_zero(); + cv_types.v[ti_source] = 0; // poison dangling leaf pointers + cv_types.count[ti_source] = 0; + } + } if (builder_flags & LNK_PDB_BuilderFlag_Modules) { ProfScope ("Alloc Modules") @@ -3113,22 +6693,89 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config task.mod_arr[obj_idx] = dbi_push_module(task.pdb->dbi, cv->obj_arr[obj_idx]->path, lnk_obj_get_lib_path(cv->obj_arr[obj_idx])); } - ProfScope("Write Modules") tp_for_parallel(tp, 0, tp->worker_count, lnk_write_pdb_modules, &task); - if (output_ptr != 0) { +ProfScope("Write Modules") + { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbMod); + U64 phase_begin_us = now_time_us(); + // FAIR-SHARE: pin the cohort, distribute objs over exactly the cohort lanes, + // then run the barrier pass at that cohort. tp_barrier_begin sets + // tp->worker_count := C for the bracket. + U32 C = tp_barrier_begin(tp); + // weight = total debug$S byte size: the module-stream write walks every + // subsection of the obj (symbols + lines + checksums + ...) + U64 *weights = push_array_no_zero(scratch.arena, U64, cv->obj_count); for EachIndex(obj_idx, cv->obj_count) { - lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.mod_arr[obj_idx]->sn); + weights[obj_idx] = cv_total_sub_section_size_from_debug_s(&cv->debug_s_arr[obj_idx]); + } + lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); + + // Per-obj counts/proc-refs plus per-lane surviving payload arenas. Exact global-symbol + // winners are copied into these arenas directly from the one transformed $S window; + // Move Global Symbols later consumes the compacted winner pointers and proc-ref tables. + task.preext = push_array(scratch.arena, LNK_GsiPreExtractObj, cv->obj_count); + task.preext_arena_count = C; + task.procref_payload_arenas = push_array(scratch.arena, Arena *, C); + for EachIndex(i, C) { + task.procref_payload_arenas[i] = arena_alloc(.commit_size = MB(2), .name = "GSI_PROC_REFS"); + } +#if BUILD_DEBUG + // the fused extraction walks objs [0, obj_count) only; the old globals collect walked + // every symbol input, which spans [0, cv->count) INCLUDING injected type-server/.ifc + // blob pseudo-objs -- prove those never carry a Symbols subsection so nothing is missed + for (U64 pseudo_idx = cv->obj_count; pseudo_idx < cv->count; pseudo_idx += 1) { + Assert(cv_sub_section_from_debug_s(cv->debug_s_arr[pseudo_idx], CV_C13SubSectionKind_Symbols).total_size == 0); } +#endif + tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast + tp_barrier_end(tp); + lnk_compressed_obj_log_phase_stats("after write modules"); + lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + lnk_pdb_output_log_mark(output_ptr, "after write modules"); + if (g_debug_s_window && lnk_get_log_status(LNK_Log_Debug)) { + // bounds the per-worker window arena growth: worst case commit = cohort x this value + lnk_log(LNK_Log_Debug, "[pdb] $S window high-water: %llu bytes (largest single obj window)", g_debug_s_window_hwm); + } + lnk_summary_phase_end(LNK_SummaryPhase_PdbMod); + } + + // the module-write phase ran the last $S fixup replay on this path -- hand the GB-class + // journal arenas to the background reaper now, before the GSI/PSI commit peak. No-op when + // the eager path already consumed it (or for the stripped/SkipSymbolTypeFixup cv, which + // never built one). P3.3: when a /PDBSTRIPPED build follows, the journal must SURVIVE -- + // the stripping pre-pass re-reads Symbols nodes through lnk_obj_window_debug_s, whose fill + // replays the journal (nothing persists into the raw views anymore); the pre-pass releases + // it when done. + if (config->pdb_stripped_name.size == 0) { + lnk_release_debug_s_fixup_journal(cv); } - ProfScope("Move Global Symbols") tp_for_parallel(tp, 0, tp->worker_count, lnk_move_global_symbols_to_gsi, &task); - ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); + + // module streams were enqueued per-obj inside lnk_write_pdb_modules +ProfScope("Move Global Symbols") + { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbGsi); + U64 phase_begin_us = now_time_us(); + U32 C = tp_barrier_begin(tp); + // P2b: no obj_indices distribution -- the pass reads no $S bytes; it flattens the + // pre-extracted per-obj segments with obj-index striding and even-split inserts + tp_for_parallel_reserve(tp, 0, C, lnk_move_global_symbols_to_gsi, &task); // BARRIER pass (path B): tp_sum_u64/tp_broadcast/barrier_wait + tp_barrier_end(tp); + lnk_log(LNK_Log_Timers, "[pdb] move global symbols in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + lnk_summary_phase_end(LNK_SummaryPhase_PdbGsi); + } + + lnk_summary_phase_begin(LNK_SummaryPhase_PdbSym); + ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); + lnk_summary_phase_end(LNK_SummaryPhase_PdbSym); if (output_ptr != 0) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->globals_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->symbols_sn); + lnk_pdb_output_log_mark(output_ptr, "after GSI/PSI streams sealed"); } } if (builder_flags & LNK_PDB_BuilderFlag_SC) { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbSc); ProfBegin("Build Section Contrib Map"); { ProfBegin("Build DBI Section Headers"); @@ -3170,6 +6817,42 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config sec_contribs->cap = new_count; } ProfEnd(); + lnk_summary_phase_end(LNK_SummaryPhase_PdbSc); + } + + // Streaming-ring P5: the SC pass above was the LAST reader of the memory-mapped input + // views on this path (audited: pass B of the module-write epilogue = last .debug$S read, + // publics in "Move Global Symbols" = last COFF symbol-table + string-table read [long + // public names alias the view's COFF string table, symbol records fall back to the view + // when no symbol_table_copy exists], SC = last section-header/name read; NatVis below + // reads its own files, /names serializes the P3.1 rehomed blob, DBI file-info hashes the + // repointed bucket copies, MSF build/serialize reads PDB pages only). Release the views + // NOW with the exit path's own capped parallel sweep, relocated: the pool is idle between + // the SC pass and the serial NatVis/DBI/MSF tail, so the sweep lands at parallel-unmap + // wall (~1s FN-scale; a single background thread here serialized ~60s of unmap CPU and + // showed up as +6.5s FN wall) and mapped input residency is gone before the tail + PDB + // write drain instead of held to the end of the link. The sweep zeroes data/owns_file_map + // per input, so the exit-time calls (lnk_inputer_release_file_maps, + // lnk_release_input_views) turn into no-ops -- idempotent. Gated OFF when a /PDBSTRIPPED + // build follows (its pre-build strip loop re-walks Symbols through lnk_obj_window_debug_s + // = raw view reads after this function returns; the caller also passes inputer==0 for the + // stripped build itself) and under /OPT:GCTYPES (copy mode: reloc-free $S slices in + // debug_s_arr alias the raw views in place -- keep the exit-time release). Only for the + // CoW read-only mapping mode, mirroring lnk_release_input_views (read-write-shared unmap + // flushes dirty pages back to the input files). + if (inputer != 0 && + config->pdb_stripped_name.size == 0 && + config->opt_gc_types != LNK_SwitchState_Yes && + (config->io_flags & LNK_IO_Flags_MemoryMapFilesReadOnly) && + !(config->io_flags & LNK_IO_Flags_MemoryMapFilesReadWrite)) { + ProfBegin("Release Input Views Early"); + U64 unmap_begin_us = now_time_us(); + // All thread-pool consumers above have joined. Keep compressed-object teardown at this + // explicit quiescence point; per-object or background close can race lazy materialization. + lnk_inputer_release_file_maps(tp, config->debug_worker_cap, inputer); + lnk_log(LNK_Log_Timers, "[pdb] early input-view release in %.2f ms", + (F64)(now_time_us() - unmap_begin_us) / 1000.0); + ProfEnd(); } if (builder_flags & LNK_PDB_BuilderFlag_NATVIS) { @@ -3205,7 +6888,10 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfEnd(); } + lnk_summary_phase_begin(LNK_SummaryPhase_PdbMsf); + lnk_pdb_output_log_mark(output_ptr, "before pdb_build_dbi_info"); pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks); + lnk_pdb_output_log_mark(output_ptr, "after pdb_build_dbi_info"); MSF_Error msf_err = msf_build(task.pdb->msf); if (msf_err != MSF_Error_OK) { @@ -3213,7 +6899,9 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config } if (output_ptr != 0) { + lnk_pdb_output_log_mark(output_ptr, "after msf_build"); lnk_pdb_output_enqueue_remaining(output_ptr, task.pdb->msf); + lnk_pdb_output_log_mark(output_ptr, "after enqueue_remaining"); } ProfBegin("Get Page Nodes"); @@ -3223,6 +6911,8 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config if (output_ptr != 0) { lnk_background_file_writer_end_file(output_ptr->writer, output_ptr->file, artifact.data.total_size); } + lnk_summary_phase_end(LNK_SummaryPhase_PdbMsf); + // NOTE: linker is about to exit so we can skip memory release // and let windows free memory since it does this faster diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 573fc3b53..c220d2f3e 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -3,6 +3,8 @@ #pragma once +typedef struct LNK_CObjDecodeWindow LNK_CObjDecodeWindow; + //////////////////////////////// // RRT @@ -76,6 +78,59 @@ typedef struct LNK_SymbolInputTask U64 weight; } LNK_SymbolInputTask; +// Streaming-ring P2 slice A: deferred .debug$S TI/kind fixup journal. Built inside +// lnk_merge_types while the merge state it needs is still alive (assigned-TI hash tables on +// merge scratch, materialized IPI leaf copies released after pdb_build_types); replayed per +// obj at the START of the module-write visit (lnk_write_pdb_modules sizing loop), before any +// consumer reads the obj's $S bytes. Entries alias the obj's own $S backing (patched copies / +// raw maps); replay applies entries in build order, so the byte end-state is identical to the +// old in-place lnk_cv_patcher_symbols / lnk_cv_patcher_inlines / lnk_fixup_symbols passes. +// Entries are 8B, keyed per NODE run: the target is a byte offset relative to the run's +// subsection-node base (symbols: the LNK_SymbolInput node; inlinees: each InlineeLines +// data_list node in list order), so no pointer is stored. off:31 | width:1 (set = 32-bit +// type-index write, clear = 16-bit kind rewrite). A node >= 2GiB cannot encode -> that run +// falls back to 16B wide entries (per-run is_wide flag), decided up front from the node size. +typedef struct LNK_DebugSPatch { U32 off_w; U32 value; } LNK_DebugSPatch; +typedef struct LNK_DebugSPatchWide { U64 off; U32 value; U32 size; } LNK_DebugSPatchWide; +typedef struct LNK_DebugSPatchArray +{ + void *v; // LNK_DebugSPatch[] or LNK_DebugSPatchWide[] (is_wide) + U64 count; + B32 is_wide; +} LNK_DebugSPatchArray; +typedef struct LNK_DebugSInlineJournal +{ + LNK_DebugSPatchArray patches; // all InlineeLines entries of the obj, node runs in data_list order + U32 *node_counts; // [InlineeLines node_count] entries per node run +} LNK_DebugSInlineJournal; + +// $T streaming (ring P4): per-real-obj journal of LF_NOTYPE kind rewrites. Raw mapped +// .debug$T/$P views are never written anymore (writes used to CoW-dirty one input page per +// discard); instead the rewrite is journaled here and mirrored into CV_DebugT.tags, and the +// downstream byte readers (hash current-leaf reads, unique-leaf sizing, materialize) replay it. +// Entry = leaf_idx | optional KIND_ONLY flag: +// full rewrite (default): header becomes { size=sizeof(CV_LeafKind), kind=LF_NOTYPE } -- the +// ENDPRECOMP strip and the invalid-TI/cyclic discards in lnk_hash_cv_leaf; +// KIND_ONLY: only the kind becomes LF_NOTYPE, size/payload preserved -- the LF_IFC_RECORD +// (0x1522) placeholder rewrite in lnk_ifc_resolve_task. +// Entries are kept sorted by leaf_idx (sorted insert; pushes are single-threaded per obj). +// Pseudo objs (type servers, .ifc blobs; obj_idx >= obj_count) keep their in-place writes: +// their $T is arena-backed, not a mapped input view. +// +// HOT-PATH CONTRACT: readers stay on the ORIGINAL raw-byte code path unless the per-obj +// bitmap says the leaf is journaled. `bitmap == 0` (the overwhelmingly common case: journals +// exist only for $P objs, IFC consumers, and invalid-input error paths) costs one load+branch; +// only journaled leaves pay the bsearch + reconstruct. +#define LNK_NOTYPE_JOURNAL_KIND_ONLY (1u << 31) +typedef struct LNK_NotypeJournal +{ + U32 *v; // (leaf_idx | LNK_NOTYPE_JOURNAL_KIND_ONLY?) ascending by leaf_idx + U32 count; + U32 cap; + U64 *bitmap; // 1 bit / parse-time leaf slot; 0 until the first push + U64 bit_cap; // bits in bitmap +} LNK_NotypeJournal; + typedef struct { LNK_Config *config; @@ -92,6 +147,7 @@ typedef struct U64 *obj_to_ts; String8List *debug_s_list_arr; + U32Array *debug_s_sect_idx_arr; // per obj: 0-based sect_idx of each debug_s_list_arr node (provenance tagging) U32Array int_obj_indices; U32Array ext_obj_indices; @@ -105,9 +161,44 @@ typedef struct U64 symbol_input_count; LNK_SymbolInput *symbol_inputs; // [symbol_input_count] - Rng1U64 *symbol_input_ranges; // [worker_count] + Rng1U64 *symbol_input_ranges; // [symbol_input_range_count] U64 symbol_patch_task_count; // LNK_SymbolInputTask *symbol_patch_task; // [symbol_patch_task_count] + // FAIR-SHARE: fixed lane count symbol_input_ranges was built for (full pool width at build + // time). Barrier passes may run at a pinned cohort C < this; they must walk lanes + // [task_id, symbol_input_range_count) strided by the cohort, NOT index by task_id alone. + U64 symbol_input_range_count; + + // deferred $S TI/kind fixup journal (see LNK_DebugSPatch). Null / flag == 0 when the merge + // ran with LNK_MergeTypeFlag_SkipSymbolTypeFixup, or after the journal was consumed + // (lnk_release_debug_s_fixup_journal: eager path, or end of the module-write pass). + // All journal storage (entry arrays + the three tables below) lives inside dedicated + // per-worker DEBUG_S_FIXUP_JOURNAL arenas, handed to the background reaper at consume + // time -- GB-class at FN scale, dead after the last per-obj replay. + B32 has_debug_s_fixup_journal; + TP_Arena *debug_s_fixup_journal_arenas; + LNK_DebugSPatchArray *debug_s_sym_fixups; // [symbol_input_count] one node run per symbol input + LNK_DebugSInlineJournal *debug_s_inline_fixups; // [count] + U64 *debug_s_sym_fixup_offsets; // [count+1] obj -> symbol_inputs range (inputs are obj-contiguous) + + // IFC (header-unit debug-record) resolution: + // redirects a consuming obj's local LF_IFC_RECORD placeholder TI to a leaf in + // an injected .ifc debug-records blob "obj". Consulted first in lnk_leaf_ref_from_ti. + // key = Compose64Bit(obj_idx, local_ti) + // value = Compose64Bit(blob_obj_idx, blob_leaf_idx) + B32 has_ifc_redirects; + HashMap ifc_redirect_hm; + Rng1U64 ifc_obj_range; // [min,max) range of injected blob objs in the parallel arrays + U32Array ifc_indices; // obj indices of injected .ifc blob objs (hashed/deduped first) + // exact per-obj key filter for ifc_redirect_hm: bit set iff Compose64Bit(obj_idx, ti) was pushed. + // lets lnk_leaf_ref_from_ti skip the (miss-dominated) hash-map search entirely; on a set bit the + // original map is searched unchanged, so results are bit-identical to always searching. + U64 **ifc_redirect_bits; // [count]; null == obj has no redirect keys + Rng1U64 *ifc_redirect_ti_rng; // [count]; [min,max) local-TI span covered by the obj's bitset + + // $T streaming (ring P4): [obj_count] NOTYPE-rewrite journals, real objs only (see + // LNK_NotypeJournal). Pseudo objs at [obj_count, count) mutate their arena-backed $T in place. + LNK_NotypeJournal *notype_journal; } LNK_CodeViewInput; typedef struct @@ -115,6 +206,7 @@ typedef struct LNK_CodeViewInput *input; String8Array *raw_types; // [obj_count] CV_DebugT *out_types; // [obj_count] + B32 is_debug_p; } LNK_ParseCvTypes; //////////////////////////////// @@ -131,11 +223,14 @@ typedef struct LNK_LeafRef *bucket_arr; } LNK_LeafHashTable; +#define LNK_ASSIGNED_TI_ENTRY_SIZE 12 +#define LNK_ASSIGNED_TI_HASH_OFF 0 +#define LNK_ASSIGNED_TI_TI_OFF 8 + typedef struct { - U64 cap; - CV_TypeIndex *ti_arr; - U64 *hash_arr; + U64 cap; + U8 *v; } LNK_AssignedTiHash; typedef struct LNK_LeafRange @@ -162,6 +257,10 @@ typedef struct // @type_server CV_TypeIndex **obj_ti_maps; U64 *hashes[CV_TypeIndexSource_COUNT]; + + // standalone backing for the merged leaf bytes v[] points into (one per source); + // owned by the caller, released after pdb_build_types on the linker path + String8 leaf_buffers[CV_TypeIndexSource_COUNT]; } LNK_MergedTypes; typedef struct @@ -171,6 +270,7 @@ typedef struct LNK_LeafHashTable leaf_ht_arr[CV_TypeIndexSource_COUNT]; LNK_AssignedTiHash assigned_ti_arr[CV_TypeIndexSource_COUNT]; Arena **fixed_arenas; + struct LNK_CObjDecodeWindow *decode_windows; CV_TypeIndexSource ti_source; U32Array indices; Rng1U64 *ranges; @@ -193,6 +293,37 @@ typedef struct U64 pop_obj_idx; Rng1U64 *pop_range; + // deterministic unique-leaf estimate: distinct-hash bitmaps (per ti source) filled with + // commutative atomic ORs over the precomputed debug_h hashes -> same input, same bits, same + // estimate every run. sized pow2 so bit index is hash & (bits-1). + U32 *estimate_bitmap [CV_TypeIndexSource_COUNT]; // [estimate_bitmap_bits/32] + U64 estimate_bitmap_bits[CV_TypeIndexSource_COUNT]; // pow2 + + // set when a probe wraps without finding a slot (estimate-sized table overflowed); dedup is + // retried once with the always-sufficient total-based caps. deterministic: overflow happens + // iff the unique count exceeds cap, which is a function of the input alone. + U32 leaf_ht_overflow; + + // materialize unique leaves (unbucket + leaf TI-fixup fused, applied to a private copy so the + // fixup never dirties the copy-on-write input mapping) + U64 *leaf_buffer_offsets; // [worker_count+1] per-lane byte offsets into leaf_buffer + U8 *leaf_buffer; + U64 *materialize_obj_offsets[CV_TypeIndexSource_COUNT]; // [input->count+1], sorted-ref ranges + + // Compressed winner payload prefetch. segment_offsets maps an OBJ-local segment index to a + // compact global bit/key index. Copy-present atomically claims segments as it discovers the + // winning leaves, so no later scan of the multi-million-entry winner arrays is needed. + U64 *winner_segment_offsets; // [input->obj_count+1] + U64 *winner_segment_bitmap; + U64 *winner_segment_worker_bitmaps; // [worker_count * winner_segment_word_count] + U64 winner_segment_word_count; + U64 *winner_segment_keys; // obj_idx:32 | segment_idx:32 + U64 winner_segment_key_count; + + // $S fixup journal build: per-worker arenas the LNK_DebugSPatch entry arrays land on + // (must outlive the merge -- replay happens at module write) + TP_Arena *journal_arena; + LNK_MergedTypes result; } LNK_MergeTypes; @@ -209,6 +340,21 @@ typedef enum LNK_PDB_BuilderFlag_NATVIS = (1<<5), } LNK_PDB_BuilderFlags; +// Streaming-ring P2b: per-obj global-symbol counts + proc-refs. Global-symbol records are +// content-deduped directly from the one module-write window and only exact winners are copied +// to surviving storage; no candidate metadata or payload survives the window. Proc-ref +// payloads AND the per-obj procref value/hash arrays live on the surviving +// procref_payload_arenas (consumed post-release by lnk_move_global_symbols_to_gsi, which +// flattens per-obj segments in ascending obj-index order -- deterministic and +// cohort-independent -- and otherwise reads only the materialized winner records). +typedef struct +{ + U64 cand_count; // global records (cv_is_global_symbol + top-level typedefs) + U64 procref_count; // GPROC32/LPROC32 records + CV_Symbol *procref_syms; // [procref_count] cv_make_proc_ref results (arrays + payload on survive arenas) + U32 *procref_hashes; // [procref_count] gsi_hash(name) +} LNK_GsiPreExtractObj; + typedef struct { String8 image_data; @@ -230,6 +376,36 @@ typedef struct Rng1U64Array image_section_file_ranges; U64 *image_section_file_section_numbers; PDB_DbiSCArray *sc_arrays; // [obj_count] + struct LNK_PdbOutput *output; // when non-null, module streams enqueue to the background writer as they complete + + // when set, lnk_build_pdb drops every obj's patched debug-section copies + // (LNK_Obj.section_data_copies) and releases the SECT_DATA_COPIES arenas at the END of the + // module-write phase, right after direct winner dedup/materialization (the last $S reader + // on this path). Must be 0 when a /PDBSTRIPPED build follows -- it re-walks + // cv->debug_s_arr after lnk_build_pdb. P3.1: the /names bucket rehome and the + // mod->source_file_list repoint are UNCONDITIONAL (they must also cover string tables in + // reloc-free RAW-MAPPED $S sections, which never had copies to begin with). + B32 free_sect_copies; + + // P2b pre-extraction state (see LNK_GsiPreExtractObj). procref_payload_arenas hold exact + // global-symbol winners plus the procref value/hash arrays and cv_make_proc_ref payloads. + // They must survive until GSI serialization + // (like the old proc_ref_arenas, they live to process exit). + LNK_GsiPreExtractObj *preext; // [cv->obj_count] + Arena **procref_payload_arenas; // [preext_arena_count] + U64 preext_arena_count; // Write Modules cohort width + + // Transient exact-content set used only inside Write Modules. Empty slots are claimed with + // a reservation sentinel before a winner is copied, so duplicates allocate no payload. + U64 gsi_dedup_bucket_cap; + void **gsi_dedup_buckets; + + // Direct exact-dedup output: the winner pointer array is compacted from hash-table slot + // order onto gsi->arena; payload bytes remain on the per-worker surviving arenas above. + // Both stay alive through GSI serialization. This is the only global-symbol input read by + // lnk_move_global_symbols_to_gsi. + U64 gsi_winner_count; + void **gsi_winner_ptrs; // [gsi_winner_count] } LNK_BuildPdb; typedef struct @@ -263,13 +439,24 @@ internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp internal int lnk_leaf_ref_compare (LNK_LeafRef a, LNK_LeafRef b); internal B32 lnk_match_leaf_ref (LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b); -internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); -internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); +internal void lnk_notype_journal_push (Arena *arena, LNK_NotypeJournal *journal, U32 leaf_idx, B32 kind_only, U64 bit_cap); +internal B32 lnk_notype_journal_test (LNK_NotypeJournal *journal, U64 leaf_idx); +internal B32 lnk_notype_journal_find (LNK_NotypeJournal *journal, U32 leaf_idx, B32 *kind_only_out); +internal CV_Leaf lnk_cv_leaf_from_leaf_ref (Arena *arena, LNK_CObjDecodeWindow *decode_window, LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx); +internal U64 lnk_leaf_ref_materialize_meta (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); +internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef leaf_ref, CV_Leaf leaf, CV_TiOffsets ti_offs, B32 discard_cycles); +internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CObjDecodeWindow *decode_window, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); +internal void lnk_apply_debug_s_fixups_for_obj (LNK_CodeViewInput *cv, U64 obj_idx); +internal void lnk_apply_debug_s_fixups_eager (TP_Context *tp, LNK_CodeViewInput *cv); +internal CV_DebugS lnk_obj_window_debug_s (Arena *arena, LNK_CodeViewInput *cv, U64 obj_idx, U64 image_base, COFF_SectionHeader **image_section_table, B32 symbols_only); +internal void lnk_release_debug_s_fixup_journal (LNK_CodeViewInput *cv); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); //////////////////////////////// // PDB -internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags); +internal void lnk_gc_types (TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types); +struct LNK_Inputer; +internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags, struct LNK_Inputer *inputer); diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index e5318d09c..2cea45899 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -217,6 +217,50 @@ THREAD_POOL_TASK_FUNC(lnk_data_from_file_path_task) } } +#if OS_WINDOWS +// Input views are mapped from PAGE_WRITECOPY sections with FILE_MAP_READ access, so an +// untouched view carries no pagefile commit charge. Mapping with FILE_MAP_COPY instead +// would charge commit for the ENTIRE view at map time -- even for pages never written -- +// so N concurrent big links would hold input-set-sized commit for their whole runtime +// (measured 22.4 GiB of a 49.7 GiB peak on a large editor DLL link) and feed build-farm +// memory admission limits for no benefit. +// +// The few remaining writers that patch input bytes in place (IFC 0x1522 LF_IFC_RECORD +// NOTYPE pokes, LF_ENDPRECOMP removal in debug$P; a couple of bytes per page, a handful +// of pages per link) hit this vectored handler, which promotes JUST the faulting page to +// PAGE_WRITECOPY and retries. Commit is then charged per dirtied page instead of per +// view. Write semantics are identical to the old FILE_MAP_COPY mapping: the first write +// makes the page private, the input file is never modified. Pages of ordinary allocations +// or of read-write mappings never reach the handler (they do not fault on write), so +// /RAD_MEMORY_MAP_FILES:READ_WRITE and no-map modes are unaffected. +global volatile LONG g_lnk_cow_veh_installed; +global volatile LONG g_lnk_cow_promoted_pages; + +internal LONG NTAPI +lnk_cow_page_promote_veh(EXCEPTION_POINTERS *info) +{ + EXCEPTION_RECORD *er = info->ExceptionRecord; + if (er->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && er->NumberParameters >= 2 && er->ExceptionInformation[0] == 1) { + void *addr = (void *)er->ExceptionInformation[1]; + MEMORY_BASIC_INFORMATION mbi = {0}; + if (VirtualQuery(addr, &mbi, sizeof(mbi)) >= sizeof(mbi) && mbi.Type == MEM_MAPPED) { + if (mbi.Protect == PAGE_READONLY) { + void *page = (void *)((UINT_PTR)addr & ~(UINT_PTR)(KB(4) - 1)); + DWORD old_protect = 0; + if (VirtualProtect(page, KB(4), PAGE_WRITECOPY, &old_protect)) { + InterlockedIncrement(&g_lnk_cow_promoted_pages); + return EXCEPTION_CONTINUE_EXECUTION; + } + } else if (mbi.Protect == PAGE_WRITECOPY || mbi.Protect == PAGE_READWRITE) { + // another thread promoted this page between our fault and the query; retry the write + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + internal THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) { @@ -254,11 +298,16 @@ THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) } else { HANDLE file_handle = CreateFileW(path16.str, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (file_handle != INVALID_HANDLE_VALUE) { + if (InterlockedCompareExchange(&g_lnk_cow_veh_installed, 1, 0) == 0) { + AddVectoredExceptionHandler(1, lnk_cow_page_promote_veh); + } HANDLE mapping_handle = CreateFileMappingA(file_handle, 0, PAGE_WRITECOPY, 0, 0, 0); if (mapping_handle != INVALID_HANDLE_VALUE) { LARGE_INTEGER file_size = {0}; GetFileSizeEx(file_handle, &file_size); - void *file_data = MapViewOfFile(mapping_handle, FILE_MAP_COPY, 0, 0, file_size.QuadPart); + // FILE_MAP_READ view of a WRITECOPY section: zero commit charge at map time; + // pages become writable one at a time through lnk_cow_page_promote_veh + void *file_data = MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_size.QuadPart); if (file_data) { task->data_arr.v[task_id] = str8(file_data, file_size.QuadPart); if (task->was_read) { @@ -291,6 +340,27 @@ THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) scratch_end(scratch); } +// Experimental lane cap for opening thousands of independent object mappings. Each lane pulls +// file indices from a shared cursor, so this preserves the ordinary per-file mapping operation and +// output ordering while letting us measure the Windows object-manager concurrency knee. +typedef struct +{ + LNK_DiskReader *reader; + U64 item_count; + U64 cursor; +} LNK_MemoryMapCappedTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_memory_map_file_capped_task) +{ + LNK_MemoryMapCappedTask *task = raw_task; + for (;;) { + U64 item_idx = ins_atomic_u64_inc_eval(&task->cursor) - 1; + if (item_idx >= task->item_count) { break; } + lnk_memory_map_file_task(arena, worker_id, item_idx, task->reader, tp); + } +} + internal String8Array lnk_read_data_from_file_path_parallel(TP_Context *tp, Arena *arena, LNK_IO_Flags io_flags, String8Array path_arr, B8 *was_read) { @@ -303,7 +373,33 @@ lnk_read_data_from_file_path_parallel(TP_Context *tp, Arena *arena, LNK_IO_Flags reader.data_arr.count = path_arr.count; reader.data_arr.v = push_array(arena, String8, path_arr.count); reader.was_read = was_read; - tp_for_parallel(tp, 0, path_arr.count, lnk_memory_map_file_task, &reader); + char *map_worker_env = getenv("RAD_COBJ_MAP_WORKERS"); + U64 map_worker_cap = map_worker_env ? strtoull(map_worker_env, 0, 10) : 0; +#if OS_WINDOWS + if (!map_worker_env && path_arr.count >= 64) { + // Standalone compressed objects expose a sparse raw COFF view. A small sample detects the + // homogeneous compressed corpus without opening or touching ordinary raw object contents. + // Raw links retain the original unrestricted tp_for_parallel call below. + Temp detect_scratch = scratch_begin(&arena, 1); + U64 sample_count = Min(path_arr.count, 32); + for EachIndex(i, sample_count) { + String16 path16 = str16_from_8(detect_scratch.arena, path_arr.v[i]); + DWORD attrs = GetFileAttributesW(path16.str); + if (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_SPARSE_FILE)) { + map_worker_cap = 8; + break; + } + } + scratch_end(detect_scratch); + } +#endif + if (map_worker_cap > 0 && map_worker_cap < path_arr.count) { + LNK_MemoryMapCappedTask map_task = { .reader = &reader, .item_count = path_arr.count }; + tp_for_parallel(tp, 0, map_worker_cap, lnk_memory_map_file_capped_task, &map_task); + } else { + // Default/raw behavior remains the original unrestricted parallel-for. + tp_for_parallel(tp, 0, path_arr.count, lnk_memory_map_file_task, &reader); + } } else { Temp scratch = scratch_begin(&arena,1); @@ -451,10 +547,12 @@ struct LNK_BackgroundFile B32 is_finished; B32 is_complete; B32 write_failed; + B32 open_failed; }; typedef enum { + LNK_BackgroundFileJobKind_OpenFile, LNK_BackgroundFileJobKind_Write, LNK_BackgroundFileJobKind_EndFile, LNK_BackgroundFileJobKind_EndWriter, @@ -467,11 +565,77 @@ typedef struct U64 file_off; U64 expected_byte_count; String8 data; + B32 decommit_after_write; // hand the buffer's pages back to the OS once written } LNK_BackgroundFileWriteJob; +typedef struct +{ + void *ptr; + U64 size; // zero marks the end of the queue +} LNK_BackgroundFileDecommitJob; + +internal void +lnk_background_file_decommit_thread(void *raw_writer) +{ + ProfBeginFunction(); + set_thread_namef("PDB Page Decommit"); + + LNK_BackgroundFileWriter *writer = raw_writer; + for (;;) { + LNK_BackgroundFileDecommitJob job = {0}; + RingGuard guard = guarded_ring_open(writer->decommit_queue); + B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_read); + if (job.size == 0) { break; } + decommit_memory(job.ptr, job.size); + ins_atomic_u64_add_eval(&writer->decommit_bytes_completed, job.size); + } + + ProfEnd(); +} + +// Opening the output file can cost seconds on the caller's thread when it +// replaces a previous multi-GB artifact (NTFS truncate/dealloc of the old +// allocation happens inside CreateFile). Jobs on one file are queue-ordered, +// so deferring the open to the writer thread keeps every write behind it +// while the caller returns immediately. +internal void +lnk_background_file_writer_open_file_on_thread(LNK_BackgroundFile *file) +{ + if (file->open_with_rename) { + file->file = lnk_file_open_with_rename_permissions(file->temp_path); + file->open_path = file->temp_path; + } else { + lnk_open_file_write((char *)file->path.str, file->path.size, &file->file, sizeof(file->file)); + file->open_path = file->path; + } + + file->is_open = !file_match(file->file, file_zero()); + if (!file->is_open) { + file->open_failed = 1; + file->write_failed = 1; + return; + } + + if (file->open_with_rename && !lnk_file_set_delete_on_close(file->file, 1)) { + lnk_error(LNK_Error_IO, "failed to update file disposition on %S", file->open_path); + lnk_close_file(&file->file); + file->is_open = 0; + file->open_failed = 1; + file->write_failed = 1; + } +} + internal void lnk_background_file_writer_end_file_on_thread(LNK_BackgroundFile *file, U64 expected_byte_count) { + if (file->open_failed) { + lnk_error(LNK_Error_NoAccess, "don't have access to write to %S", file->path); + file->is_complete = 0; + return; + } + B32 is_complete = !file->write_failed && file->bytes_written == expected_byte_count; if (is_complete && file->open_with_rename) { if (!lnk_file_set_delete_on_close(file->file, 0)) { @@ -494,6 +658,99 @@ lnk_background_file_writer_end_file_on_thread(LNK_BackgroundFile *file, U64 expe } } +internal int +lnk_background_file_write_job_is_before(const void *raw_a, const void *raw_b) +{ + const LNK_BackgroundFileWriteJob *a = raw_a, *b = raw_b; + if (a->file != b->file) { return a->file < b->file ? -1 : +1; } + if (a->file_off != b->file_off) { return a->file_off < b->file_off ? -1 : +1; } + return 0; +} + +// Write jobs on one file cover disjoint file ranges (each MSF page is enqueued exactly +// once: sealed streams are disjoint page sets, and the final remainder pass skips pages +// already enqueued), so reordering Write jobs among themselves cannot change the output. +// Control jobs (OpenFile/EndFile/EndWriter) are ordering barriers: the batch drain stops +// at the first control job and the batched writes are issued BEFORE it runs, preserving +// the queue-order guarantees (every write lands behind its file's open and before its +// file's end). +internal void +lnk_background_file_writer_issue_write_batch(LNK_BackgroundFileWriter *writer, LNK_BackgroundFileWriteJob *batch, U64 count) +{ + qsort(batch, count, sizeof(batch[0]), lnk_background_file_write_job_is_before); + for (U64 i = 0; i < count; ) { + // coalesce jobs that are contiguous in BOTH file space and memory into one write; + // MSF page numbers map linearly to page-data-node memory, so page runs from + // interleaved stream enqueues merge back into large sequential writes here + U64 j = i; + U64 end_off = batch[i].file_off + batch[i].data.size; + U8 *end_ptr = batch[i].data.str + batch[i].data.size; + for (; j + 1 < count; j += 1) { + LNK_BackgroundFileWriteJob *next = &batch[j+1]; + if (next->file != batch[i].file || + next->decommit_after_write != batch[i].decommit_after_write || + next->file_off != end_off || + next->data.str != end_ptr) { + break; + } + end_off += next->data.size; + end_ptr += next->data.size; + } + U64 merged_size = end_off - batch[i].file_off; + + // Keep at most one MSF data node of dead-but-still-committed page data in + // flight. Besides bounding the memory lag versus synchronous decommit, + // this prevents one multi-node VirtualFree from delaying all reclamation. + U64 decommit_pipeline_cap = MB(128); + U64 part_cap = writer->is_decommit_running && batch[i].decommit_after_write ? decommit_pipeline_cap : max_U64; + for (U64 part_off = 0; part_off < merged_size; ) { + U64 part_size = Min(merged_size - part_off, part_cap); + U8 *part_ptr = batch[i].data.str + part_off; + if (batch[i].file->is_open) { + U64 write_size = lnk_write_file(&batch[i].file->file, batch[i].file_off + part_off, part_ptr, part_size); + if (write_size != part_size) { + batch[i].file->write_failed = 1; + } + batch[i].file->bytes_written += write_size; + U64 done_now = ins_atomic_u64_add_eval(&writer->bytes_completed, write_size); + writer->writes_issued += 1; + if ((done_now >> 29) != ((done_now - write_size) >> 29)) { // every 512MiB + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs written=%.2f GiB enq=%.2f GiB jobs=%llu/%llu writes=%llu avg=%llu KiB", (F64)(now_time_us() - writer->begin_time_us) / 1e6, (F64)done_now / GB(1), (F64)ins_atomic_u64_eval(&writer->bytes_enqueued) / GB(1), writer->jobs_completed, ins_atomic_u64_eval(&writer->jobs_enqueued), writer->writes_issued, (done_now / writer->writes_issued) / KB(1)); + } + } + if (batch[i].decommit_after_write) { + // The buffer is immutable and write-once (a sealed MSF stream run); once it is + // on disk its pages are dead weight in the commit charge. Round INWARD to whole + // OS pages so neighbors sharing the boundary pages are untouched. + U64 lo = AlignPow2((U64)part_ptr, KB(4)); + U64 hi = ((U64)part_ptr + part_size) & ~(U64)(KB(4) - 1); + if (lo < hi) { + U64 decommit_size = hi - lo; + if (writer->is_decommit_running) { + while (ins_atomic_u64_eval(&writer->decommit_bytes_enqueued) - + ins_atomic_u64_eval(&writer->decommit_bytes_completed) + decommit_size > decommit_pipeline_cap) { + sleep_ms(0); + } + ins_atomic_u64_add_eval(&writer->decommit_bytes_enqueued, decommit_size); + LNK_BackgroundFileDecommitJob job = { .ptr = (void *)lo, .size = decommit_size }; + RingGuard guard = guarded_ring_open(writer->decommit_queue); + B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_written); + } else { + decommit_memory((void *)lo, decommit_size); + } + } + } + part_off += part_size; + } + if (batch[i].file->is_open) { + writer->jobs_completed += (j + 1 - i); + } + i = j + 1; + } +} + internal void lnk_background_file_writer_thread(void *raw_writer) { @@ -501,25 +758,77 @@ lnk_background_file_writer_thread(void *raw_writer) set_thread_namef("Background File Writer"); LNK_BackgroundFileWriter *writer = raw_writer; - for (;;) { - LNK_BackgroundFileWriteJob job = {0}; - RingGuard guard = guarded_ring_open(writer->queue); - B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64); - guarded_ring_close(&guard); - Assert(is_read); - if (job.kind == LNK_BackgroundFileJobKind_EndWriter) { break; } + writer->decommit_thread = thread_launch(lnk_background_file_decommit_thread, writer); + writer->is_decommit_running = writer->decommit_thread.u64[0] != 0; + U64 batch_cap = 8192; + LNK_BackgroundFileWriteJob *batch = (LNK_BackgroundFileWriteJob *)reserve_memory(batch_cap * sizeof(batch[0])); + commit_memory(batch, batch_cap * sizeof(batch[0])); + + for (B32 quit = 0; !quit; ) { + LNK_BackgroundFileWriteJob job = {0}; + { + RingGuard guard = guarded_ring_open(writer->queue); + B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_read); + } + + // batch up every immediately-available Write job (stop at the first control job -- + // it is an ordering barrier) so scattered small page runs coalesce into few large + // sequential writes instead of thousands of QD-1 16KB-class WriteFile calls + U64 batch_count = 0; + B32 have_control = 0; + LNK_BackgroundFileWriteJob control = {0}; if (job.kind == LNK_BackgroundFileJobKind_Write) { - U64 write_size = lnk_write_file(&job.file->file, job.file_off, job.data.str, job.data.size); - if (write_size != job.data.size) { - job.file->write_failed = 1; + batch[batch_count++] = job; + while (batch_count < batch_cap) { + LNK_BackgroundFileWriteJob more = {0}; + RingGuard guard = guarded_ring_open(writer->queue); + B32 is_read = guarded_ring_try_read(&guard, sizeof(more), &more); + guarded_ring_close(&guard); + if (!is_read) { break; } + if (more.kind == LNK_BackgroundFileJobKind_Write) { + batch[batch_count++] = more; + } else { + control = more; + have_control = 1; + break; + } + } + } else { + control = job; + have_control = 1; + } + + if (batch_count > 0) { + lnk_background_file_writer_issue_write_batch(writer, batch, batch_count); + } + + if (have_control) { + if (control.kind == LNK_BackgroundFileJobKind_EndWriter) { + quit = 1; + } else if (control.kind == LNK_BackgroundFileJobKind_OpenFile) { + lnk_background_file_writer_open_file_on_thread(control.file); + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs file open done (%S)", (F64)(now_time_us() - writer->begin_time_us) / 1e6, control.file->open_path); + } else if (control.kind == LNK_BackgroundFileJobKind_EndFile) { + lnk_background_file_writer_end_file_on_thread(control.file, control.expected_byte_count); } - job.file->bytes_written += write_size; - } else if (job.kind == LNK_BackgroundFileJobKind_EndFile) { - lnk_background_file_writer_end_file_on_thread(job.file, job.expected_byte_count); } } + + if (writer->is_decommit_running) { + LNK_BackgroundFileDecommitJob job = {0}; + RingGuard guard = guarded_ring_open(writer->decommit_queue); + B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_written); + thread_join(writer->decommit_thread, max_U64); + writer->is_decommit_running = 0; + } + + release_memory(batch, batch_cap * sizeof(batch[0])); ProfEnd(); } @@ -528,8 +837,10 @@ lnk_background_file_writer_begin(LNK_BackgroundFileWriter *writer) { ProfBegin("Background File Writer Begin"); - writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "BACKGROUND_FILE_WRITE_QUEUE"); - writer->queue = guarded_ring_alloc(writer->queue_arena, MB(1)); + writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "BACKGROUND_FILE_WRITE_QUEUE"); + writer->queue = guarded_ring_alloc(writer->queue_arena, MB(1)); + writer->decommit_queue = guarded_ring_alloc(writer->queue_arena, KB(64)); + writer->begin_time_us = now_time_us(); ProfEnd(); } @@ -554,48 +865,41 @@ lnk_background_file_writer_begin_file(LNK_BackgroundFileWriter *writer, String8 file->temp_path = temp_path; file->open_with_rename = (temp_path.size > 0); - if (file->open_with_rename) { - file->file = lnk_file_open_with_rename_permissions(temp_path); - file->open_path = temp_path; - } else { - lnk_open_file_write((char *)path.str, path.size, &file->file, sizeof(file->file)); - file->open_path = path; - } - - file->is_open = !file_match(file->file, file_zero()); - if (!file->is_open) { - lnk_error(LNK_Error_NoAccess, "don't have access to write to %S", path); - goto exit; - } + SLLQueuePush(writer->file_first, writer->file_last, file); - if (file->open_with_rename && !lnk_file_set_delete_on_close(file->file, 1)) { - lnk_error(LNK_Error_IO, "failed to update file disposition on %S", file->open_path); - lnk_close_file(&file->file); - file->is_open = 0; - goto exit; + // the open runs on the writer thread (multi-GB truncate of a previous + // artifact is seconds of caller-thread stall otherwise); jobs on one file + // are queue-ordered, so every write lands behind the open. An open failure + // flips write_failed and is reported from the EndFile job. + { + LNK_BackgroundFileWriteJob job = { .kind = LNK_BackgroundFileJobKind_OpenFile, .file = file }; + RingGuard guard = guarded_ring_open(writer->queue); + B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_written); } - SLLQueuePush(writer->file_first, writer->file_last, file); - - exit:; ProfEnd(); - return file->is_open ? file : 0; + return file; } internal void -lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data) +lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data, B32 decommit_after_write) { ProfBegin("Background File Writer Enqueue"); Assert(writer->is_running); - Assert(file->is_open && !file->is_finished); + Assert(!file->is_finished); // is_open is owned by the writer thread (open is a queued job) if (data.size > 0) { + ins_atomic_u64_add_eval(&writer->bytes_enqueued, data.size); + ins_atomic_u64_add_eval(&writer->jobs_enqueued, 1); LNK_BackgroundFileWriteJob job = { - .kind = LNK_BackgroundFileJobKind_Write, - .file = file, - .file_off = file_off, - .data = data, + .kind = LNK_BackgroundFileJobKind_Write, + .file = file, + .file_off = file_off, + .data = data, + .decommit_after_write = decommit_after_write, }; RingGuard guard = guarded_ring_open(writer->queue); B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); @@ -612,7 +916,7 @@ lnk_background_file_writer_end_file(LNK_BackgroundFileWriter *writer, LNK_Backgr ProfBegin("Background File Writer End File"); Assert(writer->is_running); - Assert(file->is_open && !file->is_finished); + Assert(!file->is_finished); // is_open is owned by the writer thread (open is a queued job) file->is_finished = 1; LNK_BackgroundFileWriteJob job = { @@ -634,12 +938,17 @@ lnk_background_file_writer_end(LNK_BackgroundFileWriter *writer) ProfBegin("Background File Writer End"); if (writer->is_running) { + U64 wait_begin_us = now_time_us(); LNK_BackgroundFileWriteJob job = { .kind = LNK_BackgroundFileJobKind_EndWriter }; RingGuard guard = guarded_ring_open(writer->queue); B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); guarded_ring_close(&guard); Assert(is_written); thread_join(writer->thread, -1); + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs writer end: drain wait %.2f ms (enq=%.2f GiB done=%.2f GiB)", + (F64)(now_time_us() - writer->begin_time_us) / 1e6, + (F64)(now_time_us() - wait_begin_us) / 1000.0, + (F64)writer->bytes_enqueued / GB(1), (F64)writer->bytes_completed / GB(1)); } for EachNode(file, LNK_BackgroundFile, writer->file_first) { diff --git a/src/linker/lnk_io.h b/src/linker/lnk_io.h index 442456e46..dfc7414bb 100644 --- a/src/linker/lnk_io.h +++ b/src/linker/lnk_io.h @@ -30,10 +30,21 @@ typedef struct { Arena *queue_arena; GuardedRing *queue; + GuardedRing *decommit_queue; Thread thread; + Thread decommit_thread; LNK_BackgroundFile *file_first; LNK_BackgroundFile *file_last; B32 is_running; + B32 is_decommit_running; + U64 begin_time_us; // Timers telemetry anchor + U64 bytes_enqueued; // atomic; producer side + U64 bytes_completed; // atomic; writer thread + U64 decommit_bytes_enqueued; // atomic; writer thread + U64 decommit_bytes_completed; // atomic; decommit thread + U64 jobs_enqueued; // atomic; producer side + U64 jobs_completed; // writer thread + U64 writes_issued; // writer thread; post-coalesce WriteFile count } LNK_BackgroundFileWriter; // --- Shared File API --------------------------------------------------------- @@ -62,6 +73,6 @@ internal String8 lnk_data_from_file_artifact(Arena *arena, LNK_FileArtifact *art internal void lnk_background_file_writer_begin (LNK_BackgroundFileWriter *writer); internal LNK_BackgroundFile *lnk_background_file_writer_begin_file(LNK_BackgroundFileWriter *writer, String8 path, String8 temp_path); -internal void lnk_background_file_writer_enqueue (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data); +internal void lnk_background_file_writer_enqueue (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data, B32 decommit_after_write); internal void lnk_background_file_writer_end_file (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 expected_byte_count); internal void lnk_background_file_writer_end (LNK_BackgroundFileWriter *writer); diff --git a/src/linker/lnk_lib.c b/src/linker/lnk_lib.c index 50bc4bd25..76173acfe 100644 --- a/src/linker/lnk_lib.c +++ b/src/linker/lnk_lib.c @@ -20,6 +20,25 @@ lnk_first_member_sort_key_is_before(void *raw_a, void *raw_b) return str8_is_before_case_sensitive(&a->symbol_name, &b->symbol_name); } +internal force_inline U64 +lnk_symbol_name_disc(String8 name) +{ + // pack the first 8 bytes big-endian so an integer compare orders exactly like memcmp; pad short + // names with zeros. this is faithful to str8_compar_case_sensitive INCLUDING the size tie-break + // (base_strings.c: shorter prefix precedes): at the first differing padded byte either both + // strings have a real byte there (== memcmp order), or the shorter string's zero pad compares + // below the longer string's next real byte (== shorter-prefix-precedes). a zero pad byte can tie + // only with another pad byte or an embedded NUL, and any such tie leaves the discriminators + // marching in lockstep until a real difference or full equality -- so disc inequality ALWAYS + // decides the compare, and disc equality falls through to the full str8_compar. + U64 disc = 0; + U64 n = Min(name.size, 8); + for (U64 i = 0; i < n; i += 1) { + disc |= (U64)name.str[i] << (56 - i*8); + } + return disc; +} + internal B32 lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_Lib *lib_out) { @@ -128,7 +147,14 @@ lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_L scratch_end(scratch); } - + + // build packed discriminators parallel to the (sorted) symbol name dir; bsearch probes read this + // contiguous array and touch archive string-table bytes only on discriminator ties + U64 *symbol_discs = push_array_no_zero(arena, U64, symbol_names.count); + for EachIndex(symbol_idx, symbol_names.count) { + symbol_discs[symbol_idx] = lnk_symbol_name_disc(symbol_names.v[symbol_idx]); + } + // init lib lib_out->path = push_str8_copy(arena, path); lib_out->data = data; @@ -138,6 +164,7 @@ lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_L lib_out->member_offsets = member_offsets; lib_out->symbol_indices = symbol_indices; lib_out->symbol_names = symbol_names; + lib_out->symbol_discs = symbol_discs; lib_out->long_names = parse.long_names; lib_out->input_idx = input_idx; @@ -158,7 +185,7 @@ THREAD_POOL_TASK_FUNC(lnk_lib_initer) U64 valid_lib_idx = ins_atomic_u64_inc_eval(&task->valid_libs_count)-1; task->valid_libs[valid_lib_idx] = lib_node; } else { - U64 invalid_lib_idx = ins_atomic_u64_inc_eval(&task->invalid_libs_count); + U64 invalid_lib_idx = ins_atomic_u64_inc_eval(&task->invalid_libs_count)-1; task->invalid_libs[invalid_lib_idx] = lib_node; } } @@ -201,7 +228,7 @@ lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U radsort(task.invalid_libs, task.invalid_libs_count, lnk_lib_node_ptr_is_before); for EachIndex(i, task.invalid_libs_count) { U64 input_idx = task.invalid_libs[i]->data.input_idx; - lnk_error(LNK_Error_InvalidLib, "%S: failed to parse library", inputs[input_idx]->path); + lnk_error(LNK_Error_InvalidLib, "%S: failed to parse library", inputs[input_idx - lib_id_base]->path); } // push parsed libs @@ -216,10 +243,61 @@ lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U return result; } +internal force_inline int +lnk_disc_str8_compar(U64 a_disc, String8 *a, U64 b_disc, String8 *b) +{ + // discriminator inequality decides the compare without touching string bytes (see + // lnk_symbol_name_disc for the order-fidelity argument); ties fall through to the full compare + if (a_disc != b_disc) { + return a_disc < b_disc ? -1 : +1; + } + return str8_compar_case_sensitive(a, b); +} + +// str8_array_bsearch with a packed-discriminator pre-filter: identical probe sequence and result +// (each probe's compare outcome is identical), but probes read the contiguous disc[] array instead +// of chasing symbol_names.v[].str into scattered archive string-table bytes +internal U64 +lnk_lib_bsearch_symbol_name(LNK_Lib *lib, String8 value) +{ + String8Array arr = lib->symbol_names; + U64 *disc = lib->symbol_discs; + if (arr.count > 1) { + U64 value_disc = lnk_symbol_name_disc(value); + + int lo_compar = lnk_disc_str8_compar(value_disc, &value, disc[0], &arr.v[0]); + if (lo_compar == 0) { + return 0; + } + + int hi_compar = lnk_disc_str8_compar(value_disc, &value, disc[arr.count-1], &arr.v[arr.count-1]); + if (hi_compar == 0) { + return arr.count-1; + } + + if (lo_compar > 0 && hi_compar < 0) { + for (U64 l = 0, r = arr.count-1; l <= r; ) { + U64 m = l + (r - l) / 2; + int cmp = lnk_disc_str8_compar(disc[m], &arr.v[m], value_disc, &value); + if (cmp == 0) { + return m; + } else if (cmp < 0) { + l = m + 1; + } else { + r = m - 1; + } + } + } + } else if (arr.count == 1 && str8_match(arr.v[0], value, 0)) { + return 0; + } + return max_U64; +} + internal force_inline B32 lnk_search_lib(LNK_Lib *lib, String8 symbol_name, U32 *member_idx_out) { - U64 symbol_idx = str8_array_bsearch(lib->symbol_names, symbol_name); + U64 symbol_idx = lnk_lib_bsearch_symbol_name(lib, symbol_name); if (symbol_idx < lib->symbol_count) { if (member_idx_out) { *member_idx_out = lib->symbol_indices[symbol_idx]-1; diff --git a/src/linker/lnk_lib.h b/src/linker/lnk_lib.h index 6db16e293..589fb898a 100644 --- a/src/linker/lnk_lib.h +++ b/src/linker/lnk_lib.h @@ -13,12 +13,16 @@ typedef struct LNK_Lib U32 *member_offsets; U16 *symbol_indices; String8Array symbol_names; + + // symbol-dir bsearch pre-filter: symbol_names.v[].str points into the mapped archive's string + // table, so every bsearch probe's MemCompare chases scattered archive bytes (sorted order != + // memory order -> no locality). symbol_discs[i] packs the first 8 bytes of symbol_names.v[i] + // big-endian (zero-padded), so an integer compare of discriminators decides str8_compar order + // whenever they differ; probes touch archive bytes only on discriminator ties. Parallel to + // symbol_names, built once at parse time. + U64 *symbol_discs; String8 long_names; U64 input_idx; - - struct LNK_SymbolHashTrieChunk **search_cursor_chunks; - U64 *search_cursor_indices; - B32 searched_anti_deps; } LNK_Lib; typedef struct LNK_LibNode @@ -70,5 +74,6 @@ internal LNK_Lib ** lnk_array_from_lib_list(Arena *arena, LNK_LibList list internal void lnk_lib_list_push_node(LNK_LibList *list, LNK_LibNode *node); internal LNK_LibNodeArray lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U64 inputs_count, struct LNK_Input **inputs); +internal U64 lnk_lib_bsearch_symbol_name(LNK_Lib *lib, String8 value); internal force_inline B32 lnk_search_lib(LNK_Lib *lib, String8 symbol_name, U32 *member_idx_out); diff --git a/src/linker/lnk_log.c b/src/linker/lnk_log.c index 5dce5875d..d8700d3c1 100644 --- a/src/linker/lnk_log.c +++ b/src/linker/lnk_log.c @@ -28,6 +28,9 @@ lnk_fprintf(FILE *f, char *fmt, ...) internal void lnk_exit(int code) { + // one-line summary must reach the build log on error exits too (best-effort; + // takes/drops g_log_mutex internally, so print BEFORE taking it here) + lnk_print_summary(code); mutex_take(g_log_mutex); fflush(stdout); fflush(stderr); diff --git a/src/linker/lnk_log.h b/src/linker/lnk_log.h index b10c836b5..01bf5fae2 100644 --- a/src/linker/lnk_log.h +++ b/src/linker/lnk_log.h @@ -13,7 +13,8 @@ X(SizeBreakdown) \ X(LinkStats) \ X(Timers) \ - X(Links) + X(Links) \ + X(Summary) typedef enum { diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index b79edf31b..d55e003f1 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -1,6 +1,25 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) +typedef struct LNK_CompressedObjCensus +{ + U64 obj_count; + U64 raw_bytes; + U64 section_bytes; + U64 debug_s_bytes; + U64 debug_t_bytes; + U64 debug_p_bytes; + U64 debug_h_bytes; + U64 other_section_bytes; + U64 symbol_bytes; + U64 string_bytes; + U64 section_table_bytes; + U64 reloc_bytes; +} LNK_CompressedObjCensus; + +global LNK_CompressedObjCensus g_lnk_compressed_obj_census; +global B32 g_lnk_compressed_obj_census_enabled; + internal String8 lnk_loc_from_obj(Arena *arena, LNK_Obj *obj) { @@ -107,6 +126,12 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) U32 debug_p_section_number = 0; U32 debug_h_section_number = 0; U32 llvm_addrsig_section_number = 0; + U64 census_section_bytes = 0; + U64 census_debug_s_bytes = 0; + U64 census_debug_t_bytes = 0; + U64 census_debug_p_bytes = 0; + U64 census_debug_h_bytes = 0; + U64 census_reloc_bytes = 0; for EachIndex(sect_idx, header.section_count_no_null) { U64 section_number = sect_idx + 1; COFF_SectionHeader *coff_sect_header = &coff_section_table[sect_idx]; @@ -114,6 +139,15 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) String8 sect_name = coff_name_from_section_header(raw_coff_string_table, coff_sect_header); *section_flags = coff_sect_header->flags & ~3; // linker reserves low 2 bits for internal flags + if (g_lnk_compressed_obj_census_enabled) { + census_section_bytes += coff_sect_header->fsize; + census_reloc_bytes += (U64)coff_sect_header->reloc_count * sizeof(COFF_Reloc); + if (str8_match(sect_name, str8_lit(".debug$S"), 0)) { census_debug_s_bytes += coff_sect_header->fsize; } + if (str8_match(sect_name, str8_lit(".debug$T"), 0)) { census_debug_t_bytes += coff_sect_header->fsize; } + if (str8_match(sect_name, str8_lit(".debug$P"), 0)) { census_debug_p_bytes += coff_sect_header->fsize; } + if (str8_match(sect_name, str8_lit(".debug$H"), 0)) { census_debug_h_bytes += coff_sect_header->fsize; } + } + if (str8_starts_with(sect_name, str8_lit(".debug$"))) { *section_flags |= LNK_SECTION_FLAG_DEBUG; } @@ -169,6 +203,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) } } + U32 *associated_section_offsets = push_array(arena, U32, header.section_count_no_null + 2); U64 symbol_block_count = CeilIntegerDiv(header.symbol_count, 64); LNK_ObjSymbolArray symbols = { .count = primary_symbol_count, @@ -220,6 +255,8 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) if (select == COFF_ComdatSelect_Associative) { if (section_number == 0 || section_number > header.section_count_no_null) { lnk_error_input_obj(LNK_Error_IllData, input, "section definition symbol %S (No. 0x%x) associates with an out of bounds section 0x%x", symbol.name, symbol_idx, section_number); + } else if (symbol.section_number > 0 && symbol.section_number <= header.section_count_no_null) { + associated_section_offsets[section_number + 1] += 1; } } } @@ -231,6 +268,14 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) Assert(primary_idx == primary_symbol_count); } + // Convert per-parent association counts to CSR offsets. The offsets double as fill cursors + // below; the backwards pass restores them and the old SLLStackPush sibling order. + for (U64 section_number = 1; section_number <= header.section_count_no_null; section_number += 1) { + associated_section_offsets[section_number + 1] += associated_section_offsets[section_number]; + } + U32 associated_section_count = associated_section_offsets[header.section_count_no_null + 1]; + U32 *associated_section_numbers = push_array_no_zero(arena, U32, associated_section_count); + // // create symbol links to COMDAT sections // @@ -248,10 +293,16 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) if (symbol.storage_class == COFF_SymStorageClass_Static) { if (symbol.section_number > 0 && symbol.section_number <= header.section_count_no_null) { COFF_SectionHeader *sect_header = &coff_section_table[symbol.section_number-1]; - if (section_headers[symbol.section_number].flags & COFF_SectionFlag_LnkCOMDAT) { - if (symbol.aux_symbol_count) { - U32 section_length = 0; - coff_parse_secdef(symbol, header.is_big_obj, 0, 0, §ion_length, 0); + if (symbol.aux_symbol_count) { + COFF_ComdatSelectType selection = COFF_ComdatSelect_Null; + U32 section_number = 0; + U32 section_length = 0; + coff_parse_secdef(symbol, header.is_big_obj, &selection, §ion_number, §ion_length, 0); + if (selection == COFF_ComdatSelect_Associative && section_number > 0 && section_number <= header.section_count_no_null) { + U32 cursor = associated_section_offsets[section_number]++; + associated_section_numbers[cursor] = symbol.section_number; + } + if (section_headers[symbol.section_number].flags & COFF_SectionFlag_LnkCOMDAT) { if (sect_header->fsize == section_length) { if (comdats[symbol.section_number] == ~0) { comdats[symbol.section_number] = symbol_idx; @@ -270,6 +321,14 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) } } } + for (U64 section_number = header.section_count_no_null; section_number > 0; section_number -= 1) { + associated_section_offsets[section_number] = associated_section_offsets[section_number - 1]; + U32 min = associated_section_offsets[section_number]; + U32 max = associated_section_offsets[section_number + 1]; + for (U32 child_idx = 0; child_idx < (max - min) / 2; child_idx += 1) { + Swap(U32, associated_section_numbers[min + child_idx], associated_section_numbers[max - child_idx - 1]); + } + } // // COMDAT loop checker @@ -320,28 +379,6 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) scratch_end(scratch); } - // - // collect sections associations - // - U32Node **associated_sections = push_array(arena, U32Node *, header.section_count_no_null + 1); - { - COFF_ParsedSymbol symbol; - for (U32 symbol_idx = 0; symbol_idx < header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { - symbol = coff_parse_symbol_no_name(header, raw_coff_symbol_table, symbol_idx); - COFF_SymbolValueInterpType interp = coff_interp_from_parsed_symbol(symbol); - if (interp == COFF_SymbolValueInterp_Regular && symbol.storage_class == COFF_SymStorageClass_Static && symbol.aux_symbol_count > 0) { - COFF_ComdatSelectType selection = COFF_ComdatSelect_Null; - U32 section_number = 0; - coff_parse_secdef(symbol, header.is_big_obj, &selection, §ion_number, 0, 0); - if (selection != COFF_ComdatSelect_Associative) { continue; } - - U32Node *associated_node = push_array(arena, U32Node, 1); - associated_node->data = symbol.section_number; - SLLStackPush(associated_sections[section_number], associated_node); - } - } - } - B8 hotpatch = 0; if (header.machine == COFF_MachineType_X64) { hotpatch = 1; @@ -359,19 +396,44 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) String8 name = str8_cstring_capped(sect_header->name, sect_header->name+sizeof(sect_header->name)); if (str8_match(name, str8_lit(".debug$S"), 0)) { Temp temp = temp_begin(scratch.arena); - String8 debug_s_data = str8_substr(input->data, rng_1u64(sect_header->foff, sect_header->foff+sect_header->fsize)); - CV_DebugS debug_s = cv_debug_s_from_data(temp.arena, debug_s_data); - for EachNode(symbols_n, String8Node, debug_s.data_list[CV_C13SubSectionIdxKind_Symbols].first) { - for (U64 cursor = 0, count = 0; cursor < symbols_n->string.size && count < 2; count += 1) { - CV_SymbolHeader symbol_header; - TryReadBreak(str8_deserial_read_struct(symbols_n->string, cursor, &symbol_header), cursor); - if (symbol_header.kind == CV_SymKind_COMPILE3) { - String8 raw_symbol = str8_substr(symbols_n->string, r1u64(cursor, cursor + symbol_header.size + sizeof(CV_SymSize))); - comp_symbol = cv_symbol_from_ptr(raw_symbol.str); - goto found_comp_symbol; + Rng1U64 debug_s_range = rng_1u64(sect_header->foff, sect_header->foff+sect_header->fsize); + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(input->compressed_obj, debug_s_range, &indexed)) { + // COMPILE3 is expected in the first two records of a Symbols payload. The sidecar + // lets this early object-feature probe skip the otherwise full C13 header walk. + for EachIndex(entry_idx, indexed.count) { + LNK_CObjDebugSEntry *entry = &indexed.v[entry_idx]; + if (entry->kind != CV_C13SubSectionKind_Symbols) { continue; } + String8 symbols = str8(input->data.str + entry->raw_payload_offset, entry->raw_payload_size); + for (U64 cursor = 0, count = 0; cursor < symbols.size && count < 2; count += 1) { + CV_SymbolHeader symbol_header; + TryReadBreak(str8_deserial_read_struct(symbols, cursor, &symbol_header), cursor); + if (symbol_header.kind == CV_SymKind_COMPILE3) { + String8 raw_symbol = str8_substr(symbols, r1u64(cursor, cursor + symbol_header.size + sizeof(CV_SymSize))); + comp_symbol = cv_symbol_from_ptr(raw_symbol.str); + goto found_comp_symbol; + } + cursor += symbol_header.size + sizeof(CV_SymSize); + cursor = AlignPow2(cursor, CV_SymbolAlign); + } + } + } else { + String8 debug_s_data = str8_substr(input->data, debug_s_range); + CV_DebugS debug_s = cv_debug_s_from_data(temp.arena, debug_s_data); + cv_debug_s_tag_prov_sect(&debug_s, (U32)sect_idx); + String8List symbols_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + for EachNode(symbols_n, String8Node, symbols_list.first) { + for (U64 cursor = 0, count = 0; cursor < symbols_n->string.size && count < 2; count += 1) { + CV_SymbolHeader symbol_header; + TryReadBreak(str8_deserial_read_struct(symbols_n->string, cursor, &symbol_header), cursor); + if (symbol_header.kind == CV_SymKind_COMPILE3) { + String8 raw_symbol = str8_substr(symbols_n->string, r1u64(cursor, cursor + symbol_header.size + sizeof(CV_SymSize))); + comp_symbol = cv_symbol_from_ptr(raw_symbol.str); + goto found_comp_symbol; + } + cursor += symbol_header.size + sizeof(CV_SymSize); + cursor = AlignPow2(cursor, CV_SymbolAlign); } - cursor += symbol_header.size + sizeof(CV_SymSize); - cursor = AlignPow2(cursor, CV_SymbolAlign); } } temp_end(temp); @@ -396,7 +458,8 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) .count_no_null = header.section_count_no_null, .headers = section_headers, .comdats = comdats, - .associated_section_numbers = associated_sections, + .associated_section_offsets = associated_section_offsets, + .associated_section_numbers = associated_section_numbers, }, .symbols = symbols, .debug_t_section_number = debug_t_section_number, @@ -405,10 +468,41 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) .llvm_addrsig_section_number = llvm_addrsig_section_number, .hotpatch = hotpatch, }; + obj->compressed_obj = input->compressed_obj; obj->path = push_str8_copy(arena, input->path); obj->exclude_from_debug_info = input->exclude_from_debug_info; obj->self = &task->objs[task_id]; obj->link_member = input->link_member; + + if (input->compressed_obj != 0 && g_lnk_compressed_obj_census_enabled) { + U64 known_debug = census_debug_s_bytes + census_debug_t_bytes + census_debug_p_bytes + census_debug_h_bytes; + ins_atomic_u64_inc_eval(&g_lnk_compressed_obj_census.obj_count); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.raw_bytes, input->data.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.section_bytes, census_section_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_s_bytes, census_debug_s_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_t_bytes, census_debug_t_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_p_bytes, census_debug_p_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_h_bytes, census_debug_h_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.other_section_bytes, census_section_bytes - known_debug); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.symbol_bytes, raw_coff_symbol_table.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.string_bytes, raw_coff_string_table.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.section_table_bytes, raw_coff_section_table.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.reloc_bytes, census_reloc_bytes); + } +} + +internal void +lnk_obj_log_compressed_census(void) +{ + if (!g_lnk_compressed_obj_census_enabled || g_lnk_compressed_obj_census.obj_count == 0) { return; } + LNK_CompressedObjCensus *c = &g_lnk_compressed_obj_census; + lnk_log(LNK_Log_Timers, + "[cobj census] objs=%llu raw=%.2f GiB sections=%.2f GiB debugS=%.2f GiB debugT=%.2f GiB debugP=%.2f GiB debugH=%.2f GiB other=%.2f GiB symbols=%.2f GiB strings=%.2f GiB sect_headers=%.2f GiB relocs=%.2f GiB", + c->obj_count, (F64)c->raw_bytes/GB(1), (F64)c->section_bytes/GB(1), + (F64)c->debug_s_bytes/GB(1), (F64)c->debug_t_bytes/GB(1), (F64)c->debug_p_bytes/GB(1), + (F64)c->debug_h_bytes/GB(1), (F64)c->other_section_bytes/GB(1), + (F64)c->symbol_bytes/GB(1), (F64)c->string_bytes/GB(1), + (F64)c->section_table_bytes/GB(1), (F64)c->reloc_bytes/GB(1)); } internal LNK_ObjNode * @@ -416,6 +510,8 @@ lnk_obj_from_input_many(TP_Context *tp, TP_Arena *arena, LNK_Config *config, U64 { LNK_ObjNode *objs = 0; if (inputs_count) { + char *census_env = getenv("RAD_COBJ_CENSUS"); + g_lnk_compressed_obj_census_enabled = census_env != 0 && census_env[0] != 0 && census_env[0] != '0'; objs = push_array(arena->v[0], LNK_ObjNode, inputs_count); LNK_ObjIniter task = { .inputs = inputs, @@ -569,8 +665,9 @@ lnk_obj_collect_associated_section_numbers(Arena *arena, LNK_Obj *obj, U32 root_ // walk the complete descendant chain because associated COMDATs can nest for EachNode(parent_n, U32Node, queue.first) { - for EachNode(associated_n, U32Node, obj->coff.sections.associated_section_numbers[parent_n->data]) { - U32 child_section_number = associated_n->data; + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(obj, parent_n->data); + for EachIndex(associated_idx, associated_sections.count) { + U32 child_section_number = associated_sections.v[associated_idx]; if (child_section_number == 0) { continue; } if (hash_map_search_u64_u64(&seen_hm, child_section_number)) { continue; } @@ -673,13 +770,30 @@ lnk_obj_get_comdat_symlink_from_section_number(LNK_Obj *obj, U64 section_number, return is_valid; } +internal U32Array +lnk_obj_associated_sections_from_section_number(LNK_Obj *obj, U32 section_number) +{ + Assert(section_number <= obj->coff.sections.count_no_null); + U32 min = obj->coff.sections.associated_section_offsets[section_number]; + U32 max = obj->coff.sections.associated_section_offsets[section_number + 1]; + U32Array result = { .count = max - min }; + if (result.count) { + result.v = obj->coff.sections.associated_section_numbers + min; + } + return result; +} internal String8 lnk_obj_section_data_from_number(LNK_Obj *obj, U64 section_number) { Assert(1 <= section_number && section_number <= obj->coff.sections.count_no_null); + if (obj->section_data_copies != 0 && obj->section_data_copies[section_number].size != 0) { + return obj->section_data_copies[section_number]; + } COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(obj->coff.data, obj->coff.header.section_table_range).str; COFF_SectionHeader *section = §ion_table[section_number-1]; - return str8_substr(obj->coff.data, r1u64s(section->foff, section->fsize)); + Rng1U64 range = r1u64s(section->foff, section->fsize); + String8 direct = lnk_compressed_obj_direct_range(obj->compressed_obj, range); + return direct.size ? direct : str8_substr(obj->coff.data, range); } internal U64 @@ -851,12 +965,179 @@ lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) return result; } +// Drop the obj's patched debug-section copies: zero each entry so +// lnk_obj_section_data_from_number +// falls back to the (still-mapped) input view. The copy BYTES live on the shared +// per-worker SECT_DATA_COPIES arenas (see lnk_obj_reloc_patcher) and are handed back +// wholesale via arena_release at the caller; only call once every reader of the patched +// bytes is done. Idempotent. +internal void +lnk_obj_drop_section_data_copies(LNK_Obj *obj) +{ + if (obj->section_data_copies == 0) { return; } + for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { + obj->section_data_copies[section_number] = str8_zero(); + } +} + +// Resolve a .debug$S subsection provenance record (streaming-ring P1) back to its bytes. +// Returns exactly what the parse consumed: lnk_obj_section_data_from_number prefers the reloc-patched +// private copy when one exists (the parsed node slices point into that same copy), otherwise +// the raw mapped input. Synthetic provenance (linker-made bytes, is_synthetic) and untagged +// records (sect_idx == CV_DebugSProvSect_Nil) have no backing section to resolve through: +// returns str8_zero and the caller must fall back to the node's String8. +internal String8 +lnk_resolve_debug_s_node(LNK_Obj *obj, CV_DebugSProvNode *prov) +{ + if (prov == 0 || prov->is_synthetic || prov->sect_idx == CV_DebugSProvSect_Nil) { + return str8_zero(); + } + U64 section_number = (U64)prov->sect_idx + 1; + String8 sect_data = lnk_obj_section_data_from_number(obj, section_number); + return str8_substr(sect_data, rng_1u64(prov->off, prov->off + prov->size)); +} + +// reloc patch ordering: sort each section's relocs by apply_off so the RMW write stream into +// the destination buffer is monotone-forward (HW-prefetchable) instead of scattered in on-disk +// table order. apply_off is the primary key; orig_idx is a tiebreak so the total order is +// deterministic and the final bytes are identical to the unsorted patch order (each reloc +// writes its own disjoint field; only a pathological same-apply_off overlap could depend on +// order, and the orig_idx tiebreak preserves the original sequence there too). +typedef struct LNK_RelocSortKey +{ + COFF_Reloc reloc; + U32 orig_idx; +} LNK_RelocSortKey; + +internal int +lnk_reloc_sort_key_is_before(void *raw_a, void *raw_b) +{ + LNK_RelocSortKey *a = raw_a, *b = raw_b; + if (a->reloc.apply_off != b->reloc.apply_off) { + return a->reloc.apply_off < b->reloc.apply_off; + } + return a->orig_idx < b->orig_idx; +} + +// Applies one section's relocations into `section_data` -- a writable buffer holding that +// section's bytes (the section's slice of the image at image-build time, a private patched +// copy for non-$S debug sections, or the streaming-ring window copy at module-write time). +// Factored out of lnk_obj_reloc_patcher so the image patch pass and the P3.3 window fill +// share one definition: same reloc order, same symbol resolution, same skip semantics +// (debug-section relocs against removed sections are silently dropped), byte-identical +// application at either call time (relocs + symbol tables + image section table are all +// immutable after the image build). +internal void +lnk_obj_apply_relocs_to_buffer(LNK_Obj *obj, U64 section_number, COFF_SectionHeader *section_header, String8 section_data, U64 image_base, COFF_SectionHeader **image_section_table) +{ + Assert(1 <= section_number && section_number <= obj->coff.sections.count_no_null); + COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + if (relocs.count == 0) { return; } + + Temp scratch = scratch_begin(0, 0); + COFF_SectionFlags section_flags = obj->coff.sections.headers[section_number].flags; + + // apply relocs (sorted by apply_off for monotone-forward writes) + LNK_RelocSortKey *sorted_relocs = push_array_no_zero(scratch.arena, LNK_RelocSortKey, relocs.count); + for EachIndex(reloc_idx, relocs.count) { + sorted_relocs[reloc_idx].reloc = relocs.v[reloc_idx]; + sorted_relocs[reloc_idx].orig_idx = (U32)reloc_idx; + } + radsort(sorted_relocs, relocs.count, lnk_reloc_sort_key_is_before); + for EachIndex(reloc_idx, relocs.count) { + COFF_Reloc *reloc = &sorted_relocs[reloc_idx].reloc; + + // error check relocation + if (obj->coff.header.machine == COFF_MachineType_X64) { + if (reloc->type > COFF_Reloc_X64_Last) { + lnk_error_obj(LNK_Error_IllegalRelocation, obj, "unknown relocation type 0x%x", reloc->type); + } + } else if (obj->coff.header.machine != COFF_MachineType_Unknown) { + lnk_not_implemented("relocation patching is not implemented for %S", coff_string_from_machine_type(obj->coff.header.machine)); + continue; + } + + // compute virtual offsets + U64 reloc_voff = section_header->voff + reloc->apply_off; + + // compute symbol location values + U32 symbol_secnum = 0; + U32 symbol_secoff = 0; + S64 symbol_voff = 0; + { + COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, reloc->isymbol); + COFF_SymbolValueInterpType interp = coff_interp_from_parsed_symbol(symbol); + if (interp == COFF_SymbolValueInterp_Regular) { + if (symbol.section_number == lnk_obj_get_removed_section_number(obj)) { + if (~section_flags & LNK_SECTION_FLAG_DEBUG) { + String8 sect_name = lnk_obj_section_name_from_section_number(obj, section_number); + String8 symbol_name = lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol); + lnk_error_obj(LNK_Error_RelocationAgainstRemovedSection, obj, "relocating against symbol that is in a removed section (symbol: %S, reloc-section: %S 0x%llx, reloc-index: 0x%llx)", symbol_name, sect_name, section_number, reloc_idx); + } + continue; + } + symbol_secnum = symbol.section_number; + symbol_secoff = symbol.value; + symbol_voff = safe_cast_u32((U64)image_section_table[symbol.section_number]->voff + (U64)symbol_secoff); + } else if (interp == COFF_SymbolValueInterp_Abs) { + // There aren't enough bits in COFF symbol to store full image base address, + // so we special case __ImageBase. A better solution would be to add + // a 64-bit symbol format to COFF. + if (str8_match(lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol), str8_lit("__ImageBase"), 0)) { + symbol.value = image_base; + } + symbol_secnum = 0; + symbol_secoff = 0; + symbol_voff = (S64)symbol.value - (S64)image_base; + } else if (interp == COFF_SymbolValueInterp_Weak) { + // unresolved weak + } else if (interp == COFF_SymbolValueInterp_Undefined) { + // unresolved undefined + } else { + InvalidPath; + } + } + + // pick reloc value + COFF_RelocValue reloc_value = {0}; + switch (obj->coff.header.machine) { + case COFF_MachineType_Unknown: {} break; + case COFF_MachineType_X64: { reloc_value = coff_pick_reloc_value_x64(reloc->type, image_base, reloc_voff, symbol_secnum, symbol_secoff, symbol_voff); } break; + default: { NotImplemented; } break; + } + + // read addend + Assert(reloc_value.size <= section_data.size); + U64 raw_addend = 0; + str8_deserial_read(section_data, reloc->apply_off, &raw_addend, reloc_value.size, 1); + + // compute new reloc value + S64 addend = extend_sign64(raw_addend, reloc_value.size); + U64 reloc_result = reloc_value.value + addend; + + // commit new reloc value + MemoryCopy(section_data.str + reloc->apply_off, &reloc_result, reloc_value.size); + } + scratch_end(scratch); +} + internal THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) { LNK_SectionCollector *task = raw_task; LNK_Obj *obj = task->objs[task_id]; + // Optional 0-based section-index sidecar, kept in the same order as out_lists. + if (task->out_sect_indices != 0) { + U64 match_count = 0; + for LNK_EachCoffSection(count_it, obj) { + if (*count_it.v.flags & COFF_SectionFlag_LnkRemove && !task->collect_discarded) { continue; } + if (str8_match(lnk_obj_section_name_from_section_number(obj, count_it.v.section_number), task->name, 0)) { match_count += 1; } + } + task->out_sect_indices[task_id].count = 0; + task->out_sect_indices[task_id].v = push_array_no_zero(arena, U32, match_count ? match_count : 1); + } + for LNK_EachCoffSection(it, obj) { LNK_ObjSection section = it.v; @@ -868,19 +1149,29 @@ THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) if (str8_match(section_name, task->name, 0)) { String8 section_data = lnk_obj_section_data_from_number(obj, section.section_number); str8_list_push(arena, &task->out_lists[task_id], section_data); + if (task->out_sect_indices != 0) { + U32Array *indices = &task->out_sect_indices[task_id]; + indices->v[indices->count++] = safe_cast_u32(section.section_number - 1); + } } } } internal String8List * -lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded) +lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded, U32Array **sect_indices_out) { LNK_SectionCollector task = {0}; task.objs = objs; task.name = name; task.collect_discarded = collect_discarded; task.out_lists = push_array(arena->v[0], String8List, objs_count); + if (sect_indices_out != 0) { + task.out_sect_indices = push_array(arena->v[0], U32Array, objs_count); + } tp_for_parallel(tp, arena, objs_count, lnk_collect_obj_chunks_task, &task); + if (sect_indices_out != 0) { + *sect_indices_out = task.out_sect_indices; + } return task.out_lists; } @@ -1045,42 +1336,34 @@ lnk_symbol_from_section_offset(LNK_ObjSymbolMap *map, U32 section_number, U32 of internal CV_DebugS lnk_debug_s_from_obj(Arena *arena, LNK_Obj *obj) { - Temp scratch = scratch_begin(&arena, 1); - - String8List raw_debug_s = {0}; - { - for LNK_EachCoffSection(it, obj) { - LNK_ObjSection section = it.v; - String8 section_name = lnk_obj_section_name_from_section_number(obj, section.section_number); - if (str8_match(section_name, str8_lit(".debug$S"), 0)) { - String8 debug_s = lnk_obj_section_data_from_number(obj, section.section_number); - str8_list_push(scratch.arena, &raw_debug_s, debug_s); - } - } - } - + // (single loop so each parse can tag provenance with its section index; the old + // collect-then-parse split had no side effects between the loops) CV_DebugS debug_s = {0}; - { - for (String8Node *node = raw_debug_s.first; node != 0; node = node->next) { - // parse & merge sub sections - CV_DebugS ds = cv_debug_s_from_data(arena, node->string); - cv_debug_s_concat_in_place(&debug_s, &ds); - - // make sure there is one string table - String8List string_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_StringTable); - if (string_data_list.node_count > 1) { - break; - } + for LNK_EachCoffSection(it, obj) { + LNK_ObjSection section = it.v; + String8 section_name = lnk_obj_section_name_from_section_number(obj, section.section_number); + if (!str8_match(section_name, str8_lit(".debug$S"), 0)) { continue; } + + // parse & merge sub sections + String8 raw_debug_s = lnk_obj_section_data_from_number(obj, section.section_number); + CV_DebugS ds = cv_debug_s_from_data(arena, raw_debug_s); + cv_debug_s_tag_prov_sect(&ds, safe_cast_u32(section.section_number - 1)); + cv_debug_s_concat_in_place(&debug_s, &ds); + + // make sure there is one string table + String8List string_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_StringTable); + if (string_data_list.node_count > 1) { + break; + } - // make sure there is one file checksum table - String8List checksum_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_FileChksms); - if (checksum_data_list.node_count > 1) { - continue; - } + // make sure there is one file checksum table + String8List checksum_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_FileChksms); + if (checksum_data_list.node_count > 1) { + continue; } } - scratch_end(scratch); + cv_debug_s_validate_prov(&debug_s); return debug_s; } diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index e3d50cdba..bca540d25 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -25,7 +25,8 @@ typedef struct LNK_ObjSectionArray U64 count_no_null; COFF_SectionHeader *headers; U32 *comdats; - U32Node **associated_section_numbers; + U32 *associated_section_offsets; + U32 *associated_section_numbers; } LNK_ObjSectionArray; typedef struct LNK_ObjCoff @@ -41,11 +42,23 @@ typedef struct LNK_ObjCoff B8 hotpatch; } LNK_ObjCoff; +// /OPT:ICF fold record (one per section number; slot zero is null), filled at fold-apply. +// Distinguishes ICF folds from same-name COMDAT selection and /OPT:REF removal (all three end +// up LnkRemove'd with a redirected symlink, but only ICF folds join DIFFERENT-named sections, +// which is what the debug-info aliasing below needs to know). set==0 means not ICF-folded. +typedef struct LNK_ICFFold +{ + U32 leader_obj_idx; // input_idx of the leader's obj + U32 leader_sn; // leader section number + B8 set; +} LNK_ICFFold; + typedef struct LNK_Obj { String8 path; LNK_ObjCoff coff; + struct LNK_CompressedObj *compressed_obj; // flags B8 exclude_from_debug_info; @@ -53,12 +66,27 @@ typedef struct LNK_Obj U32 input_idx; // link state - LNK_ObjSymbolRef *symlinks; // indexed by COFF section number; slot zero is null + LNK_ObjSymbolRef *symlinks; // indexed by COFF section number; slot zero is null + LNK_ICFFold *icf_fold; // /OPT:ICF fold map (per section number); 0 if ICF off + String8 icf_file_chksms; // memoized obj-wide FILECHKSMS slice (see lnk_icf_obj_file_chksms); + B32 icf_file_chksms_init; // idempotent racy fill, flag published last + B8 *icf_lines_only; // .debug$S sections associated to an ICF-folded function: stay + // LnkRemove'd, but merge into the module remapped to the leader RVA + // (section-number indexed; slot zero is null; 0 array ptr when + // ICF is off or no folds exist). + // 1 = C13 Lines only (source breakpoints bind); 2 = full record + // tree (fold joins a DIFFERENT source location and has locals -- + // watch-window labels come from the right source) // link struct LNK_LibMemberRef *link_member; struct LNK_ObjNode *self; + // Reloc-patched private copies of debug sections, indexed by COFF section + // number (slot zero is null). Keeping patches out of the copy-on-write input + // view avoids committing private pages for the mapped OBJ. + String8 *section_data_copies; + // @type_server Rng1U64 ti_range; CV_TypeIndex *ti_map; @@ -225,6 +253,7 @@ typedef struct String8 name; B32 collect_discarded; String8List *out_lists; + U32Array *out_sect_indices; // optional; per obj, 0-based sect_idx of each collected node (parallel to out_lists[obj]) } LNK_SectionCollector; // --- Error ------------------------------------------------------------------- @@ -246,6 +275,7 @@ internal struct LNK_Lib * lnk_obj_get_lib(LNK_Obj *obj); internal String8 lnk_obj_get_lib_path(LNK_Obj *obj); internal U32 lnk_obj_get_removed_section_number(LNK_Obj *obj); internal B32 lnk_obj_get_comdat_symlink_from_section_number(LNK_Obj *obj, U64 section_number, LNK_ObjSymbolRef *symlink_out); +internal U32Array lnk_obj_associated_sections_from_section_number(LNK_Obj *obj, U32 section_number); internal U32List lnk_obj_collect_associated_section_numbers(Arena *arena, LNK_Obj *obj, U32 root_section_number, COFF_SectionFlags skip_flags); // --- Symbol & Section Helpers ------------------------------------------------ @@ -259,6 +289,18 @@ internal COFF_SectionHeader * lnk_coff_section_header_from_section_number(LNK_Ob internal String8 lnk_obj_section_data_from_number(LNK_Obj *obj, U64 section_number); internal U64 lnk_obj_foff_from_section_data_ptr(LNK_Obj *obj, void *ptr); internal String8 lnk_obj_section_name_from_section_number(LNK_Obj *obj, U64 section_number); +internal void lnk_obj_drop_section_data_copies(LNK_Obj *obj); +internal String8 lnk_resolve_debug_s_node(LNK_Obj *obj, CV_DebugSProvNode *prov); +internal void lnk_obj_apply_relocs_to_buffer(LNK_Obj *obj, U64 section_number, COFF_SectionHeader *section_header, String8 section_data, U64 image_base, COFF_SectionHeader **image_section_table); + +// Streaming-ring P3.3: when set (default), .debug$S sections are NOT copied+patched at +// image-build time (lnk_obj_reloc_patcher skips them); the PDB module-write visit re-reads +// each obj's raw mapped bytes into a small per-worker window and applies relocs + the +// journaled TI/kind fixups there (lnk_obj_window_debug_s), so the GB-class patched-copy set +// never exists. Cleared under /OPT:GCTYPES, which reads AND rewrites $S type indices in +// place after an eager journal apply and therefore needs the persistent patched copies +// (the old path, kept intact). +global B32 g_debug_s_window = 1; internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); @@ -270,8 +312,9 @@ internal force_inline B32 lnk_obj_symbol_iter_next(LNK_Obj *obj, LNK_ObjSymbolIt // --- Helpers ----------------------------------------------------------------- -internal String8List * lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded); +internal String8List * lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded, U32Array **sect_indices_out); internal B32 lnk_obj_is_before(void *raw_a, void *raw_b); +internal void lnk_obj_log_compressed_census(void); // --- Directive Parser -------------------------------------------------------- diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 83ab62384..be7ae49d2 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -396,10 +396,9 @@ lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) *dst_section.flags |= COFF_SectionFlag_LnkRemove; // remove associated sections from the output - for (U32Node *associated_section = dst_ref.obj->coff.sections.associated_section_numbers[dst_parsed.section_number]; - associated_section != 0; - associated_section = associated_section->next) { - LNK_ObjSection section = lnk_obj_section_from_section_number(dst_ref.obj, associated_section->data); + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(dst_ref.obj, dst_parsed.section_number); + for EachIndex(associated_idx, associated_sections.count) { + LNK_ObjSection section = lnk_obj_section_from_section_number(dst_ref.obj, associated_sections.v[associated_idx]); *section.flags |= COFF_SectionFlag_LnkRemove; } } @@ -441,6 +440,7 @@ lnk_symbol_hash_trie_insert_or_replace(Arena *arena, LNK_SymbolHashTrie *new_trie = lnk_symbol_hash_trie_chunk_list_push(arena, chunks, 0x1000); new_trie->name = &symbol->name; new_trie->symbol = symbol; + new_trie->hash = hash; MemoryZeroArray(new_trie->child); // try to insert new node @@ -461,7 +461,8 @@ lnk_symbol_hash_trie_insert_or_replace(Arena *arena, // load current symbol String8 *curr_name = ins_atomic_ptr_eval(&curr_trie->name); - if (curr_name && str8_match(*curr_name, symbol->name, 0)) { + // fast-reject on stored hash before touching the name string (str8_match still gates) + if (curr_name && curr_trie->hash == hash && str8_match(*curr_name, symbol->name, 0)) { for (LNK_Symbol *src = symbol;;) { // try replacing current symbol with zero, otherwise loop back and retry LNK_Symbol *leader = ins_atomic_ptr_eval_assign(&curr_trie->symbol, 0); @@ -507,7 +508,7 @@ lnk_symbol_hash_trie_search(LNK_SymbolHashTrie *trie, U64 hash, String8 name) if (curr == 0) { break; } - if (curr->name && str8_match(*curr->name, name, 0)) { + if (curr->name && curr->hash == hash && str8_match(*curr->name, name, 0)) { result = curr; break; } @@ -658,7 +659,8 @@ lnk_symbol_table_searchf(LNK_SymbolTable *symtab, char *fmt, ...) internal ISectOff lnk_sc_from_symbol(LNK_Symbol *symbol) { - COFF_ParsedSymbol parsed_symbol = lnk_parsed_from_symbol(symbol); + LNK_ObjSymbolRef ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol parsed_symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref.obj, ref.symbol_idx); ISectOff sc = { .isect = parsed_symbol.section_number, .off = parsed_symbol.value }; return sc; } diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index 4fe2f24cd..1b2e5827e 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -63,6 +63,10 @@ typedef struct LNK_SymbolHashTrie { String8 *name; LNK_Symbol *symbol; + // full key hash stored at insert; descent fast-rejects on hash mismatch BEFORE + // dereferencing name -> String8 -> name bytes (saves 2-3 line misses/level). + // str8_match still gates the real match, so this is fast-reject only -> byte-identical. + U64 hash; struct LNK_SymbolHashTrie *child[4]; } LNK_SymbolHashTrie; @@ -79,6 +83,10 @@ typedef struct LNK_SymbolHashTrieChunkList U64 count; LNK_SymbolHashTrieChunk *first; LNK_SymbolHashTrieChunk *last; + // false-sharing pad: symtab->chunks / search_chunks are [worker_count] arrays indexed + // [worker_id]; at 24B/entry adjacent workers share a cache line on the parallel insert. + // Pad each entry to a full 64B line so each worker owns its line. Pure layout -> byte-identical. + U8 pad_[64 - 3*8]; } LNK_SymbolHashTrieChunkList; // --- Symbol Table ------------------------------------------------------------ diff --git a/src/linker/lnk_timer.c b/src/linker/lnk_timer.c index b94817d78..748c643bb 100644 --- a/src/linker/lnk_timer.c +++ b/src/linker/lnk_timer.c @@ -3,16 +3,83 @@ global LNK_Timer g_timers[LNK_Timer_Count]; +// summary (v2): every timer/phase boundary also stamps process-wide CPU + +// fault counters, so each bucket reports wall/user/kernel/faults +global LNK_SummaryCounters g_timer_counters_begin[LNK_Timer_Count]; +global LNK_SummaryCounters g_timer_counters_end [LNK_Timer_Count]; + +internal LNK_SummaryCounters +lnk_summary_counters_now(void) +{ + LNK_SummaryCounters c = { .wall_us = now_time_us() }; +#if OS_WINDOWS + FILETIME create_ft, exit_ft, kernel_ft, user_ft; + if (GetProcessTimes(GetCurrentProcess(), &create_ft, &exit_ft, &kernel_ft, &user_ft)) { + c.user_us = (((U64)user_ft.dwHighDateTime << 32) | user_ft.dwLowDateTime) / 10; + c.kern_us = (((U64)kernel_ft.dwHighDateTime << 32) | kernel_ft.dwLowDateTime) / 10; + } + PROCESS_MEMORY_COUNTERS pmc = { (DWORD)sizeof(pmc) }; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) { + c.faults = pmc.PageFaultCount; + } +#endif + return c; +} + +internal LNK_SummaryCounters +lnk_summary_counters_sub_sat(LNK_SummaryCounters a, LNK_SummaryCounters b) +{ + LNK_SummaryCounters c; + c.wall_us = a.wall_us > b.wall_us ? a.wall_us - b.wall_us : 0; + c.user_us = a.user_us > b.user_us ? a.user_us - b.user_us : 0; + c.kern_us = a.kern_us > b.kern_us ? a.kern_us - b.kern_us : 0; + c.faults = a.faults > b.faults ? a.faults - b.faults : 0; + return c; +} + +internal LNK_SummaryCounters +lnk_summary_counters_add(LNK_SummaryCounters a, LNK_SummaryCounters b) +{ + LNK_SummaryCounters c; + c.wall_us = a.wall_us + b.wall_us; + c.user_us = a.user_us + b.user_us; + c.kern_us = a.kern_us + b.kern_us; + c.faults = a.faults + b.faults; + return c; +} + internal void lnk_timer_begin(LNK_TimerType timer) { - g_timers[timer].begin = now_time_us(); + g_timer_counters_begin[timer] = lnk_summary_counters_now(); + g_timers[timer].begin = g_timer_counters_begin[timer].wall_us; } internal void lnk_timer_end(LNK_TimerType timer) { - g_timers[timer].end = now_time_us(); + g_timer_counters_end[timer] = lnk_summary_counters_now(); + g_timers[timer].end = g_timer_counters_end[timer].wall_us; +} + +global LNK_SummaryCounters g_summary_phase [LNK_SummaryPhase_Count]; +global LNK_SummaryCounters g_summary_phase_start[LNK_SummaryPhase_Count]; + +internal void +lnk_summary_phase_begin(LNK_SummaryPhase phase) +{ + g_summary_phase_start[phase] = lnk_summary_counters_now(); +} + +internal void +lnk_summary_phase_end(LNK_SummaryPhase phase) +{ + // atomic adds: the Write bracket runs on the background image-write thread + LNK_SummaryCounters now = lnk_summary_counters_now(); + ins_atomic_u64_add_eval(&g_summary_phase[phase].wall_us, now.wall_us - g_summary_phase_start[phase].wall_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].user_us, now.user_us - g_summary_phase_start[phase].user_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].kern_us, now.kern_us - g_summary_phase_start[phase].kern_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].faults, now.faults - g_summary_phase_start[phase].faults); } internal String8 diff --git a/src/linker/lnk_timer.h b/src/linker/lnk_timer.h index 9e5864549..5ea02c59f 100644 --- a/src/linker/lnk_timer.h +++ b/src/linker/lnk_timer.h @@ -22,3 +22,58 @@ typedef struct LNK_Timer internal void lnk_timer_begin(LNK_TimerType timer); internal void lnk_timer_end(LNK_TimerType timer); +// Per-phase counter snapshot for the end-of-link summary line (v2). Each +// boundary stamp is wall (QPC) + process-wide user/kernel CPU (GetProcessTimes) +// + process-wide soft+hard fault count (GetProcessMemoryInfo.PageFaultCount): +// 2 syscalls per boundary, negligible. Deltas are PROCESS-WIDE, so a phase that +// overlaps a concurrent thread's work (e.g. the image-write thread overlapping +// the debug-info phases) counts that work too -- attribution, not accounting. +typedef struct LNK_SummaryCounters +{ + U64 wall_us; + U64 user_us; // process user CPU, all threads + U64 kern_us; // process kernel CPU, all threads + U64 faults; // process page faults (soft+hard) +} LNK_SummaryCounters; + +internal LNK_SummaryCounters lnk_summary_counters_now(void); +internal LNK_SummaryCounters lnk_summary_counters_sub_sat(LNK_SummaryCounters a, LNK_SummaryCounters b); // per-field saturating a-b +internal LNK_SummaryCounters lnk_summary_counters_add(LNK_SummaryCounters a, LNK_SummaryCounters b); // per-field a+b + +// Phase accumulators for the end-of-link summary line. Unlike LNK_Timer +// (single begin/end shot), these ACCUMULATE across repeated brackets (e.g. +// lnk_load_inputs runs once per input round; the PDB sub-phases run again for +// the /PDBSTRIPPED build). Always measured. Image/Debug/PDB/RDI buckets come +// from g_timers (which stamp the same counters); these cover the phases that +// had no timer, plus the dbg/pdb sub-buckets. +typedef enum LNK_SummaryPhase +{ + LNK_SummaryPhase_Input, // lnk_load_inputs (parse/load objs+libs), all rounds + LNK_SummaryPhase_Resolve, // lnk_link_inputs minus contained Input time (lib search + member resolution) + LNK_SummaryPhase_Icf, // lnk_opt_icf + LNK_SummaryPhase_Ref, // lnk_opt_ref + LNK_SummaryPhase_Write, // image write thread (overlaps debug info) + + // dbg umbrella sub-buckets (printed as dbgg[...]) + LNK_SummaryPhase_DbgMcvi, // lnk_make_code_view_input + LNK_SummaryPhase_DbgMerge, // lnk_merge_types + + // pdb sub-buckets (printed as pdbg[...]); brackets sit on the pre-existing + // Prof/timer boundaries inside lnk_build_pdb + the write at its call site + LNK_SummaryPhase_PdbHsh, // lnk_replace_type_names_with_hashes (/RAD_PDB_HASH_TYPE_NAMES): parallel rewrite touching every merged TPI leaf -- storm re-fault amplifier + LNK_SummaryPhase_PdbIni, // lnk_build_pdb task init: pdb_alloc_ (MSF + type-server tables, ~132K fresh commits on the editor link) + LNK_SummaryPhase_PdbGsi, // lnk_move_global_symbols_to_gsi barrier pass ("Move Global Symbols") + LNK_SummaryPhase_PdbSym, // pdb_build_gsi_psi ("Build GSI and PSI": symrec + GSI/PSI hash streams) + LNK_SummaryPhase_PdbMod, // lnk_write_pdb_modules barrier pass ("Write Modules") + LNK_SummaryPhase_PdbTpi, // pdb_type_server_push_parallel TPI+IPI + pdb_type_server_build TPI/IPI + LNK_SummaryPhase_PdbStr, // string tables: cv_dedup_string_tables + offset assign + strtab add ("Merge String Tables"/"Add string tables") + LNK_SummaryPhase_PdbSc, // "Build Section Contrib Map" (per-obj section contribs + DBI section headers) + LNK_SummaryPhase_PdbMsf, // dbi_build + pdb_info_build + msf_build + page-node gather + LNK_SummaryPhase_PdbWr, // PDB file write (lnk_write_data_list_to_file_path in lnk_io) + + LNK_SummaryPhase_Count +} LNK_SummaryPhase; + +internal void lnk_summary_phase_begin(LNK_SummaryPhase phase); +internal void lnk_summary_phase_end(LNK_SummaryPhase phase); + diff --git a/src/linker/pdb_ext/msf_builder.c b/src/linker/pdb_ext/msf_builder.c index b19560af3..9574977e8 100644 --- a/src/linker/pdb_ext/msf_builder.c +++ b/src/linker/pdb_ext/msf_builder.c @@ -800,6 +800,7 @@ msf_stream_alloc_ex(MSF_Context *msf, MSF_UInt size) { MSF_StreamNode *node = msf_stream_alloc_(msf->arena, &msf->sectab); MSF_Stream *stream = &node->data; + msf->stream_from_sn[stream->sn] = node; msf_stream_resize_ex(msf, stream, size); return stream->sn; } @@ -854,6 +855,7 @@ msf_stream_free(MSF_Context *msf, MSF_StreamNumber sn) B32 is_free_ok = 0; MSF_StreamNode *stream_node = msf_find_stream_node(msf, sn); if (stream_node) { + msf->stream_from_sn[sn] = 0; msf_stream_list_remove(&msf->sectab, stream_node); msf_stream_resize_ex(msf, &stream_node->data, 0); stream_node->data.size = MSF_DELETED_STREAM_STAMP; @@ -1406,6 +1408,7 @@ msf_alloc__(Arena *arena, MSF_UInt page_size, MSF_PageNumber active_fpm) msf->arena = arena; msf->page_size = page_size; msf->active_fpm = active_fpm; + msf->stream_from_sn = push_array(arena, MSF_StreamNode *, MSF_STREAM_NUMBER_MAX); ProfEnd(); return msf; @@ -1438,13 +1441,10 @@ msf_alloc(MSF_UInt page_size, MSF_UInt active_fpm) internal MSF_StreamNode * msf_find_stream_node(MSF_Context *msf, MSF_StreamNumber sn) { - MSF_StreamNode *node; - for (node = msf->sectab.first; node != 0; node = node->next) { - if (node->data.sn == sn) { - break; - } + if (sn < MSF_STREAM_NUMBER_MAX) { + return msf->stream_from_sn[sn]; } - return node; + return 0; } internal MSF_Stream * diff --git a/src/linker/pdb_ext/msf_builder.h b/src/linker/pdb_ext/msf_builder.h index c07b582a8..4bbb36422 100644 --- a/src/linker/pdb_ext/msf_builder.h +++ b/src/linker/pdb_ext/msf_builder.h @@ -83,6 +83,10 @@ typedef struct MSF_Context MSF_PageList st_page_list; MSF_PageList page_pool; MSF_StreamList sectab; + // direct stream-number -> node table (sn is allocated sequentially, so the + // linear sectab walk in msf_find_stream_node is O(streams) per lookup and + // O(streams^2) across the module passes); fixed 64K-entry table = 512KiB + MSF_StreamNode **stream_from_sn; } MSF_Context; typedef enum MSF_Error diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index fbe2c9aed..f5f183140 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1133,7 +1133,8 @@ pdb_type_server_push_udt_arr(PDB_TypeServer *ts, U64 count, U32 *hash_arr, Strin bucket->raw_leaf = raw_leaf; bucket->type_index = ts->ti_lo + ts->leaf_list.node_count + leaf_idx; - U32 bucket_idx = hash % ts->bucket_cap; + Assert(ts->bucket_cap == PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT); + U32 bucket_idx = hash % PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT; SLLStackPush(ts->buckets[bucket_idx], bucket); } @@ -1204,7 +1205,8 @@ THREAD_POOL_TASK_FUNC(pdb_push_udt_leaf_task) if (~udt_info.props & CV_TypeProp_FwdRef) { // hash udt and compute bucket index U32 hash = pdb_hash_udt(udt_info, leaf.data); - U32 bucket_idx = hash % type_ht_cap; + Assert(type_ht_cap == PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT); + U32 bucket_idx = hash % PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT; // fill out & insert bucket PDB_TypeBucket *bucket = &new_buckets[bucket_cursor++]; @@ -1349,14 +1351,15 @@ pdb_load_types_from_leaf_list(PDB_TypeServer **type_server_arr, CV_LeafList leaf // get offsets for type indices in data blob CV_Leaf *leaf = &node->data; - CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf->kind, leaf->data); - - for (CV_TypeIndexInfo *ti_info = ti_info_list.first; ti_info != 0; ti_info = ti_info->next) { - Assert(ti_info->offset + sizeof(CV_TypeIndex) <= leaf->data.size); - CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(leaf->data.str + ti_info->offset); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf->kind, leaf->data); + + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); + Assert(ti_info.offset + sizeof(CV_TypeIndex) <= leaf->data.size); + CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(leaf->data.str + ti_info.offset); CV_TypeIndex external_ti = *ti_ptr; - - B32 is_complex_type = external_ti >= ti_map->min_itype[ti_info->source]; + + B32 is_complex_type = external_ti >= ti_map->min_itype[ti_info.source]; if (is_complex_type) { // search external type index CV_TypeIndex internal_tpi_idx = pdb_type_index_map_search(ti_map, CV_TypeIndexSource_TPI, external_ti); @@ -1693,7 +1696,7 @@ gsi_alloc(void) gsi->word_size = PDB_GSI_V70_WORD_SIZE; gsi->symbol_align = PDB_GSI_V70_SYMBOL_ALIGN; gsi->bucket_count = PDB_GSI_V70_BUCKET_COUNT; - gsi->bucket_arr = push_array(arena, CV_SymbolList, gsi->bucket_count); + gsi->bucket_arr = push_array(arena, PDB_GsiSymbolBucket, gsi->bucket_count); ProfEnd(); return gsi; } @@ -1724,10 +1727,6 @@ gsi_write_build_result(TP_Context *tp, msf_stream_reserve(msf, gsi_sn, gsi_size); ProfEnd(); - ProfBeginV("Reserve %M for symbols", build.symbol_data.size); - msf_stream_reserve(msf, symbols_sn, build.symbol_data.size); - ProfEnd(); - ProfBegin("Write GSI header"); msf_stream_write_struct(msf, gsi_sn, &build.header); ProfEnd(); @@ -1744,9 +1743,8 @@ gsi_write_build_result(TP_Context *tp, msf_stream_write(msf, gsi_sn, &build.compressed_bucket_arr[0], compressed_bucket_arr_size); ProfEnd(); - ProfBegin("Write symbols [%M]", build.symbol_data.size); - msf_stream_write_string_parallel(tp, msf, symbols_sn, build.symbol_data); - ProfEnd(); + // NOTE: the symbol payload itself was streamed into symbols_sn inside + // gsi_build_ex, window by window ProfEnd(); } @@ -2076,11 +2074,11 @@ psi_addr_map_from_gsi_records(TP_Context *tp, Arena *arena, PDB_GsiSortRecord *g internal THREAD_POOL_TASK_FUNC(gsi_size_buckets_task) { - U64 bucket_idx = task_id; - PDB_GsiSerializeSymbolsTask *task = raw_task; - CV_SymbolList *bucket_list = &task->bucket_arr[bucket_idx]; - for (CV_SymbolNode *node = bucket_list->first; node != 0; node = node->next) { - task->bucket_size_arr[bucket_idx] += cv_size_from_symbol(&node->data, task->symbol_align); + PDB_GsiSerializeSymbolsTask *task = raw_task; + U64 bucket_idx = task->bucket_base + task_id; + PDB_GsiSymbolBucket *bucket = &task->bucket_arr[bucket_idx]; + for EachIndex(i, bucket->count) { + task->bucket_size_arr[bucket_idx] += cv_size_from_symbol(&bucket->v[i], task->symbol_align); } } @@ -2149,14 +2147,13 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_pub32) { Temp scratch = scratch_begin(&arena, 1); - U64 bucket_idx = task_id; - PDB_GsiSerializeSymbolsTask *task = raw_task; + PDB_GsiSerializeSymbolsTask *task = raw_task; + U64 bucket_idx = task->bucket_base + task_id; - CV_SymbolList bucket = task->bucket_arr[bucket_idx]; + PDB_GsiSymbolBucket bucket = task->bucket_arr[bucket_idx]; - CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); - U64 symbol_arr_count = 0; - for EachNode(n, CV_SymbolNode, bucket.first) { symbol_arr[symbol_arr_count++] = &n->data; } + CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); + for EachIndex(i, bucket.count) { symbol_arr[i] = &bucket.v[i]; } // sort symbols within bucket radsort(symbol_arr, bucket.count, gsi_pub_symbol_is_before); @@ -2164,7 +2161,7 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_pub32) PDB_GsiSortRecord *sort_record_arr = task->sort_record_arr_arr[bucket_idx]; U64 buffer_size = task->bucket_size_arr[bucket_idx]; U64 buffer_base = task->bucket_off_arr[bucket_idx]; - U8 *buffer = task->buffer + buffer_base; + U8 *buffer = task->buffer + (buffer_base - task->window_base); U64 sort_idx = 0; U64 buffer_cursor = 0; @@ -2200,15 +2197,12 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task) { Temp scratch = scratch_begin(&arena, 1); - U64 bucket_idx = task_id; - PDB_GsiSerializeSymbolsTask *task = raw_task; - CV_SymbolList bucket = task->bucket_arr[bucket_idx]; + PDB_GsiSerializeSymbolsTask *task = raw_task; + U64 bucket_idx = task->bucket_base + task_id; + PDB_GsiSymbolBucket bucket = task->bucket_arr[bucket_idx]; - CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); - { - U64 i = 0; - for EachNode(n, CV_SymbolNode, bucket.first) { symbol_arr[i++] = &n->data; } - } + CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); + for EachIndex(i, bucket.count) { symbol_arr[i] = &bucket.v[i]; } // sort symbols within bucket radsort(symbol_arr, bucket.count, gsi_symbol_is_before); @@ -2218,7 +2212,7 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task) PDB_GsiSortRecord *sort_record_arr = task->sort_record_arr_arr[bucket_idx]; U64 buffer_size = task->bucket_size_arr[bucket_idx]; U64 buffer_base = task->bucket_off_arr[bucket_idx]; - U8 *buffer = task->buffer + buffer_base; + U8 *buffer = task->buffer + (buffer_base - task->window_base); U64 sort_idx = 0; U64 buffer_cursor = 0; @@ -2242,7 +2236,7 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task) } internal PDB_GsiBuildResult -gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size) +gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber symbols_sn, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size) { ProfBeginFunction(); Temp scratch = scratch_begin(&arena,1); @@ -2258,9 +2252,17 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_ // estimate each bucket size tp_for_parallel(tp, 0, gsi->bucket_count, gsi_size_buckets_task, &serial_task); - // prepare serial buffer + // prepare serial buffer -- a REUSED WINDOW, not a full staging copy: the + // whole symbol payload is GB-class at FN scale and used to sit committed + // alongside the payload copies it was serialized from (the link's commit + // peak). Buckets are serialized in groups that fit the window and streamed + // into the symbols stream immediately; absolute offsets are unchanged, so + // the emitted bytes are identical to the single-buffer path. U64 buffer_size = sum_array_u64(gsi->bucket_count, serial_task.bucket_size_arr); - serial_task.buffer = push_array_no_zero(arena, U8, buffer_size); + U64 max_bucket_size = 0; + for EachIndex(bucket_idx, gsi->bucket_count) { max_bucket_size = Max(max_bucket_size, serial_task.bucket_size_arr[bucket_idx]); } + U64 window_cap = Max(MB(256), max_bucket_size); + serial_task.buffer = push_array_no_zero(scratch.arena, U8, Min(window_cap, buffer_size ? buffer_size : 1)); serial_task.bucket_off_arr = push_array_copy_u64(scratch.arena, serial_task.bucket_size_arr, gsi->bucket_count); u64_array_counts_to_offsets(gsi->bucket_count, serial_task.bucket_off_arr); @@ -2272,9 +2274,26 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_ cursor += gsi->bucket_arr[bucket_idx].count; } - // fill out sort records & serialize symbols + // fill out sort records & serialize symbols, window by window + msf_stream_reserve(msf, symbols_sn, buffer_size); TP_TaskFunc *serial_func = is_pub32 ? gsi_serialize_pub32 : gsi_serialize_symbols_task; - tp_for_parallel(tp, 0, gsi->bucket_count, serial_func, &serial_task); + for (U64 window_lo = 0; window_lo < gsi->bucket_count;) { + U64 window_base = serial_task.bucket_off_arr[window_lo]; + U64 window_hi = window_lo; + while (window_hi < gsi->bucket_count && + (serial_task.bucket_off_arr[window_hi] + serial_task.bucket_size_arr[window_hi]) - window_base <= window_cap) { + window_hi += 1; + } + Assert(window_hi > window_lo); // window_cap >= max_bucket_size guarantees progress + serial_task.bucket_base = window_lo; + serial_task.window_base = window_base; + tp_for_parallel(tp, 0, window_hi - window_lo, serial_func, &serial_task); + U64 window_end = (window_hi < gsi->bucket_count) ? serial_task.bucket_off_arr[window_hi] : buffer_size; + if (window_end > window_base) { + msf_stream_write_parallel(tp, msf, symbols_sn, serial_task.buffer, window_end - window_base); + } + window_lo = window_hi; + } ProfEnd(); @@ -2288,7 +2307,7 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_ ProfBegin("Write Bitmap & Record Offsets"); for (U64 bucket_idx = 0, hash_idx = 0; bucket_idx < gsi->bucket_count; bucket_idx += 1) { // set bit for each occupied bucket - CV_SymbolList bucket_list = gsi->bucket_arr[bucket_idx]; + PDB_GsiSymbolBucket bucket_list = gsi->bucket_arr[bucket_idx]; if (bucket_list.count) { U64 word_idx = bucket_idx / gsi->word_size; Assert(word_idx < bitmap_count); @@ -2325,7 +2344,7 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_ result.compressed_bucket_count = compressed_offset_count; result.compressed_bucket_arr = compressed_offset_arr; result.total_hash_size = sizeof(header) + header.hash_record_arr_size + header.bucket_data_size; - result.symbol_data = str8(serial_task.buffer, buffer_size); + result.symbol_data = str8(0, buffer_size); // already streamed into symbols_sn window by window; size kept for logging scratch_end(scratch); ProfEnd(); @@ -2339,7 +2358,7 @@ gsi_build(TP_Context *tp, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumbe Temp scratch = scratch_begin(0,0); U64 symbol_data_base = msf_stream_get_pos(msf, symbols_sn); - PDB_GsiBuildResult build = gsi_build_ex(tp, scratch.arena, gsi, symbol_data_base, /* is_pub32: */ 0, msf->page_size); + PDB_GsiBuildResult build = gsi_build_ex(tp, scratch.arena, gsi, msf, symbols_sn, symbol_data_base, /* is_pub32: */ 0, msf->page_size); gsi_write_build_result(tp, build, msf, sn, symbols_sn); scratch_end(scratch); @@ -2354,28 +2373,41 @@ gsi_hash(PDB_GsiContext *gsi, String8 input) } internal void -gsi_push_(PDB_GsiContext *gsi, U32 hash, CV_SymbolNode *node) +gsi_reserve(PDB_GsiContext *gsi, U64 bucket_idx, U64 additional) +{ + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + if (bucket->count + additional > bucket->cap) { + U64 new_cap = bucket->count + additional; + CV_Symbol *new_v = push_array_no_zero(gsi->arena, CV_Symbol, new_cap); + MemoryCopy(new_v, bucket->v, sizeof(bucket->v[0]) * bucket->count); + bucket->v = new_v; + bucket->cap = new_cap; + } +} + +internal CV_Symbol * +gsi_push_(PDB_GsiContext *gsi, U32 hash, CV_Symbol *symbol) { - U64 bucket_idx = hash % gsi->bucket_count; - CV_SymbolList *list = &gsi->bucket_arr[bucket_idx]; - cv_symbol_list_push_node(list, node); + Assert(gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); + U64 bucket_idx = hash & (PDB_GSI_V70_BUCKET_COUNT - 1); + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + if (bucket->count == bucket->cap) { + // rare path: bulk inserters reserve up front via gsi_reserve + gsi_reserve(gsi, bucket_idx, Max(bucket->cap, 8)); + } + CV_Symbol *dst = &bucket->v[bucket->count]; + *dst = *symbol; + bucket->count += 1; gsi->symbol_count += 1; + return dst; } -internal CV_SymbolNode * +internal CV_Symbol * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol) { String8 name = cv_name_from_symbol(symbol->kind, symbol->data); U32 hash = gsi_hash(gsi, name); - - CV_SymbolNode *node = push_array_no_zero(gsi->arena, CV_SymbolNode, 1); - node->next = 0; - node->prev = 0; - node->data = *symbol; - - gsi_push_(gsi, hash, node); - - return node; + return gsi_push_(gsi, hash, symbol); } internal @@ -2408,7 +2440,7 @@ gsi_push_many_arr(TP_Context *tp, PDB_GsiContext *gsi, U64 count, CV_SymbolNode ProfEnd(); for (U64 i = 0; i < count; ++i) { - gsi_push_(gsi, task.hashes[i], symbols[i]); + gsi_push_(gsi, task.hashes[i], &symbols[i]->data); } scratch_end(scratch); @@ -2423,28 +2455,25 @@ gsi_push_many_list(PDB_GsiContext *gsi, U64 count, U32 *hash_arr, CV_SymbolList U64 hash_idx = 0; for (CV_SymbolNode *curr = list->first, *next = 0; curr != 0; curr = next, ++hash_idx) { next = curr->next; - - curr->prev = 0; - curr->next = 0; - - gsi_push_(gsi, hash_arr[hash_idx], curr); + gsi_push_(gsi, hash_arr[hash_idx], &curr->data); } MemoryZeroStruct(list); } -internal CV_SymbolNode * +internal CV_Symbol * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol) { + Assert(gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); String8 name = cv_name_from_symbol(symbol->kind, symbol->data); U32 hash = gsi_hash(gsi, name); - U64 ibucket = hash % gsi->bucket_count; + U64 ibucket = hash & (PDB_GSI_V70_BUCKET_COUNT - 1); - CV_SymbolList bucket_list = gsi->bucket_arr[ibucket]; - for (CV_SymbolNode *node = bucket_list.first; node != 0; node = node->next) { - String8 that_name = cv_name_from_symbol(node->data.kind, node->data.data); + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[ibucket]; + for EachIndex(i, bucket->count) { + String8 that_name = cv_name_from_symbol(bucket->v[i].kind, bucket->v[i].data); if (str8_match(name, that_name, 0)) { - return node; + return &bucket->v[i]; } } @@ -2472,7 +2501,7 @@ psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumbe Temp scratch = scratch_begin(0,0); U64 symbol_data_base = msf_stream_get_pos(msf, symbols_sn); - PDB_GsiBuildResult gsi_build = gsi_build_ex(tp, scratch.arena, psi->gsi, symbol_data_base, /* is_pub32: */ 1, msf->page_size); + PDB_GsiBuildResult gsi_build = gsi_build_ex(tp, scratch.arena, psi->gsi, msf, symbols_sn, symbol_data_base, /* is_pub32: */ 1, msf->page_size); ProfBegin("Address Map"); @@ -2511,12 +2540,12 @@ psi_release(PDB_PsiContext *psi) ProfEnd(); } -internal CV_SymbolNode * +internal CV_Symbol * psi_push(PDB_PsiContext *psi, CV_Pub32Flags flags, U32 offset, U16 isect, String8 name) { CV_Symbol pub = cv_make_pub32(psi->arena, flags, offset, isect, name); - CV_SymbolNode *node = gsi_push(psi->gsi, &pub); - return node; + CV_Symbol *symbol = gsi_push(psi->gsi, &pub); + return symbol; } //////////////////////////////// diff --git a/src/linker/pdb_ext/pdb_builder.h b/src/linker/pdb_ext/pdb_builder.h index 016e4aeb2..4816c22fc 100644 --- a/src/linker/pdb_ext/pdb_builder.h +++ b/src/linker/pdb_ext/pdb_builder.h @@ -176,6 +176,15 @@ typedef enum #define PDB_GSI_V70_BITMAP_COUNT ((PDB_GSI_V70_BUCKET_COUNT / PDB_GSI_V70_WORD_SIZE) + 1) #define PDB_GSI_V70_BITMAP_SIZE (PDB_GSI_V70_BITMAP_COUNT * sizeof(U32)) +// per-bucket array of CV_Symbol values (replaces linked lists of 48-byte +// CV_SymbolNode; cuts PDB-phase commit and pointer-chase on serialize) +typedef struct PDB_GsiSymbolBucket +{ + U64 count; + U64 cap; + CV_Symbol *v; +} PDB_GsiSymbolBucket; + typedef struct PDB_GsiContext { Arena *arena; @@ -183,7 +192,7 @@ typedef struct PDB_GsiContext U64 symbol_align; U64 bucket_count; U64 symbol_count; - CV_SymbolList *bucket_arr; + PDB_GsiSymbolBucket *bucket_arr; } PDB_GsiContext; typedef struct PDB_GsiSortRecord @@ -211,10 +220,12 @@ typedef struct PDB_GsiSerializeSymbolsTask { U64 symbol_data_base; U64 symbol_align; - CV_SymbolList *bucket_arr; + PDB_GsiSymbolBucket *bucket_arr; U64 *bucket_size_arr; - U64 *bucket_off_arr; - U8 *buffer; + U64 *bucket_off_arr; // absolute offsets within the full symbol payload + U8 *buffer; // window backing: holds buckets [bucket_base, bucket_base+N) + U64 bucket_base; // first bucket of the current window + U64 window_base; // absolute offset the window starts at PDB_GsiSortRecord **sort_record_arr_arr; PDB_GsiSortRecord *sort_record_arr; } PDB_GsiSerializeSymbolsTask; @@ -365,12 +376,13 @@ internal PDB_GsiContext * gsi_alloc(void); internal void gsi_build(TP_Context *tp, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber gsi_sn, MSF_StreamNumber symbols_sn); internal void gsi_release(PDB_GsiContext *gsi); internal void gsi_write_build_result(TP_Context *tp, PDB_GsiBuildResult build, MSF_Context *msf, MSF_StreamNumber sn, MSF_StreamNumber symbols_sn); -internal PDB_GsiBuildResult gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_base, B32 export_symbol_ptr_arr, U64 msf_page_size); +internal PDB_GsiBuildResult gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber symbols_sn, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size); internal U32 gsi_hash(PDB_GsiContext *gsi, String8 input); -internal CV_SymbolNode * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol); +internal void gsi_reserve(PDB_GsiContext *gsi, U64 bucket_idx, U64 additional); +internal CV_Symbol * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol); internal void gsi_push_many_arr(TP_Context *tp, PDB_GsiContext *gsi, U64 count, CV_SymbolNode **symbol_arr); internal void gsi_push_many_list(PDB_GsiContext *gsi, U64 count, U32 *hash_arr, CV_SymbolList *list); -internal CV_SymbolNode * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol); +internal CV_Symbol * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol); //////////////////////////////// // PSI @@ -378,7 +390,7 @@ internal CV_SymbolNode * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol); internal PDB_PsiContext * psi_alloc(void); internal void psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumber sn, MSF_StreamNumber symbols_sn); internal void psi_release(PDB_PsiContext *psi); -internal CV_SymbolNode * psi_push(PDB_PsiContext *psi, CV_Pub32Flags flags, U32 offset, U16 isect, String8 name); +internal CV_Symbol * psi_push(PDB_PsiContext *psi, CV_Pub32Flags flags, U32 offset, U16 isect, String8 name); // TODO: //internal CV_Symbol psi_neareset_symbol(PDB_PsiContext *psi, U16 isect, U32 off); diff --git a/src/linker/rad_obj_compress.c b/src/linker/rad_obj_compress.c new file mode 100644 index 000000000..626e21578 --- /dev/null +++ b/src/linker/rad_obj_compress.c @@ -0,0 +1,744 @@ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include +#include + +typedef uint8_t U8; +typedef uint16_t U16; +typedef uint32_t U32; +typedef uint64_t U64; +#include "lnk_compressed_obj_format.h" + +// This standalone reference writer emits one self-contained portable OBJ. It independently +// compresses every 512 KiB segment by default, preserves incompressible segments as direct-mapped +// raw runs, builds the linker sidecars, verifies every compressed round trip, and publishes via +// an atomic rename. A compiler-integrated writer can emit the same format without the raw input. + +//////////////////////////////// +// Minimal COFF declarations + +#pragma pack(push, 1) +typedef struct ObjFileHeader { + U16 machine, section_count; U32 time_stamp, symbol_table_foff, symbol_count; + U16 optional_header_size, flags; +} ObjFileHeader; +typedef struct ObjBigHeader { + U16 sig1, sig2, version, machine; U32 time_stamp; U8 magic[16], unused[16]; + U32 section_count, symbol_table_foff, symbol_count; +} ObjBigHeader; +typedef struct ObjSectionHeader { + U8 name[8]; U32 vsize, voff, fsize, foff, relocs_foff, lines_foff; + U16 reloc_count, line_count; U32 flags; +} ObjSectionHeader; +typedef struct ObjReloc { + U32 apply_off, isymbol; U16 type; +} ObjReloc; +#pragma pack(pop) + +typedef struct TypeBuild { + LNK_CObjTypeIndex disk; + LNK_CObjUdtHashIndex udt_disk; + U32 *offsets; + U16 *sizes; + U16 *kinds; + U8 *kind_codes; + U16 kind_dictionary[256]; + U16 kind_count; + int packed_kinds_ok; + U64 *udt_hashes; + U32 udt_cap; + U32 cap; +} TypeBuild; + +typedef struct BaseRelocBuild { + LNK_CObjBaseRelocIndex disk; + LNK_CObjBaseRelocEntry *entries; + U32 cap; + int supported; +} BaseRelocBuild; + +typedef struct DebugSBuild { + LNK_CObjDebugSIndex disk; + LNK_CObjDebugSEntry *entries; + LNK_CObjDebugSSummary *summaries; + U32 cap; + int supported; +} DebugSBuild; + +//////////////////////////////// +// File output + +static int +write_at(FILE *file, U64 offset, const void *data, U64 size) +{ + if (_fseeki64(file, (long long)offset, SEEK_SET) != 0) { fprintf(stderr, "seek failed at %llu\n", offset); return 0; } + size_t written = fwrite(data, 1, (size_t)size, file); + if (written != size) { fprintf(stderr, "write failed at %llu: %zu/%llu\n", offset, written, size); return 0; } + return 1; +} + +//////////////////////////////// +// Type sidecar construction + +static U64 +numeric_size(const U8 *p, const U8 *opl) +{ + if (p + 2 > opl) return 0; + U16 k; memcpy(&k, p, 2); + if (k < 0x8000) return 2; + U32 n = 0; + switch (k) { + case 0x8000: n=1; break; + case 0x8001: case 0x8002: case 0x801c: n=2; break; + case 0x8003: case 0x8004: case 0x8005: n=4; break; + case 0x8006: case 0x8009: case 0x800a: case 0x800c: n=8; break; + case 0x8007: n=10; break; + case 0x8008: n=16; break; + case 0x800b: n=6; break; + case 0x800d: n=16; break; + case 0x800e: n=20; break; + case 0x800f: n=32; break; + case 0x8017: case 0x8018: n=16; break; + default: return 0; + } + return p + 2 + n <= opl ? 2 + n : 0; +} + +static U64 +complete_udt_hash(U16 kind, const U8 *p, U64 size) +{ + const U8 *opl = p + size; + U32 props = 0; U64 cursor = 0; + if (kind == 0x1504 || kind == 0x1505 || kind == 0x1519) { + if (size < 16) return 0; memcpy(&props, p + 2, 2); cursor = 16; + } else if (kind == 0x1506) { + if (size < 8) return 0; memcpy(&props, p + 2, 2); cursor = 8; + } else if (kind == 0x1507) { + if (size < 12) return 0; memcpy(&props, p + 2, 2); cursor = 12; + } else if (kind == 0x1608 || kind == 0x1609) { + // CV_LeafStruct2 ends in a U16 but has U32 alignment, so its serialized + // fixed header includes two bytes of tail padding (sizeof == 20). Using + // the sum of the fields (18) makes the numeric parser start in padding and + // silently drops every CLASS2/STRUCT2 complete-definition hash. + if (size < 20) return 0; memcpy(&props, p, 4); cursor = 20; + } else return 0; + if (!(props & 0x200) || (props & 0x80)) return 0; + if (kind != 0x1507) { U64 n = numeric_size(p + cursor, opl); if (!n) return 0; cursor += n; } + const U8 *name_end = cursor < size ? memchr(p + cursor, 0, (size_t)(size - cursor)) : 0; + if (!name_end) return 0; + const U8 *unique = name_end + 1; + const U8 *unique_end = unique < opl ? memchr(unique, 0, (size_t)(opl - unique)) : 0; + if (!unique_end || unique_end == unique) return 0; + U64 h = 5381; + for (const U8 *c = unique; c < unique_end; ++c) h = ((h << 5) + h) ^ *c; + return h | 1; +} + +static int +is_type_section(const U8 name[8]) +{ + static const U8 t[8] = {'.','d','e','b','u','g','$','T'}; + static const U8 p[8] = {'.','d','e','b','u','g','$','P'}; + return memcmp(name, t, 8) == 0 || memcmp(name, p, 8) == 0; +} + +static int +build_type_indices(const U8 *file, U64 file_size, TypeBuild **builds_out, U32 *count_out) +{ + *builds_out = 0; *count_out = 0; + if (file_size < sizeof(ObjFileHeader)) return 1; + U64 section_off; + U32 section_count; + const ObjBigHeader *big = (const ObjBigHeader *)file; + if (file_size >= sizeof(*big) && big->sig1 == 0 && big->sig2 == 0xffff && big->version >= 2) { + section_off = sizeof(*big); section_count = big->section_count; + } else { + const ObjFileHeader *file_header = (const ObjFileHeader *)file; + section_off = sizeof(*file_header); section_count = file_header->section_count; + } + if (section_off > file_size || (U64)section_count * sizeof(ObjSectionHeader) > file_size - section_off) return 0; + TypeBuild *builds = (TypeBuild *)calloc(section_count ? section_count : 1, sizeof(*builds)); + if (!builds) return 0; + const ObjSectionHeader *sections = (const ObjSectionHeader *)(file + section_off); + U16 *kind_to_code = (U16 *)malloc(65536 * sizeof(U16)); + if (!kind_to_code) { free(builds); return 0; } + U32 build_count = 0; + for (U32 s = 0; s < section_count; ++s) { + const ObjSectionHeader *sec = §ions[s]; + if (!is_type_section(sec->name) || sec->fsize < 4) continue; + if (sec->foff > file_size || sec->fsize > file_size - sec->foff) { free(builds); return 0; } + TypeBuild *b = &builds[build_count++]; + memset(kind_to_code, 0xff, 65536 * sizeof(U16)); + b->packed_kinds_ok = 1; + b->disk.raw_section_offset = (U64)sec->foff + 4; + b->disk.raw_section_size = sec->fsize - 4; + const U8 *data = file + sec->foff + 4; + U64 data_size = sec->fsize - 4; + for (U64 cursor = 0; cursor + 4 <= data_size; ) { + U16 size, kind; + memcpy(&size, data + cursor, 2); memcpy(&kind, data + cursor + 2, 2); + U64 stride = (U64)size + 2; + if (size < 2 || stride > data_size - cursor || cursor > UINT32_MAX) break; + if (b->disk.leaf_count == b->cap) { + U32 new_cap = b->cap ? b->cap * 2 : 4096; + U32 *new_offsets = (U32 *)realloc(b->offsets, (size_t)new_cap * sizeof(U32)); + if (!new_offsets) return 0; + b->offsets = new_offsets; + U16 *new_sizes = (U16 *)realloc(b->sizes, (size_t)new_cap * sizeof(U16)); + if (!new_sizes) return 0; + b->sizes = new_sizes; + U16 *new_kinds = (U16 *)realloc(b->kinds, (size_t)new_cap * sizeof(U16)); + if (!new_kinds) return 0; + b->kinds = new_kinds; + U8 *new_kind_codes = (U8 *)realloc(b->kind_codes, (size_t)new_cap); + if (!new_kind_codes) return 0; + b->kind_codes = new_kind_codes; + b->cap = new_cap; + } + U32 i = b->disk.leaf_count++; + b->offsets[i] = (U32)cursor; b->sizes[i] = size; b->kinds[i] = kind; + U16 code = kind_to_code[kind]; + if (code == 0xffff) { + if (b->kind_count == 256) { + b->packed_kinds_ok = 0; + code = 0; + } else { + code = b->kind_count++; + kind_to_code[kind] = code; + b->kind_dictionary[code] = kind; + } + } + b->kind_codes[i] = (U8)code; + U64 udt_hash = complete_udt_hash(kind, data + cursor + 4, (U64)size - 2); + if (udt_hash) { + if (b->udt_disk.hash_count == b->udt_cap) { + U32 cap = b->udt_cap ? b->udt_cap * 2 : 1024; + U64 *hashes = realloc(b->udt_hashes, (size_t)cap * sizeof(U64)); + if (!hashes) return 0; + b->udt_hashes = hashes; b->udt_cap = cap; + } + b->udt_hashes[b->udt_disk.hash_count++] = udt_hash; + } + cursor += stride; + } + } + free(kind_to_code); + *builds_out = builds; *count_out = build_count; + return 1; +} + +//////////////////////////////// +// .debug$S sidecar construction + +static int +debug_s_is_global_symbol(U16 kind) +{ + return kind == 0x1107 || kind == 0x0102 || kind == 0x0202 || kind == 0x1008 || + kind == 0x110d || kind == 0x020e || kind == 0x100f || kind == 0x1113; +} + +static int +debug_s_is_typedef(U16 kind) +{ + return kind == 0x0004 || kind == 0x1003 || kind == 0x1108; +} + +static int +debug_s_is_scope(U16 kind) +{ + return kind == 0x1110 || kind == 0x110f || kind == 0x1103 || kind == 0x1102 || + kind == 0x114d || kind == 0x115d || kind == 0x1104 || kind == 0x1132 || + kind == 0x1147 || kind == 0x1146; +} + +static int +debug_s_is_end(U16 kind) +{ + return kind == 0x0006 || kind == 0x114f || kind == 0x114e; +} + +static void +debug_s_summarize_symbols(const U8 *data, U64 size, U64 *module_scope_depth, LNK_CObjDebugSSummary *out) +{ + U64 cand_depth = 0; + int cand_active = 1; + for (U64 cursor = 0; cursor + 4 <= size; ) { + U16 record_size, kind; + memcpy(&record_size, data + cursor, 2); + memcpy(&kind, data + cursor + 2, 2); + U64 raw_size = (U64)record_size + 2; + U64 pdb_size = (raw_size + 3) & ~(U64)3; + // OBJ .debug$S symbol records are byte-packed (CV_SymbolAlign == 1). PDB module + // streams rewrite them at PDB_SYMBOL_ALIGN == 4, so input traversal and output sizing + // deliberately use different strides. + if (record_size < 2 || raw_size > size - cursor) break; + + if (cand_active && (debug_s_is_global_symbol(kind) || (cand_depth == 0 && debug_s_is_typedef(kind)))) { + out->gsi_candidate_count += 1; + } + if (kind == 0x1110 || kind == 0x110f || kind == 0x1147 || kind == 0x1146) { + out->proc_ref_count += 1; + } + if (kind == 0x113e || kind == 0x1111 || kind == 0x110c || kind == 0x110d || + kind == 0x1112 || kind == 0x1113 || kind == 0x1153 || kind == 0x1107) { + out->flags |= LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS; + } + if (cand_active) { + if (debug_s_is_scope(kind)) cand_depth += 1; + else if (debug_s_is_end(kind)) { + if (cand_depth == 0) cand_active = 0; + else cand_depth -= 1; + } + } + + int is_module = kind != 0x0007 && !debug_s_is_global_symbol(kind) && + !(debug_s_is_typedef(kind) && *module_scope_depth == 0) && kind != 0x1176; + if (is_module) { + if (debug_s_is_scope(kind)) *module_scope_depth += 1; + else if (debug_s_is_end(kind) && *module_scope_depth) *module_scope_depth -= 1; + out->module_symbol_size += (U32)pdb_size; + } + cursor += raw_size; + } +} + +static int +build_debug_s_index(const U8 *file, U64 file_size, DebugSBuild *build) +{ + memset(build, 0, sizeof(*build)); + if (file_size < sizeof(ObjFileHeader)) return 1; + U64 section_off; + U32 section_count; + const ObjBigHeader *big = (const ObjBigHeader *)file; + if (file_size >= sizeof(*big) && big->sig1 == 0 && big->sig2 == 0xffff && big->version >= 2) { + section_off = sizeof(*big); section_count = big->section_count; + } else { + const ObjFileHeader *h = (const ObjFileHeader *)file; + section_off = sizeof(*h) + h->optional_header_size; section_count = h->section_count; + } + if (section_off > file_size || (U64)section_count * sizeof(ObjSectionHeader) > file_size - section_off) return 0; + const ObjSectionHeader *sections = (const ObjSectionHeader *)(file + section_off); + static const U8 debug_s_name[8] = {'.','d','e','b','u','g','$','S'}; + U64 module_scope_depth = 0; + for (U32 sect_idx = 0; sect_idx < section_count; ++sect_idx) { + const ObjSectionHeader *sec = §ions[sect_idx]; + if (memcmp(sec->name, debug_s_name, 8) != 0) continue; + if (sec->foff > file_size || sec->fsize > file_size - sec->foff) return 0; + if (sec->fsize < sizeof(U32)) continue; + U64 c13_base = (U64)sec->foff + sizeof(U32); // skip CV_Signature + U64 c13_size = sec->fsize - sizeof(U32); + U64 cursor = 0; + while (cursor + 2 * sizeof(U32) <= c13_size) { + U32 kind, payload_size; + memcpy(&kind, file + c13_base + cursor, sizeof(kind)); + memcpy(&payload_size, file + c13_base + cursor + sizeof(kind), sizeof(payload_size)); + U64 payload_rel = cursor + 2 * sizeof(U32); + U64 clamped_size = payload_size < c13_size - payload_rel ? payload_size : c13_size - payload_rel; + U64 payload_off = c13_base + payload_rel; + // Match cv_debug_s_from_data: ignored records do not create nodes, all unknown kinds are + // retained in the NULL bucket, and an overrun is represented by a clamped String8. + if (!(kind & 0x80000000u)) { + if (build->disk.entry_count == build->cap) { + U32 new_cap = build->cap ? build->cap * 2 : 256; + LNK_CObjDebugSEntry *entries = realloc(build->entries, (size_t)new_cap * sizeof(*entries)); + if (!entries) return 0; + build->entries = entries; + LNK_CObjDebugSSummary *summaries = realloc(build->summaries, (size_t)new_cap * sizeof(*summaries)); + if (!summaries) return 0; + build->summaries = summaries; + build->cap = new_cap; + } + if (payload_off > UINT32_MAX || clamped_size > UINT32_MAX) return 0; + U32 entry_idx = build->disk.entry_count++; + LNK_CObjDebugSEntry *dst = &build->entries[entry_idx]; + dst->raw_section_offset = sec->foff; + dst->raw_payload_offset = (U32)payload_off; + dst->raw_payload_size = (U32)clamped_size; + dst->kind = kind; + LNK_CObjDebugSSummary *summary = &build->summaries[entry_idx]; + memset(summary, 0, sizeof(*summary)); + if (kind == 0xF1) { + debug_s_summarize_symbols(file + payload_off, clamped_size, &module_scope_depth, summary); + } + } + if ((U64)payload_size > UINT64_MAX - payload_rel) break; + U64 next = payload_rel + payload_size; + if (next > UINT64_MAX - 3) break; + cursor = (next + 3) & ~(U64)3; + if (cursor <= payload_rel) break; + } + } + build->supported = 1; + return 1; +} + +//////////////////////////////// +// Base-relocation sidecar construction + +static int +build_base_reloc_index(const U8 *file, U64 file_size, BaseRelocBuild *build) +{ + memset(build, 0, sizeof(*build)); + if (file_size < sizeof(ObjFileHeader)) return 1; + U64 section_off; + U32 section_count; + U16 machine; + const ObjBigHeader *big = (const ObjBigHeader *)file; + if (file_size >= sizeof(*big) && big->sig1 == 0 && big->sig2 == 0xffff && big->version >= 2) { + section_off = sizeof(*big); section_count = big->section_count; machine = big->machine; + } else { + const ObjFileHeader *h = (const ObjFileHeader *)file; + section_off = sizeof(*h); section_count = h->section_count; machine = h->machine; + } + // RAD Link's base-reloc builder currently supports x64 only as well. + if (machine != 0x8664) return 1; + build->supported = 1; + if (section_off > file_size || (U64)section_count * sizeof(ObjSectionHeader) > file_size - section_off) return 0; + const ObjSectionHeader *sections = (const ObjSectionHeader *)(file + section_off); + for (U32 sect_idx = 0; sect_idx < section_count; ++sect_idx) { + const ObjSectionHeader *sec = §ions[sect_idx]; + U64 reloc_off = sec->relocs_foff; + U64 reloc_count = sec->reloc_count; + if ((sec->flags & 0x01000000u) && sec->reloc_count == 0xffff) { + if (reloc_off > file_size || sizeof(ObjReloc) > file_size - reloc_off) return 0; + const ObjReloc *counter = (const ObjReloc *)(file + reloc_off); + if (counter->apply_off == 0) return 0; + reloc_count = counter->apply_off - 1; + reloc_off += sizeof(ObjReloc); + } + if (reloc_off > file_size || reloc_count > (file_size - reloc_off) / sizeof(ObjReloc)) return 0; + const ObjReloc *relocs = (const ObjReloc *)(file + reloc_off); + for (U64 i = 0; i < reloc_count; ++i) { + U8 addr_size = relocs[i].type == 1 ? 8 : relocs[i].type == 2 ? 4 : 0; + if (!addr_size) continue; + if (build->disk.entry_count == build->cap) { + U32 new_cap = build->cap ? build->cap * 2 : 1024; + LNK_CObjBaseRelocEntry *entries = realloc(build->entries, (size_t)new_cap * sizeof(*entries)); + if (!entries) return 0; + build->entries = entries; + build->cap = new_cap; + } + LNK_CObjBaseRelocEntry *dst = &build->entries[build->disk.entry_count++]; + memset(dst, 0, sizeof(*dst)); + dst->sect_idx = sect_idx; + dst->apply_off = relocs[i].apply_off; + dst->isymbol = relocs[i].isymbol; + dst->addr_size = addr_size; + } + } + return 1; +} + +//////////////////////////////// +// Packed type-offset encoding + +static U64 +packed_v2_offset_storage_size(TypeBuild *b) +{ + U64 group_size = (U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_count = ((U64)b->disk.leaf_count + group_size - 1) / group_size; + U64 directory_bytes = group_count * 2 * sizeof(U32); + U64 payload_bytes = 0; + for (U64 group_idx = 0; group_idx < group_count; ++group_idx) { + U64 first = group_idx * group_size; + U64 remain = (U64)b->disk.leaf_count - first; + U64 count = group_size < remain ? group_size : remain; + U32 base = b->offsets[first]; + U32 last_delta = b->offsets[first + count - 1] - base; + payload_bytes += count * (last_delta <= 0xffffu ? 2 : 3); + } + return ((directory_bytes + 7) & ~(U64)7) + payload_bytes; +} + +//////////////////////////////// +// Entry point + +int +main(int argc, char **argv) +{ + if (argc < 3 || argc > 7) { + fprintf(stderr, "usage: rad_obj_compress [segment-kib=512] [selkie|mermaid|kraken] [space-speed=256] [superfast|veryfast|fast|normal|optimal1..5]\n"); + return 2; + } + U32 segment_size = argc >= 4 ? (U32)strtoul(argv[3], 0, 10) * 1024u : 512u * 1024u; + OodleLZ_Compressor compressor = OodleLZ_Compressor_Kraken; + if (argc >= 5) { + if (_stricmp(argv[4], "mermaid") == 0) compressor = OodleLZ_Compressor_Mermaid; + else if (_stricmp(argv[4], "selkie") == 0) compressor = OodleLZ_Compressor_Selkie; + else if (_stricmp(argv[4], "kraken") == 0) compressor = OodleLZ_Compressor_Kraken; + else { fprintf(stderr, "invalid compressor: %s\n", argv[4]); return 2; } + } + OodleLZ_CompressionLevel compression_level = OodleLZ_CompressionLevel_Normal; + if (argc >= 7) { + if (_stricmp(argv[6], "superfast") == 0) compression_level = OodleLZ_CompressionLevel_SuperFast; + else if (_stricmp(argv[6], "veryfast") == 0) compression_level = OodleLZ_CompressionLevel_VeryFast; + else if (_stricmp(argv[6], "fast") == 0) compression_level = OodleLZ_CompressionLevel_Fast; + else if (_stricmp(argv[6], "normal") == 0) compression_level = OodleLZ_CompressionLevel_Normal; + else if (_stricmp(argv[6], "optimal1") == 0) compression_level = OodleLZ_CompressionLevel_Optimal1; + else if (_stricmp(argv[6], "optimal2") == 0) compression_level = OodleLZ_CompressionLevel_Optimal2; + else if (_stricmp(argv[6], "optimal3") == 0) compression_level = OodleLZ_CompressionLevel_Optimal3; + else if (_stricmp(argv[6], "optimal4") == 0) compression_level = OodleLZ_CompressionLevel_Optimal4; + else if (_stricmp(argv[6], "optimal5") == 0) compression_level = OodleLZ_CompressionLevel_Optimal5; + else { fprintf(stderr, "invalid compression level: %s\n", argv[6]); return 2; } + } + OodleLZ_CompressOptions options = *OodleLZ_CompressOptions_GetDefault(compressor, compression_level); + options.spaceSpeedTradeoffBytes = argc >= 6 ? (OO_S32)strtol(argv[5], 0, 10) : 256; + int packed_sidecar = 1; + OodleLZ_CompressOptions_Validate(&options); + if (segment_size < 64 * 1024 || (segment_size & (segment_size - 1)) != 0) { + fprintf(stderr, "segment size must be a power of two and at least 64 KiB\n"); + return 2; + } + + HANDLE input_file = CreateFileA(argv[1], GENERIC_READ, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + LARGE_INTEGER input_size = {0}; + if (input_file == INVALID_HANDLE_VALUE || !GetFileSizeEx(input_file, &input_size) || input_size.QuadPart <= 0) { + fprintf(stderr, "unable to open input: %s\n", argv[1]); return 1; + } + U64 raw_size = (U64)input_size.QuadPart; + HANDLE input_mapping = CreateFileMappingW(input_file, 0, PAGE_READONLY, 0, 0, 0); + U8 *mapped_input = input_mapping ? (U8 *)MapViewOfFile(input_mapping, FILE_MAP_READ, 0, 0, 0) : 0; + U8 *input_data = mapped_input; + if (!input_data) { fprintf(stderr, "unable to map input: %s\n", argv[1]); return 1; } + if (raw_size >= sizeof(LNK_CObjHeader) && ((LNK_CObjHeader *)input_data)->magic == LNK_COBJ_MAGIC) { + fprintf(stderr, "input is already a compressed object: %s\n", argv[1]); + return 1; + } + TypeBuild *type_builds = 0; + U32 type_count = 0; + if (!build_type_indices(input_data, raw_size, &type_builds, &type_count) || type_count > 0xffff) { + fprintf(stderr, "unable to build type sidecar: %s\n", argv[1]); return 1; + } + if (packed_sidecar) { + for (U32 i = 0; i < type_count; ++i) { + if (!type_builds[i].packed_kinds_ok) { packed_sidecar = 0; break; } + } + } + BaseRelocBuild base_relocs = {0}; + if (!build_base_reloc_index(input_data, raw_size, &base_relocs)) { + fprintf(stderr, "unable to build base relocation sidecar: %s\n", argv[1]); return 1; + } + DebugSBuild debug_s_index = {0}; + if (!build_debug_s_index(input_data, raw_size, &debug_s_index)) { + fprintf(stderr, "unable to build .debug$S sidecar: %s\n", argv[1]); return 1; + } + U64 segment_count_u64 = (raw_size + segment_size - 1) / segment_size; + if (segment_count_u64 > UINT32_MAX) { fprintf(stderr, "input has too many segments\n"); return 1; } + U32 segment_count = (U32)segment_count_u64; + LNK_CObjSegment *directory = (LNK_CObjSegment *)calloc(segment_count, sizeof(*directory)); + U8 *verify = (U8 *)malloc(segment_size); + OO_SINTa comp_cap = OodleLZ_GetCompressedBufferSizeNeeded(compressor, segment_size); + U8 *comp = (U8 *)malloc((size_t)comp_cap); + size_t temp_name_size = strlen(argv[2]) + 5; + char *temp_name = (char *)malloc(temp_name_size); + if (!temp_name) { fprintf(stderr, "unable to allocate output path\n"); return 1; } + snprintf(temp_name, temp_name_size, "%s.tmp", argv[2]); + FILE *out = fopen(temp_name, "wb+"); + if (!directory || !verify || !comp || !out) { fprintf(stderr, "allocation/output open failed\n"); return 1; } + + LNK_CObjHeader header = {0}; + header.magic = LNK_COBJ_MAGIC; + header.version = LNK_COBJ_VERSION; + header.header_size = sizeof(header); + header.raw_size = raw_size; + header.segment_size = segment_size; + header.segment_count = segment_count; + U64 layout_cursor = (sizeof(header) + 7) & ~(U64)7; + U64 directory_bytes = (U64)segment_count * sizeof(*directory); + header.directory_offset = layout_cursor; + layout_cursor = (layout_cursor + directory_bytes + 7) & ~(U64)7; + header.flags = LNK_COBJ_FLAG_PORTABLE_RAW_MAP; + if (packed_sidecar) { + header.flags |= LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR | LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2; + } + if (type_count || base_relocs.supported || debug_s_index.supported) { + U64 dirs_bytes = 0; + header.flags |= LNK_COBJ_FLAG_TYPE_INDEX | LNK_COBJ_FLAG_UDT_HASH_INDEX | + (type_count << LNK_COBJ_TYPE_INDEX_COUNT_SHIFT); + if (type_count == 0) { + header.flags &= ~(LNK_COBJ_FLAG_TYPE_INDEX | LNK_COBJ_FLAG_UDT_HASH_INDEX); + } + dirs_bytes += (U64)type_count * (sizeof(LNK_CObjTypeIndex) + sizeof(LNK_CObjUdtHashIndex)); + if (base_relocs.supported) { + header.flags |= LNK_COBJ_FLAG_BASE_RELOC_INDEX; + dirs_bytes += sizeof(LNK_CObjBaseRelocIndex); + } + if (debug_s_index.supported) { + header.flags |= LNK_COBJ_FLAG_DEBUG_S_INDEX | LNK_COBJ_FLAG_DEBUG_S_SUMMARY; + dirs_bytes += sizeof(LNK_CObjDebugSIndex); + } + header.reserved = layout_cursor; + layout_cursor = (layout_cursor + dirs_bytes + 7) & ~(U64)7; + for (U32 i = 0; i < type_count; ++i) { + TypeBuild *b = &type_builds[i]; + U64 offset_count = b->disk.leaf_count; + U64 offsets_bytes = offset_count * sizeof(U32); + U64 sizes_bytes = (U64)b->disk.leaf_count * sizeof(U16); + U64 kinds_bytes = (U64)b->disk.leaf_count * sizeof(U16); + if (packed_sidecar) { + offsets_bytes = packed_v2_offset_storage_size(b); + sizes_bytes = 256 * sizeof(U16); + kinds_bytes = offset_count; + } + U64 hashes_bytes = (U64)b->udt_disk.hash_count * sizeof(U64); + b->disk.offsets_file_offset = layout_cursor; + layout_cursor = (layout_cursor + offsets_bytes + 7) & ~(U64)7; + b->disk.sizes_file_offset = layout_cursor; + layout_cursor = (layout_cursor + sizes_bytes + 7) & ~(U64)7; + b->disk.kinds_file_offset = layout_cursor; + layout_cursor = (layout_cursor + kinds_bytes + 7) & ~(U64)7; + b->udt_disk.hashes_file_offset = hashes_bytes ? layout_cursor : 0; + layout_cursor = (layout_cursor + hashes_bytes + 7) & ~(U64)7; + } + if (base_relocs.supported) { + U64 entry_bytes = (U64)base_relocs.disk.entry_count * sizeof(LNK_CObjBaseRelocEntry); + base_relocs.disk.entries_file_offset = entry_bytes ? layout_cursor : 0; + layout_cursor = (layout_cursor + entry_bytes + 7) & ~(U64)7; + } + if (debug_s_index.supported) { + U64 entry_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSEntry); + debug_s_index.disk.entries_file_offset = entry_bytes ? layout_cursor : 0; + layout_cursor = (layout_cursor + entry_bytes + 7) & ~(U64)7; + U64 summary_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSSummary); + layout_cursor = (layout_cursor + summary_bytes + 7) & ~(U64)7; + } + } + header.compressor = (U32)compressor; + header.data_offset = layout_cursor; + if (!write_at(out, 0, &header, sizeof(header))) return 1; + U64 sidecar_cursor = header.reserved; + for (U32 i = 0; i < type_count; ++i) { + if (!write_at(out, sidecar_cursor, &type_builds[i].disk, sizeof(LNK_CObjTypeIndex))) return 1; + sidecar_cursor += sizeof(LNK_CObjTypeIndex); + } + for (U32 i = 0; i < type_count; ++i) { + if (!write_at(out, sidecar_cursor, &type_builds[i].udt_disk, sizeof(LNK_CObjUdtHashIndex))) return 1; + sidecar_cursor += sizeof(LNK_CObjUdtHashIndex); + } + if (base_relocs.supported) { + if (!write_at(out, sidecar_cursor, &base_relocs.disk, sizeof(base_relocs.disk))) return 1; + sidecar_cursor += sizeof(base_relocs.disk); + U64 entry_bytes = (U64)base_relocs.disk.entry_count * sizeof(LNK_CObjBaseRelocEntry); + if (entry_bytes && !write_at(out, base_relocs.disk.entries_file_offset, base_relocs.entries, entry_bytes)) return 1; + } + if (debug_s_index.supported) { + if (!write_at(out, sidecar_cursor, &debug_s_index.disk, sizeof(debug_s_index.disk))) return 1; + sidecar_cursor += sizeof(debug_s_index.disk); + U64 entry_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSEntry); + if (entry_bytes && !write_at(out, debug_s_index.disk.entries_file_offset, debug_s_index.entries, entry_bytes)) return 1; + U64 summary_offset = (debug_s_index.disk.entries_file_offset + entry_bytes + 7) & ~(U64)7; + U64 summary_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSSummary); + if (summary_bytes && !write_at(out, summary_offset, debug_s_index.summaries, summary_bytes)) return 1; + } + for (U32 i = 0; i < type_count; ++i) { + TypeBuild *b = &type_builds[i]; + if (packed_sidecar) { + U64 group_size = (U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_count = ((U64)b->disk.leaf_count + group_size - 1) / group_size; + U64 group_bytes = group_count * 2 * sizeof(U32); + U64 payload_off = b->disk.offsets_file_offset + ((group_bytes + 7) & ~(U64)7); + U64 storage_bytes = packed_v2_offset_storage_size(b); + U64 payload_cap = storage_bytes - (payload_off - b->disk.offsets_file_offset); + U32 *groups = calloc(group_count ? (size_t)group_count * 2 : 1, sizeof(U32)); + U8 *deltas = malloc((size_t)(payload_cap ? payload_cap : 1)); + if (!groups || !deltas) return 1; + U64 payload_cursor = 0; + for (U64 group_idx = 0; group_idx < group_count; ++group_idx) { + U64 first = group_idx * group_size; + U64 remain = (U64)b->disk.leaf_count - first; + U64 count = group_size < remain ? group_size : remain; + U32 base = b->offsets[first]; + U32 last_delta = b->offsets[first + count - 1] - base; + U32 width = last_delta <= 0xffffu ? 2 : 3; + groups[group_idx*2 + 0] = base; + if (payload_cursor > UINT32_MAX) { fprintf(stderr, "packed v2 payload offset overflow\n"); return 1; } + groups[group_idx*2 + 1] = (U32)payload_cursor | (width == 3); + for (U64 i = 0; i < count; ++i) { + U32 delta = b->offsets[first + i] - base; + if ((width == 2 && delta > 0xffffu) || delta > 0xffffffu) { + fprintf(stderr, "packed v2 type offset overflow\n"); return 1; + } + deltas[payload_cursor++] = (U8)(delta >> 0); + deltas[payload_cursor++] = (U8)(delta >> 8); + if (width == 3) { deltas[payload_cursor++] = (U8)(delta >> 16); } + } + } + if (!write_at(out, b->disk.offsets_file_offset, groups, group_bytes) || + !write_at(out, payload_off, deltas, payload_cursor) || + !write_at(out, b->disk.sizes_file_offset, b->kind_dictionary, sizeof(b->kind_dictionary)) || + !write_at(out, b->disk.kinds_file_offset, b->kind_codes, b->disk.leaf_count)) return 1; + free(groups); + free(deltas); + continue; + } + if (!write_at(out, b->disk.offsets_file_offset, b->offsets, + (U64)b->disk.leaf_count * sizeof(U32)) || + !write_at(out, b->disk.sizes_file_offset, b->sizes, (U64)b->disk.leaf_count * sizeof(U16)) || + !write_at(out, b->disk.kinds_file_offset, b->kinds, (U64)b->disk.leaf_count * sizeof(U16))) return 1; + if (b->udt_disk.hash_count && + !write_at(out, b->udt_disk.hashes_file_offset, b->udt_hashes, + (U64)b->udt_disk.hash_count * sizeof(U64))) return 1; + } + + U64 payload_cursor = header.data_offset; + int in_raw_run = 0; + for (U32 seg_idx = 0; seg_idx < segment_count; ++seg_idx) { + U32 raw_len = (U32)((raw_size - (U64)seg_idx * segment_size) < segment_size ? + (raw_size - (U64)seg_idx * segment_size) : segment_size); + U64 seg_min = (U64)seg_idx * segment_size; + U8 *raw = input_data + seg_min; + LNK_CObjSegment *entry = &directory[seg_idx]; + entry->raw_size = raw_len; + OO_SINTa comp_len = OodleLZ_Compress(compressor, raw, raw_len, comp, + compression_level, &options, 0, 0, 0, 0); + if (comp_len <= 0 || comp_len >= raw_len) { + entry->flags |= LNK_COBJ_SEGMENT_RAW; + entry->stored_size = raw_len; + if (!in_raw_run) { payload_cursor = (payload_cursor + 65535) & ~(U64)65535; } + entry->file_offset = payload_cursor; + if (!write_at(out, entry->file_offset, raw, raw_len)) return 1; + U64 mapped_len = (seg_idx + 1 == segment_count) ? ((raw_len + 65535) & ~(U64)65535) : segment_size; + payload_cursor += mapped_len; + in_raw_run = 1; + } else { + in_raw_run = 0; + entry->stored_size = (U32)comp_len; + OO_SINTa verify_len = OodleLZ_Decompress(comp, comp_len, verify, raw_len, + OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, + 0, 0, 0, 0, 0, 0, OodleLZ_Decode_Unthreaded); + if (verify_len != raw_len || memcmp(raw, verify, raw_len) != 0) { + fprintf(stderr, "round-trip verification failed in segment %u\n", seg_idx); + return 1; + } + payload_cursor = (payload_cursor + 7) & ~(U64)7; + entry->file_offset = payload_cursor; + if (!write_at(out, entry->file_offset, comp, (U64)comp_len)) return 1; + payload_cursor += (U64)comp_len; + } + } + if (!write_at(out, header.directory_offset, directory, (U64)segment_count * sizeof(*directory))) return 1; + if (_chsize_s(_fileno(out), payload_cursor) != 0) { + fprintf(stderr, "unable to finalize portable container size\n"); return 1; + } + fclose(out); + UnmapViewOfFile(mapped_input); + CloseHandle(input_mapping); + CloseHandle(input_file); + if (!MoveFileExA(temp_name, argv[2], MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { + fprintf(stderr, "unable to publish output: %s\n", argv[2]); + DeleteFileA(temp_name); + return 1; + } + fprintf(stdout, "%s: %.2f MiB -> %.2f MiB (%.1f%%), %u segments\n", argv[1], + (double)raw_size / (1024.0 * 1024.0), (double)payload_cursor / (1024.0 * 1024.0), + raw_size ? 100.0 * (double)payload_cursor / (double)raw_size : 0.0, segment_count); + for (U32 i = 0; i < type_count; ++i) { free(type_builds[i].offsets); free(type_builds[i].sizes); free(type_builds[i].kinds); free(type_builds[i].kind_codes); free(type_builds[i].udt_hashes); } + free(debug_s_index.summaries); free(debug_s_index.entries); free(base_relocs.entries); + free(type_builds); free(temp_name); free(comp); free(verify); free(directory); + return 0; +} diff --git a/src/linker/tests/abs_vs_abs.tst b/src/linker/tests/abs_vs_abs.tst new file mode 100644 index 000000000..518c298fe --- /dev/null +++ b/src/linker/tests/abs_vs_abs.tst @@ -0,0 +1,20 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 97, storage: external } } } } } + b_obj: { file_name: "b.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 98, storage: external } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:error.exe a.obj b.obj entry.obj", expect_exit: 24 } } + steps: {} +} diff --git a/src/linker/tests/abs_vs_common.tst b/src/linker/tests/abs_vs_common.tst new file mode 100644 index 000000000..a102dea25 --- /dev/null +++ b/src/linker/tests/abs_vs_common.tst @@ -0,0 +1,25 @@ +test: +{ + artifacts: + { + common_obj: { file_name: "common.obj", coff: { object: { machine: x64, symbols: { foo: { kind: common, name: "foo", size: 321 } } } } } + abs_obj: { file_name: "abs.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 4660, storage: external } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + } + build: + { + link: { args: "/subsystem:console /entry:my_entry /out:common_after_absolute_definition_first.exe abs.obj common.obj entry.obj", expect_exit: any } + // TODO: validate that linker issues multiply defined symbol error + link: { args: "/subsystem:console /entry:my_entry /out:common_after_absolute_common_first.exe common.obj abs.obj entry.obj", when_previous_exit: 0, expect_exit: 24 } + } + steps: {} +} diff --git a/src/linker/tests/abs_vs_regular.tst b/src/linker/tests/abs_vs_regular.tst new file mode 100644 index 000000000..fec4172f3 --- /dev/null +++ b/src/linker/tests/abs_vs_regular.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + regular_obj: + { + file_name: "regular.obj" + coff: { object: { machine: x64, sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "c0ffee" } } }, symbols: { foo: { kind: external, name: "foo", section: data, value: 0 } } } } + } + abs_obj: { file_name: "abs.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 4660, storage: external } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { foo_ref: { type: Addr32Nb, offset: 3, symbol: foo } } + } } + symbols: + { + entry: { kind: external, name: "my_entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } } + } + } + build: + { + // TODO: validate that linker issues multiply defined symbol error + link: { args: "/subsystem:console /entry:my_entry /out:abs_first.exe abs.obj regular.obj entry.obj", expect_exit: nonzero } + // linker should complain about multiply defined symbol + link: { args: "/subsystem:console /entry:my_entry /out:regular_first.exe regular.obj abs.obj entry.obj", expect_exit: nonzero } + // linker should complain even in case regular is before abs + } + steps: {} +} diff --git a/src/linker/tests/abs_vs_weak.tst b/src/linker/tests/abs_vs_weak.tst new file mode 100644 index 000000000..295008813 --- /dev/null +++ b/src/linker/tests/abs_vs_weak.tst @@ -0,0 +1,44 @@ +test: +{ + artifacts: + { + abs_obj: { file_name: "abs.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 291, storage: external } } } } } + text_obj: + { + file_name: "text.obj" + coff: { object: + { + machine: x64 + sections: + { + mydata: { name: ".mydata", permissions: (read, execute), content: code, alignment: 1, data: { text: "mydata" } } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "48b80000000000000000c3" } + relocations: { foo_ref: { type: Addr64, offset: 2, symbol: foo } } + } + } + symbols: + { + mydata: { kind: external, name: "mydata", section: mydata, value: 0 } + foo: { kind: weak, name: "foo", fallback: mydata, search: no_library } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "text_first.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:my_entry /out:abs_first.exe abs.obj text.obj" } + link: { args: "/subsystem:console /entry:my_entry /out:text_first.exe text.obj abs.obj", artifact: image } + } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48b82301000000000000c3 } } } } } + } +} diff --git a/src/linker/tests/alt_name.tst b/src/linker/tests/alt_name.tst new file mode 100644 index 000000000..70a51f5bb --- /dev/null +++ b/src/linker/tests/alt_name.tst @@ -0,0 +1,116 @@ +test: +{ + artifacts: + { + override_entry_obj: + { + file_name: "override_entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + entry: { kind: external_function, name: "entry", section: text, value: 0 } + early: { kind: undefined, name: "early$fo$" } + caller: { kind: undefined, name: "late_caller" } + ordinary: { kind: undefined, name: "foo" } + } + } } + } + override_caller_lib: + { + file_name: "override_caller.lib" + coff: { library: { second_linker_member: true, members: { caller: { + path: "late_caller.obj" + object: { machine: x64, + directives: { directive: "/alternatename:from_archive$fo$=test" } + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + caller: { kind: external_function, name: "late_caller", section: text, value: 0 } + late: { kind: undefined, name: "late$fo$" } + from_archive: { kind: undefined, name: "from_archive$fo$" } + } + } + } } } } + } + override_trap_lib: + { + file_name: "override_trap.lib" + // These archive definitions must not be pulled in before the overrides. + // Pulling this member exposes an unresolved symbol and fails the link. + coff: { library: { second_linker_member: true, members: { trap: { + path: "override_trap.obj" + object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + early: { kind: external_function, name: "early$fo$", section: text, value: 0 } + late: { kind: external_function, name: "late$fo$", section: text, value: 0 } + from_archive: { kind: external_function, name: "from_archive$fo$", section: text, value: 0 } + missing: { kind: undefined, name: "override_archive_must_not_be_loaded" } + } + } + } } } } + } + test_obj: + { + file_name: "test.obj" + coff: { object: { machine: x64, sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "test" } } }, symbols: { test: { kind: external, name: "test", section: data, value: 0 } } } } + } + foo_obj: + { + file_name: "foo.obj" + coff: { object: { machine: x64, sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "foo" } } }, symbols: { foo: { kind: external, name: "foo", section: data, value: 0 } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { foo: { type: Addr32Nb, offset: 0, symbol: foo } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } } + } + } + build: + { + // Cover command-line overrides, duplicate directives, references appearing + // in a later archive load, and new override directives from that archive. + link: { args: "/subsystem:console /entry:entry /out:overrides.exe /alternatename:early$fo$=test /alternatename:early$fo$=test /alternatename:late$fo$=test /alternatename:foo=test override_entry.obj test.obj override_caller.lib override_trap.lib" } + // basic alternate name test + link: { args: "/subsystem:console /entry:entry /out:alternate_basic.exe /alternatename:foo=test test.obj entry.obj" } + // linker should not chase alt name links + link: { args: "/subsystem:console /entry:entry /out:alternate_chain.exe /alternatename:foo=bar /alternatename:bar=test test.obj entry.obj", expect_exit: nonzero } + // alt name conflict + link: { args: "/subsystem:console /entry:entry /out:alternate_conflict.exe /alternatename:foo=test /alternatename:foo=qwe test.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_missing_equal.exe /alternatename:foo foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_wrong_separator.exe /alternatename:foo-oof foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_extra_equal.exe /alternatename:foo=test=bar foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_empty_target.exe /alternatename:foo= foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_empty_source.exe /alternatename:= foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_empty_option.exe /alternatename: foo.obj entry.obj", expect_exit: nonzero } + // TODO: check that RAD Linker prints these warnings + // warn about alt name to self alt name? + link: { args: "/subsystem:console /entry:entry /out:alternate_self.exe /alternatename:foo=foo foo.obj entry.obj" } + // warn about alt name to unknown symbol? + link: { args: "/subsystem:console /entry:entry /out:alternate_unknown.exe /alternatename:qwe=ewq foo.obj entry.obj" } + } + steps: {} +} diff --git a/src/linker/tests/base_relocs.tst b/src/linker/tests/base_relocs.tst new file mode 100644 index 000000000..fe2db7471 --- /dev/null +++ b/src/linker/tests/base_relocs.tst @@ -0,0 +1,81 @@ +test: +{ + artifacts: + { + // main.obj + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: + { + concat: + { + hex: "48b80000000000000000" // mov rax, func_name + hex: "ffd0" // call rax + hex: "4831c0" // xor rax, rax + hex: "b800000000" // mov eax, func_name + hex: "ffd0" // call rax + hex: "c3" // ret + } + } + relocations: + { + foo_64: { type: Addr64, offset: 2, symbol: foo } + foo_32: { type: Addr32, offset: 16, symbol: foo } + } + } + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 4 } + relocations: { absolute: { type: Addr32, offset: 0, symbol: abs } } + } + } + symbols: + { + foo: { kind: undefined, name: "foo" } + abs: { kind: absolute, name: "abs", value: 305419896, storage: static } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + + // func.obj + func_obj: + { + file_name: "func.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { foo: { kind: external, name: "foo", section: text, value: 0 } } + } } + } + } + + build: + { + // linker must not produce base relocations for absolute symbol + link: { args: "/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe main.obj func.obj", output: none } + + // it is illegal to merge .reloc with other sections + link: { args: "/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.reloc=.rdata main.obj func.obj", output: none, expect_exit: 26 } + + // the other way around is illegal too + link: { args: "/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.rdata=.reloc main.obj func.obj", output: none, expect_exit: 26 } + } + steps: {} +} diff --git a/src/linker/tests/comdat_any.tst b/src/linker/tests/comdat_any.tst new file mode 100644 index 000000000..add120426 --- /dev/null +++ b/src/linker/tests/comdat_any.tst @@ -0,0 +1,77 @@ +test: +{ + artifacts: + { + one_obj: + { + file_name: "1.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "1" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: Any } + test: { kind: external_function, name: "TEST", section: test, value: 0 } + } + } } + } + two_obj: + { + file_name: "2.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "2" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: Any } + test: { kind: external, name: "TEST", section: test, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } + } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + one_image: { file_name: "1.exe", pe: {} } + two_image: { file_name: "2.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:1.exe 1.obj 2.obj entry.obj", artifact: one_image } + link: { args: "/subsystem:console /entry:entry /out:2.exe 2.obj 1.obj entry.obj", artifact: two_image } + } + steps: + { + expect_pe: { artifact: one_image, expected: { pe: { sections: { ".test": { data: 31 } } } } } + expect_pe: { artifact: two_image, expected: { pe: { sections: { ".test": { data: 32 } } } } } + } +} diff --git a/src/linker/tests/comdat_associative.tst b/src/linker/tests/comdat_associative.tst new file mode 100644 index 000000000..f3e936f6b --- /dev/null +++ b/src/linker/tests/comdat_associative.tst @@ -0,0 +1,79 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: "a", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } + aa: { name: "aa", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "aa" } } + } + symbols: + { + a_def: { kind: section_definition, section: a, selection: Largest } + test: { kind: external, name: "TEST", section: a, value: 0 } + aa_def: { kind: section_definition, section: aa, selection: Associative, associate: a } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + bb: { name: "bb", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "bb" } } + b: { name: "b", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "b" } } + bbb: { name: "bbb", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "bbb" } } + } + symbols: + { + bb_def: { kind: section_definition, section: bb, selection: Largest } + b_def: { kind: section_definition, section: b, selection: Associative, associate: bb } + bbb_def: { kind: section_definition, section: bbb, selection: Associative, associate: bb } + test: { kind: external, name: "TEST", section: bb, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: { pe: { sections: { @absent "a", @absent "aa", "b": { data: 62 }, "bb": { data: 6262 }, "bbb": { data: 626262 } } } } + } + } +} diff --git a/src/linker/tests/comdat_associative_loop.tst b/src/linker/tests/comdat_associative_loop.tst new file mode 100644 index 000000000..56b72b756 --- /dev/null +++ b/src/linker/tests/comdat_associative_loop.tst @@ -0,0 +1,53 @@ +test: +{ + artifacts: + { + loop_obj: + { + file_name: "loop.obj" + coff: { object: + { + machine: x64 + sections: + { + aaaa: { name: ".aaaa", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "aaaa" } } + aa: { name: ".aa", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "aa" } } + a: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } + aaa: { name: ".aaa", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "aaa" } } + } + symbols: + { + aaa_def: { kind: section_definition, section: aaa, selection: Associative, associate: aa } + aaaa_def: { kind: section_definition, section: aaaa, selection: Associative, associate: aaa } + a_def: { kind: section_definition, section: a, selection: Associative, associate: aa } + aa_def: { kind: section_definition, section: aa, selection: Associative, associate: a } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe loop.obj entry.obj", expect_exit: 29 } } + steps: {} +} diff --git a/src/linker/tests/comdat_associative_non_comdat.tst b/src/linker/tests/comdat_associative_non_comdat.tst new file mode 100644 index 000000000..f28612f9e --- /dev/null +++ b/src/linker/tests/comdat_associative_non_comdat.tst @@ -0,0 +1,53 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "a" } } + b: { name: ".b", permissions: (read, write), content: initialized_data, data: { text: "b" } } + } + symbols: + { + test: { kind: external, name: "TEST", section: a, value: 0 } + b_def: { kind: section_definition, section: b, selection: Associative, associate: a } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj test.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".a": { data: 61 }, ".b": { data: 62 } } } } } + } +} diff --git a/src/linker/tests/comdat_associative_out_of_bounds.tst b/src/linker/tests/comdat_associative_out_of_bounds.tst new file mode 100644 index 000000000..b475081fc --- /dev/null +++ b/src/linker/tests/comdat_associative_out_of_bounds.tst @@ -0,0 +1,51 @@ +test: +{ + artifacts: + { + bad_obj: + { + file_name: "bad.obj" + bytes: + { + data: { concat: + { + hex: "6486020000000000670000000500000000000000" + hex: "2e6100000000000000000000000000000100000064000000000000000000000000000000401000c0" + hex: "2e6161000000000000000000000000000200000065000000000000000000000000000000401000c0" + text: "aaa" + hex: "2e6100000000000000000000010000000301" + hex: "010000000000000000000000000002000000" + hex: "544553540000000000000000010000000200" + hex: "2e6161000000000000000000020000000301" + hex: "020000000000000000000000410105000000" + hex: "04000000" + } } + } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj", expect_exit: 7 } } + steps: {} +} diff --git a/src/linker/tests/comdat_exact_match.tst b/src/linker/tests/comdat_exact_match.tst new file mode 100644 index 000000000..f105b1984 --- /dev/null +++ b/src/linker/tests/comdat_exact_match.tst @@ -0,0 +1,78 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a_def: { kind: section_definition, section: a, selection: ExactMatch } + test: { kind: external, name: "TEST", section: a, value: 0 } + } + } } + } + a2_obj: + { + file_name: "a2.obj" + coff: { object: + { + machine: x64 + sections: { a2: { name: ".a2", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a2_def: { kind: section_definition, section: a2, selection: ExactMatch } + test: { kind: external, name: "TEST", section: a2, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "b" } } } + symbols: + { + b_def: { kind: section_definition, section: b, selection: ExactMatch } + test: { kind: external, name: "TEST", section: b, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj", expect_exit: nonzero } + link: { args: "/subsystem:console /entry:entry /out:b.exe entry.obj a2.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".a2": { data: 61 } } } } } } +} diff --git a/src/linker/tests/comdat_external_symbol_at_nonzero_offset.tst b/src/linker/tests/comdat_external_symbol_at_nonzero_offset.tst new file mode 100644 index 000000000..0f4ea3a6b --- /dev/null +++ b/src/linker/tests/comdat_external_symbol_at_nonzero_offset.tst @@ -0,0 +1,68 @@ +// MSVC vftables use COMDAT sections whose public symbol can start past the +// section definition symbol; references to a replaced copy must target the winner. +test: +{ + artifacts: + { + leader_obj: + { + file_name: "leader.obj" + coff: { object: + { + machine: x64 + sections: { data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 8 } + } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 8 } + relocations: { foo_ref: { type: Addr64, offset: 0, symbol: foo } } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + foo] + hex: "c3" + } } + relocations: { foo_ref: { type: Rel32, offset: 3, symbol: foo } } + } + } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 8 } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 8 } + } +} diff --git a/src/linker/tests/comdat_external_symbol_at_zero_offset.tst b/src/linker/tests/comdat_external_symbol_at_zero_offset.tst new file mode 100644 index 000000000..4aba77468 --- /dev/null +++ b/src/linker/tests/comdat_external_symbol_at_zero_offset.tst @@ -0,0 +1,66 @@ +test: +{ + artifacts: + { + leader_obj: + { + file_name: "leader.obj" + coff: { object: + { + machine: x64 + sections: { data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 8 } } } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 0 } + } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 8 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 8 } + relocations: { foo_ref: { type: Addr64, offset: 0, symbol: foo } } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + foo] + hex: "c3" + } } + relocations: { foo_ref: { type: Rel32, offset: 3, symbol: foo } } + } + } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 0 } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 0 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 0 } + } +} diff --git a/src/linker/tests/comdat_external_symbol_uses_leader_offset.tst b/src/linker/tests/comdat_external_symbol_uses_leader_offset.tst new file mode 100644 index 000000000..0e745e68b --- /dev/null +++ b/src/linker/tests/comdat_external_symbol_uses_leader_offset.tst @@ -0,0 +1,73 @@ +// Duplicate COMDAT sections can have identical bytes while their symbol tables +// disagree about where a same-named public symbol points inside the section. +// This mirrors MSVC vftable COMDATs: the selected copy may have leading RTTI data +// at offset 0 and the vftable symbol at offset 8, while a discarded copy's +// vftable symbol is at offset 0. Relocations against the discarded symbol must +// use the selected symbol's value, not just the selected section contribution +// plus the discarded symbol's original offset. +test: +{ + artifacts: + { + leader_obj: + { + file_name: "leader.obj" + coff: { object: + { + machine: x64 + sections: { vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Any } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 8 } + } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 8 } + relocations: { vftable_ref: { type: Addr64, offset: 0, symbol: vftable } } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + ??_7X] + hex: "c3" + } } + relocations: { vftable_ref: { type: Rel32, offset: 3, symbol: vftable } } + } + } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Any } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 0 } + entry: { kind: external_function, name: "entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 8 } + } +} diff --git a/src/linker/tests/comdat_largest.tst b/src/linker/tests/comdat_largest.tst new file mode 100644 index 000000000..d6fca51d8 --- /dev/null +++ b/src/linker/tests/comdat_largest.tst @@ -0,0 +1,83 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a_def: { kind: section_definition, section: a, selection: Largest } + test: { kind: external, name: "TEST", section: a, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "bb" } } } + symbols: + { + b_def: { kind: section_definition, section: b, selection: Largest } + test: { kind: external, name: "TEST", section: b, value: 0 } + } + } } + } + c_obj: + { + file_name: "c.obj" + coff: { object: + { + machine: x64 + sections: { c: { name: ".c", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "c" } } } + symbols: + { + c_def: { kind: section_definition, section: c, selection: Largest } + test: { kind: external, name: "TEST", section: c, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /out:a.exe /entry:entry entry.obj a.obj b.obj", artifact: a_image } + link: { args: "/subsystem:console /out:b.exe /entry:entry entry.obj c.obj a.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: a_image, expected: { pe: { sections: { @absent ".a", ".b": { data: 6262 } } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".c": { data: 63 } } } } } + } +} diff --git a/src/linker/tests/comdat_largest_external_symbol_uses_selected_offset.tst b/src/linker/tests/comdat_largest_external_symbol_uses_selected_offset.tst new file mode 100644 index 000000000..c5494f2a2 --- /dev/null +++ b/src/linker/tests/comdat_largest_external_symbol_uses_selected_offset.tst @@ -0,0 +1,92 @@ +// Chromium has duplicate vftable COMDATs where the referencing copy is an +// IMAGE_COMDAT_SELECT_ANY section with the public vftable at offset 0, while +// the selected IMAGE_COMDAT_SELECT_LARGEST copy has the same public symbol at +// offset 8. Relocations in the discarded object must resolve to the selected +// public symbol, not to the discarded section. +test: +{ + artifacts: + { + discarded_obj: + { + file_name: "discarded.obj" + coff: { object: + { + machine: x64 + sections: + { + vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 16 } + relocations: + { + vftable_ref: { type: Addr64, offset: 0, symbol: vftable } + force_selected_ref: { type: Addr64, offset: 8, symbol: force_selected } + } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + ??_7X] + hex: "c3" + } } + relocations: { vftable_ref: { type: Rel32, offset: 3, symbol: vftable } } + } + } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Any } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 0 } + force_selected: { kind: undefined, name: "force_selected" } + entry: { kind: external_function, name: "entry", section: text, value: 0 } + } + } } + } + selected_lib: + { + file_name: "selected.lib" + coff: { library: + { + second_linker_member: true + members: + { + selected: + { + path: "selected.obj" + object: + { + machine: x64 + sections: + { + vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 24 } } + force_text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Largest } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 8 } + force_selected: { kind: external_function, name: "force_selected", section: force_text, value: 0 } + } + } + } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,noicf discarded.obj selected.lib", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 8 } + } +} diff --git a/src/linker/tests/comdat_no_duplicates.tst b/src/linker/tests/comdat_no_duplicates.tst new file mode 100644 index 000000000..6032e4b85 --- /dev/null +++ b/src/linker/tests/comdat_no_duplicates.tst @@ -0,0 +1,72 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } + } + relocations: { a_ref: { type: Addr32Nb, offset: 0, symbol: a } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + a: { kind: undefined, name: "a" } + } + } } + } + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: NoDuplicates } + a: { kind: external, name: "a", section: test, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: NoDuplicates } + a: { kind: external, name: "a", section: test, value: 0 } + } + } } + } + image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj", expect_exit: 24 } + link: { args: "/subsystem:console /entry:entry /out:b.exe a.obj entry.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".test": { data: 61 } } } } } } +} diff --git a/src/linker/tests/comdat_same_size.tst b/src/linker/tests/comdat_same_size.tst new file mode 100644 index 000000000..bf4d5244a --- /dev/null +++ b/src/linker/tests/comdat_same_size.tst @@ -0,0 +1,78 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a_def: { kind: section_definition, section: a, selection: SameSize } + test: { kind: external, name: "TEST", section: a, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "b" } } } + symbols: + { + b_def: { kind: section_definition, section: b, selection: SameSize } + test: { kind: external, name: "TEST", section: b, value: 0 } + } + } } + } + c_obj: + { + file_name: "c.obj" + coff: { object: + { + machine: x64 + sections: { c: { name: ".c", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "cc" } } } + symbols: + { + c_def: { kind: section_definition, section: c, selection: SameSize } + test: { kind: external, name: "TEST", section: c, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj", artifact: image } + link: { args: "/subsystem:console /entry:entry /out:b.exe a.obj b.obj c.obj entry.obj", expect_exit: 24 } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".a": { data: 61 } } } } } } +} diff --git a/src/linker/tests/comdat_with_offset.tst b/src/linker/tests/comdat_with_offset.tst new file mode 100644 index 000000000..271af2494 --- /dev/null +++ b/src/linker/tests/comdat_with_offset.tst @@ -0,0 +1,59 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { rdata: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "1Hello, World!", zero: 1 } } } } + symbols: + { + rdata_def: { kind: section_definition, section: rdata, selection: Largest } + test: { kind: external, name: "TEST", section: rdata, value: 1 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { rdata: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "Hello, World!", zero: 1 } } } } + symbols: + { + rdata_def: { kind: section_definition, section: rdata, selection: Largest } + test: { kind: external, name: "TEST", section: rdata, value: 1 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj" } } + steps: {} +} diff --git a/src/linker/tests/common_block.tst b/src/linker/tests/common_block.tst new file mode 100644 index 000000000..1d4bc93b6 --- /dev/null +++ b/src/linker/tests/common_block.tst @@ -0,0 +1,119 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 6 } + relocations: { a_ref: { type: Addr32, offset: 0, symbol: a } } + } + // shift common block's initial position + bss: + { + name: ".bss" + permissions: (read, write) + content: uninitialized_data + data: { zero: 1 } + } + } + symbols: { a: { kind: common, name: "A", size: 3 } } + } + } + } + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 9 } + relocations: { b_ref: { type: Addr64, offset: 0, symbol: b } } + } + } + symbols: { b: { kind: common, name: "B", size: 6 } } + } + } + } + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } + } + } + image: { file_name: "common_block.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:my_entry /out:common_block.exe /fixed /largeaddressaware:no /merge:.bss=.comm a.obj b.obj entry.obj" + artifact: image + } + } + + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + sections: + { + // blocks must be sorted in descending order to reduce alignment padding + ".comm": { virtual_size: 19 } + ".data": {} + } + } + } + } + // ensure linker correctly patched addresses for symbols pointing into common block + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u32, target_section: ".comm", target_offset: 16, target_address: va } + expect_pe_word: { artifact: image, section: ".data", offset: 6, type: u64, target_section: ".comm", target_offset: 8, target_address: va } + } +} diff --git a/src/linker/tests/communal_var_vs_regular.tst b/src/linker/tests/communal_var_vs_regular.tst new file mode 100644 index 000000000..359cd05ab --- /dev/null +++ b/src/linker/tests/communal_var_vs_regular.tst @@ -0,0 +1,42 @@ +test: +{ + artifacts: + { + communal_obj: { file_name: "communal.obj", coff: { object: { machine: x64, symbols: { test: { kind: common, name: "TEST", size: 1 } } } } } + defn_obj: + { + file_name: "defn.obj" + coff: { object: { machine: x64, + sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "test" } } } + symbols: { test: { kind: external, name: "TEST", section: data, value: 0 } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + // linker should replace communal TEST with .data TEST + link: { args: "/subsystem:console /entry:entry /out:a.exe communal.obj defn.obj entry.obj", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:b.exe defn.obj communal.obj entry.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + } +} diff --git a/src/linker/tests/communal_var_vs_regular_comdat.tst b/src/linker/tests/communal_var_vs_regular_comdat.tst new file mode 100644 index 000000000..db26283db --- /dev/null +++ b/src/linker/tests/communal_var_vs_regular_comdat.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + communal_obj: { file_name: "communal.obj", coff: { object: { machine: x64, symbols: { test: { kind: common, name: "TEST", size: 1 } } } } } + large_obj: + { + file_name: "large.obj" + coff: { object: { machine: x64, + sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "test" } } } + symbols: { + data_def: { kind: section_definition, section: data, selection: Largest } + test: { kind: external, name: "TEST", section: data, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + // linker should replace communal TEST with .data TEST + link: { args: "/subsystem:console /entry:entry /out:a.exe communal.obj large.obj entry.obj", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:b.exe large.obj communal.obj entry.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + } +} diff --git a/src/linker/tests/cyclic_type.tst b/src/linker/tests/cyclic_type.tst new file mode 100644 index 000000000..cd5eddafe --- /dev/null +++ b/src/linker/tests/cyclic_type.tst @@ -0,0 +1,56 @@ +test: +{ + artifacts: + { + cycle_obj: + { + file_name: "cycle.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { hex: "040000000a00021001100000000000000a0002100010000000000000" } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { hex: "04000000f10000002b000000290010110000000000000000000000000000000000000000000000000110000000000000000000666f6f0000" } + } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full /rad_ignore:-43 cycle.obj entry.obj" + output: none + stderr_matches: "*Error(*): *: LF_POINTER(type_index: *) forward refs member type index * (leaf struct offset: *)*" + } + } + steps: {} +} diff --git a/src/linker/tests/debug_p_and_debug_t_in_obj.tst b/src/linker/tests/debug_p_and_debug_t_in_obj.tst new file mode 100644 index 000000000..c8976d173 --- /dev/null +++ b/src/linker/tests/debug_p_and_debug_t_in_obj.tst @@ -0,0 +1,81 @@ +test: +{ + artifacts: + { + pch_obj: + { + file_name: "pch.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + // signature + hex: "040000000a0002100300000000000000" + // PCH ender + hex: "06001400bebafeca" + } } + } + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000160009150010000001000000bebafeca" + text: "pch.obj" + hex: "000a00021003000000000000000e000810001000000000000000000000" + } } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000f1000000100000000e000111bebafeca" + text: "pch.obj" + hex: "00" + } } + } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full pch.obj entry.obj" + output: none + stderr_matches: "*Warning(067): *pch.obj: multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections*" + } + } + steps: {} +} diff --git a/src/linker/tests/debug_p_sig_mismatch.tst b/src/linker/tests/debug_p_sig_mismatch.tst new file mode 100644 index 000000000..7152a7998 --- /dev/null +++ b/src/linker/tests/debug_p_sig_mismatch.tst @@ -0,0 +1,126 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + // signature + hex: "04000000" + // duplicate in a.obj + hex: "0a0002100300000000000000" + // unique procedure type + hex: "0e000810001000000200000000000000" + // PCH ender + hex: "06001400bebafeca" + } } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000f1000000300000000c000111bebafeca" + text: "a.obj" + hex: "0020003c1100000000d00000000000000000000e002400197f0000544f525455524500" + } } + } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000140009150010000002000000efbeadde" + text: "a.obj" + hex: "000a00021003000000000000000e000810001000000000000000000000" + } } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000f1000000300000000c000111efbeadde" + text: "a.obj" + hex: "0020003c1100000000d00000000000000000000e002400197f0000544f525455524500" + } } + } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj" + output: none + stderr_matches: "*Error(048): *b.obj: PCH signature mismatch, expected 0xdeadbeef got 0xcafebabe; PCH obj *a.obj*" + } + } + steps: {} +} diff --git a/src/linker/tests/def_file_full.tst b/src/linker/tests/def_file_full.tst new file mode 100644 index 000000000..6aba9d648 --- /dev/null +++ b/src/linker/tests/def_file_full.tst @@ -0,0 +1,236 @@ +test: +{ + + artifacts: + { + def_full_obj: + { + file_name: "def_full.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".rdata" + permissions: (read, write) + content: initialized_data + data: { text: "test" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: external, name: "foo", section: data, value: 0 } + } + } + } + } + + full_def: + { + file_name: "full.def" + text: { data: { concat: + { + text: "; leading comment" + hex: "0a" + text: "NAME " + hex: "22" + text: "def full.exe" + hex: "22" + text: " BASE=0x140020000" + hex: "0a" + text: "VERSION 7.8" + hex: "0a" + text: "HEAPSIZE 0x30000, 0x4000" + hex: "0a" + text: "STACKSIZE 0x50000,0x6000" + hex: "0a" + text: "SECTIONS .rdata READ" + hex: "0a" + text: "EXPORTS foo @ 2 DATA" + hex: "0a" + } } } + } + bad_base_space_def: + { + file_name: "bad_base_space.def" + text: { data: { concat: + { + text: "NAME bad_base_space BASE = 0x140020000" + hex: "0a" + text: "EXPORTS foo @2 DATA" + hex: "0a" + } } } + } + bad_base_colon_def: + { + file_name: "bad_base_colon.def" + text: { data: { concat: + { + text: "NAME bad_base_colon BASE:0x140020000" + hex: "0a" + text: "EXPORTS foo @2 DATA" + hex: "0a" + } } } + } + bad_section_align_def: + { + file_name: "bad_section_align.def" + text: { data: { concat: + { + text: "NAME bad_section_align" + hex: "0a" + text: "SECTIONS .rdata READ ALIGN=8192" + hex: "0a" + text: "EXPORTS foo @2 DATA" + hex: "0a" + } } } + } + exe_image: { file_name: "def full.exe", pe: {} } + + def_full_dll_obj: + { + file_name: "def_full_dll.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { text: "test" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "c3" } + } + } + symbols: + { + dll_entry: { kind: external, name: "_DllMainCRTStartup", section: text, value: 0 } + dll_foo: { kind: external, name: "dll_foo", section: data, value: 0 } + } + } + } + } + full_dll_def: + { + file_name: "full_dll.def" + text: { data: { concat: + { + text: "LIBRARY folded BASE=0x180020000" + hex: "0a" + text: "EXPORTS" + hex: "0a" + text: " dll_foo DATA" + hex: "0a" + } } } + } + dll_image: { file_name: "folded.dll", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /def:full.def def_full.obj" + artifact: exe_image + } + + link: + { + args: "/subsystem:console /entry:entry /def:bad_base_space.def /out:bad_base_space.exe def_full.obj" + expect_exit: nonzero + } + link: + { + args: "/subsystem:console /entry:entry /def:bad_base_colon.def /out:bad_base_colon.exe def_full.obj" + expect_exit: nonzero + } + link: + { + args: "/subsystem:console /entry:entry /def:bad_section_align.def /out:bad_section_align.exe def_full.obj" + expect_exit: nonzero + } + + link: + { + args: "/dll /subsystem:console /def:full_dll.def def_full_dll.obj" + artifact: dll_image + } + } + + steps: + { + expect_pe: + { + artifact: exe_image + expected: + { + pe: + { + optional: + { + image_base: 5368840192 + major_image_version: 7 + minor_image_version: 8 + sizeof_heap_reserve: 196608 + sizeof_heap_commit: 16384 + sizeof_stack_reserve: 327680 + sizeof_stack_commit: 24576 + } + sections: + { + ".rdata": { raw_flags: 1073741888 } + } + exports: + { + count: 1 + entries: + { + export_0: { name: "foo", ordinal: 2 } + } + } + } + } + } + expect_pe: + { + artifact: dll_image + expected: + { + pe: + { + optional: { image_base: 6442582016 } + exports: + { + count: 1 + entries: + { + export_0: { name: "dll_foo" } + } + } + } + } + } + } +} diff --git a/src/linker/tests/defer_duplicate_imp_link.tst b/src/linker/tests/defer_duplicate_imp_link.tst new file mode 100644 index 000000000..93f5c40ad --- /dev/null +++ b/src/linker/tests/defer_duplicate_imp_link.tst @@ -0,0 +1,123 @@ +test: +{ + + artifacts: + { + bar_lib: + { + file_name: "defer_duplicate_bar.lib" + coff: { library: + { + members: + { + bar_scaffold: { dll_import: { name: "bar.dll", machine: x64, timestamp: 0 } } + bar_import: { import: { dll: "bar.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + qwe_member: + { + path: "member_2.obj" + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "ff2500000000" } } + } + symbols: + { + imp_bar: { kind: undefined, name: "__imp_bar" } + qwe: { kind: external_function, name: "qwe", section: text, value: 0 } + } + } + } + } + } } + } + foo_lib: + { + file_name: "defer_duplicate_foo.lib" + coff: { library: + { + members: + { + foo_scaffold: { dll_import: { name: "foo.dll", machine: x64, timestamp: 0 } } + bar_import: { import: { dll: "foo.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + thunk_member: + { + path: "member_2.obj" + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "ff2500000000" } + relocations: { bar_ref: { type: Rel32, offset: 2, symbol: bar } } + } + } + symbols: + { + bar: { kind: undefined, name: "bar" } + qwe: { kind: undefined, name: "qwe" } + thunk: { kind: external_function, name: "thunk", section: text, value: 0 } + } + } + } + } + } } + } + entry_obj: + { + file_name: "defer_duplicate_entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + image: { file_name: "defer_duplicate_imp_link.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:defer_duplicate_imp_link.exe defer_duplicate_bar.lib defer_duplicate_foo.lib defer_duplicate_entry.obj /include:thunk" + artifact: image + } + } + + steps: + { + // Exactly one import descriptor remains, and it names foo.dll. + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u32, target_section: ".data", target_offset: 56, target_address: rva } + expect_pe_word: { artifact: image, section: ".data", offset: 28, type: u32, target_section: ".data", target_offset: 78, target_address: rva } + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u32, target_section: ".data", target_offset: 0, target_address: rva } + expect_pe_bytes: { artifact: image, section: ".data", offset: 36, hex: "0000000000000000000000000000000000000000" } + + // The sole name import is bar with hint zero. Its nonzero IAT and ILT words + // are equal and point inside .data at the same hint/name record. + expect_pe_word: + { + artifact: image + section: ".data" + offset: 0 + type: u64 + nonzero: true + target_section: ".data" + target_offset: 72 + target_address: rva + other_section: ".data" + other_offset: 56 + other_type: u64 + relation: equal + } + expect_pe_bytes: { artifact: image, section: ".data", offset: 8, hex: "0000000000000000" } + expect_pe_bytes: { artifact: image, section: ".data", offset: 64, hex: "0000000000000000" } + expect_pe_bytes: { artifact: image, section: ".data", offset: 72, hex: "000062617200666f6f2e646c6c00" } + } +} diff --git a/src/linker/tests/defer_imp_link.tst b/src/linker/tests/defer_imp_link.tst new file mode 100644 index 000000000..6efa25ae6 --- /dev/null +++ b/src/linker/tests/defer_imp_link.tst @@ -0,0 +1,55 @@ +test: +{ + artifacts: + { + bar_lib: + { + file_name: "bar.lib" + coff: { library: { members: { + import_descriptor: { dll_import: { name: "bar.dll", machine: x64, timestamp: 0 } } + import_symbol: { import: { dll: "bar.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + implementation: + { + path: "member_2.obj" + object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "ff2500000000" } } } + symbols: { + imp_bar: { kind: undefined, name: "__imp_bar" } + qwe: { kind: external_function, name: "qwe", section: text, value: 0 } + } + } + } + } } } + } + foo_lib: + { + file_name: "foo.lib" + coff: { library: { members: { + import_descriptor: { dll_import: { name: "foo.dll", machine: x64, timestamp: 0 } } + import_symbol: { import: { dll: "foo.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + thunk: + { + path: "member_2.obj" + object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "ff2500000000" } + relocations: { bar_ref: { type: Rel32, offset: 2, symbol: bar } } + } }, symbols: { + bar: { kind: undefined, name: "bar" } + qwe: { kind: undefined, name: "qwe" } + thunk: { kind: external_function, name: "thunk", section: text, value: 0 } + } } + } + } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe bar.lib foo.lib entry.obj /include:thunk" } } + steps: {} +} diff --git a/src/linker/tests/defer_impl_link_to_second_search_pass.tst b/src/linker/tests/defer_impl_link_to_second_search_pass.tst new file mode 100644 index 000000000..2735f83d2 --- /dev/null +++ b/src/linker/tests/defer_impl_link_to_second_search_pass.tst @@ -0,0 +1,48 @@ +test: +{ + artifacts: + { + imp_ref_obj: + { + file_name: "imp_ref.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + entry: { kind: external_function, name: "entry", section: text, value: 0 } + imp_foo: { kind: undefined, name: "__imp_foo" } + func: { kind: undefined, name: "func" } + } + } } + } + impl_ref_lib: + { + file_name: "impl_ref.lib" + coff: { library: { second_linker_member: true, members: { implementation: { + path: "impl_ref.obj" + object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + func: { kind: external_function, name: "func", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } + } } } } + } + foo_lib: + { + file_name: "foo.lib" + coff: { library: { second_linker_member: true, members: { imported: { import: { + dll: "foo.dll", name: "foo", machine: x64, timestamp: 4294967295, type: code, lookup: name, hint: 0 + } } } } } + } + foo2_lib: + { + file_name: "foo2.lib" + coff: { library: { second_linker_member: true, members: { imported: { import: { + dll: "foo.dll", name: "foo", machine: x64, timestamp: 4294967295, type: code, lookup: name, hint: 0 + } } } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe foo.lib foo2.lib imp_ref.obj impl_ref.lib" } } + steps: {} +} diff --git a/src/linker/tests/delay_import.tst b/src/linker/tests/delay_import.tst new file mode 100644 index 000000000..4d6ee00db --- /dev/null +++ b/src/linker/tests/delay_import.tst @@ -0,0 +1,231 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + return_0: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 0 + // ret + data: { hex: "48c7c000000000c3" } + } + return_1: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 1 + // ret + data: { hex: "48c7c001000000c3" } + } + return_2: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 2 + // ret + data: { hex: "48c7c002000000c3" } + } + } + symbols: + { + return_1: { kind: external_function, name: "return_1", section: return_1, value: 0 } + return_2: { kind: external_function, name: "return_2", section: return_2, value: 0 } + } + } } + } + + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + return_0: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 0 + // ret + data: { hex: "48c7c000000000c3" } + } + return_123: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 123 + // ret + data: { hex: "48c7c07b000000c3" } + } + return_321: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 321 + // ret + data: { hex: "48c7c041010000c3" } + } + } + symbols: + { + return_123: { kind: external_function, name: "return_123", section: return_123, value: 0 } + return_321: { kind: external_function, name: "return_321", section: return_321, value: 0 } + } + } } + } + + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // push rsi + // push rdi + // sub rsp,28h + // call return_1 + // mov esi,eax + // call return_2 + // mov edi,eax + // add edi,esi + // call return_123 + // mov esi,eax + // call return_321 + // add eax,esi + // add eax,edi + // add rsp,28h + // pop rdi + // pop rsi + // ret + data: { hex: "56574883ec28e80000000089c6e80000000089c701f7e80000000089c6e80000000001f001f84883c4285f5ec3" } + relocations: + { + return_1_ref: { type: Rel32, offset: 7, symbol: return_1 } + return_2_ref: { type: Rel32, offset: 14, symbol: return_2 } + return_123_ref: { type: Rel32, offset: 23, symbol: return_123 } + return_321_ref: { type: Rel32, offset: 30, symbol: return_321 } + } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + return_1: { kind: undefined, name: "return_1" } + return_2: { kind: undefined, name: "return_2" } + return_123: { kind: undefined, name: "return_123" } + return_321: { kind: undefined, name: "return_321" } + } + } } + } + + a_dll: { file_name: "a.dll", pe: {} } + b_dll: { file_name: "b.dll", pe: {} } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/dll /implib:a.lib /export:return_1 /export:return_2 /rad_time_stamp:0x69EB0E28 a.obj libcmt.lib" + artifact: a_dll + } + + link: + { + args: "/dll /implib:b.lib /export:return_123 /export:return_321 /rad_time_stamp:0x69EB0E28 b.obj libcmt.lib" + artifact: b_dll + } + + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /fixed /debug:full /rad_time_stamp:0x69EB0E28 main.obj a.lib b.lib kernel32.lib delayimp.lib libcmt.lib /delayload:a.dll /delayload:b.dll" + artifact: image + } + } + + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + delay_imports: + { + count: 2 + dll_0: + { + name: "a.dll" + attributes: 1 + @range(1, 18446744073709551615) module_handle + @range(1, 18446744073709551615) import_name_table + @range(1, 18446744073709551615) bound_table + @range(1, 18446744073709551615) unload_table + timestamp: 0 + bound_count: 2 + unload_count: 2 + count: 2 + entries: + { + import_0: { type: name, name: "return_1", hint: 0 } + import_1: { type: name, name: "return_2", hint: 1 } + } + } + dll_1: + { + name: "b.dll" + attributes: 1 + @range(1, 18446744073709551615) module_handle + @range(1, 18446744073709551615) import_name_table + @range(1, 18446744073709551615) bound_table + @range(1, 18446744073709551615) unload_table + timestamp: 0 + bound_count: 2 + unload_count: 2 + count: 2 + entries: + { + import_0: { type: name, name: "return_123", hint: 0 } + import_1: { type: name, name: "return_321", hint: 1 } + } + } + } + } + } + } + } +} diff --git a/src/linker/tests/delay_import_user32.tst b/src/linker/tests/delay_import_user32.tst new file mode 100644 index 000000000..0a33d60fa --- /dev/null +++ b/src/linker/tests/delay_import_user32.tst @@ -0,0 +1,68 @@ +test: +{ + + artifacts: + { + delay_import_obj: + { + file_name: "delay_import.obj" + coff: + { + object: + { + machine: x64 + sections: + { + str: + { + name: ".str" + permissions: (read, write) + content: initialized_data + data: { hex: "7465737400666f6f00" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // sub rsp,28h + // xor r9d,r9d + // lea r8,[msg] + // lea rdx,[caption] + // xor ecx,ecx + // call qword ptr [__imp_MessageBoxA] + // xor eax,eax + // add rsp,28h + // ret + data: { hex: "4883ec284533c94c8d0500000000488d150000000033c9ff150000000033c04883c428c3" } + relocations: + { + msg_ref: { type: Rel32, offset: 10, symbol: msg } + caption_ref: { type: Rel32, offset: 17, symbol: caption } + message_box_ref: { type: Rel32, offset: 25, symbol: message_box } + } + } + } + symbols: + { + msg: { kind: external, name: "msg", section: str, value: 0 } + caption: { kind: external, name: "caption", section: str, value: 5 } + entry: { kind: external, name: "entry", section: text, value: 0 } + message_box: { kind: undefined, name: "__imp_MessageBoxA" } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /out:a.exe /entry:entry /fixed /delayload:user32.dll kernel32.lib user32.lib libcmt.lib delayimp.lib delay_import.obj /debug:full" + } + } + + steps: {} +} diff --git a/src/linker/tests/determ_test.tst b/src/linker/tests/determ_test.tst new file mode 100644 index 000000000..343cd30fc --- /dev/null +++ b/src/linker/tests/determ_test.tst @@ -0,0 +1,59 @@ +test: +{ + build: + { + windows: + { + outputs: + { + object: "test.obj" + baseline_exe: "main.exe" + baseline_pdb: "main.pdb" + } + + // compile the test target (torture) + compile: + { + output: "%outputs.object%" + args: "/fsanitize=address /Z7 -DBUILD_GIT_HASH=Stringify(0000000000000000000000000000000000000000) -I%source_dir% /Zc:preprocessor %source_dir%/torture/torture_main.c" + } + + // single-threaded link + link: + { + output: "%outputs.baseline_exe%" + args: "%outputs.object% /debug:full /rad_time_stamp:0 /rad_workers:1 /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74" + produces: { "%outputs.baseline_pdb%" } + } + + // read b + + // multi-threaded links + link: + { + output: "%run%.exe" + args: "%outputs.object% /debug:full /rad_time_stamp:0 /rad_imagealtpath:main.exe /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74" + produces: { "%run%.pdb" } + repeat: 25 + index: run + parallel: true + } + + // wait for linkers + } + } + + steps: + { + repeat: + { + count: 25 + index: run + steps: + { + compare_file: { left: "%build.outputs.baseline_exe%", right: "%run%.exe" } + compare_file: { left: "%build.outputs.baseline_pdb%", right: "%run%.pdb" } + } + } + } +} diff --git a/src/linker/tests/dotted_exports.tst b/src/linker/tests/dotted_exports.tst new file mode 100644 index 000000000..70e690199 --- /dev/null +++ b/src/linker/tests/dotted_exports.tst @@ -0,0 +1,31 @@ +test: +{ + artifacts: + { + exports_obj: + { + file_name: "exports.obj" + coff: { object: { machine: x64, + sections: { text: { + name: ".text", permissions: (read, execute), content: code, + alignment: 1, data: { hex: "c3" } + } } + symbols: { + dotted: { kind: external_function, name: "__RTFM.autortfm_is_context_status", section: text, value: 0 } + } + } } + } + image: { file_name: "exports.dll", pe: {} } + } + build: { link: { + args: "/dll /entry:__RTFM.autortfm_is_context_status /nodefaultlib /out:exports.dll /export:__RTFM.autortfm_is_context_status /export:forwarded=other_dll.symbol exports.obj" + artifact: image + } } + steps: { expect_pe: { artifact: image, expected: { pe: { exports: { + count: 2 + entries: { + export_0: { name: "__RTFM.autortfm_is_context_status", forwarder: "" } + export_1: { name: "forwarded", forwarder: "other_dll.symbol" } + } + } } } } } +} diff --git a/src/linker/tests/empty_section.tst b/src/linker/tests/empty_section.tst new file mode 100644 index 000000000..07decdbc8 --- /dev/null +++ b/src/linker/tests/empty_section.tst @@ -0,0 +1,41 @@ +test: +{ + artifacts: + { + empty_obj: + { + file_name: "empty_section.obj" + coff: { object: + { + machine: x64 + // TEST is defined in a zero-sized code section and cannot satisfy a live relocation. + sections: { test: { name: ".test", permissions: (read, execute), content: code, data: { zero: 0 } } } + symbols: { test: { kind: external, name: "TEST", section: test, value: 0 } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "48c7c000000000c3" } + relocations: { test: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + test: { kind: undefined, name: "TEST" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + // A relocation cannot resolve to a definition in an empty section. + build: { link: { args: "/subsystem:console /entry:entry /out:empty_section.exe empty_section.obj entry.obj", expect_exit: nonzero } } + steps: {} +} diff --git a/src/linker/tests/fail_if_mismatch.tst b/src/linker/tests/fail_if_mismatch.tst new file mode 100644 index 000000000..e3463777c --- /dev/null +++ b/src/linker/tests/fail_if_mismatch.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + a1_obj: { file_name: "a1.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=1" } } } } + a2_obj: { file_name: "a2.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=2" } } } } + a1_copy_obj: { file_name: "a1_copy.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=1" } } } } + conf_dirs_obj: { file_name: "conf_dirs.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=1 /FAILIFMISMATCH:a=2" } } } } + } + + build: + { + // ------------------------------------------------------------ + // try linking two objs with mismatching directives + link: { args: "entry.obj a1.obj a2.obj /entry:entry /subsystem:console /out:a2.exe", output: none, expect_exit: 32 } + + // ------------------------------------------------------------ + // happy case + link: { args: "entry.obj a1.obj a1_copy.obj /entry:entry /subsystem:console /out:a1.exe", output: none } + + // ------------------------------------------------------------ + // test conflicting directives in obj + link: { args: "entry.obj conf_dirs.obj /entry:entry /subsystem:console /out:conf_dirs.exe", output: none, expect_exit: 32 } + + // ------------------------------------------------------------ + // passing switch on command line + link: { args: "entry.obj a1.obj /FAILIFMISMATCH:a=2 /out:cmddir.exe", output: none, expect_exit: 32 } + } + steps: {} +} diff --git a/src/linker/tests/find_merged_pdata.tst b/src/linker/tests/find_merged_pdata.tst new file mode 100644 index 000000000..f15b339ac --- /dev/null +++ b/src/linker/tests/find_merged_pdata.tst @@ -0,0 +1,54 @@ +test: +{ + artifacts: + { + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + xdata: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, data: { hex: "191b0300090140000270000000000000f00100000104010004420000" } } + pdata: + { + name: ".pdata" + permissions: (read) + content: initialized_data + alignment: 4 + data: { zero: 12 } + relocations: + { + unwind: { type: Addr32Nb, offset: 8, symbol: unwind_foobar } + first: { type: Addr32Nb, offset: 0, symbol: foobar } + one_past_last: { type: Addr32Nb, offset: 4, symbol: foobar } + } + } + foobar: + { + name: ".foobar" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "40574881ec00020000488b05000000004833c448898424f0010000488d0424488bf833c0b9ec010000f3aab804000000486bc0028b0404488b8c24f00100004833cce8000000004881c4000200005fc3cccccccccccccccccccccccccccccccc4883ec28e8000000004883c428c3" } + } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + foobar: { kind: static, name: "foobar", section: foobar, value: 0 } + xdata_definition: { kind: section_definition, section: xdata, selection: Null } + unwind_foobar: { kind: static, name: "$unwind$foobar", section: xdata, value: 0 } + pdata_definition: { kind: section_definition, section: pdata, selection: Null } + pdata_foobar: { kind: static, name: "$pdata$foobar", section: pdata, value: 0 } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe main.obj /merge:.pdata=.rdata", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { data_directories: { exceptions: { file_size: 12 } } } } } } +} diff --git a/src/linker/tests/first_member_header.tst b/src/linker/tests/first_member_header.tst new file mode 100644 index 000000000..d4d887f31 --- /dev/null +++ b/src/linker/tests/first_member_header.tst @@ -0,0 +1,82 @@ +test: +{ + + artifacts: + { + test_lib: + { + file_name: "test.lib" + coff: + { + library: + { + second_linker_member: false + members: + { + symbols: + { + path: "obj.obj" + object: + { + machine: x64 + symbols: + { + eight: { kind: absolute, name: "8", value: 8, storage: external } + one: { kind: absolute, name: "1", value: 1, storage: external } + nine: { kind: absolute, name: "9", value: 9, storage: external } + seven: { kind: absolute, name: "7", value: 7, storage: external } + four: { kind: absolute, name: "4", value: 4, storage: external } + five: { kind: absolute, name: "5", value: 5, storage: external } + two: { kind: absolute, name: "2", value: 2, storage: external } + three: { kind: absolute, name: "3", value: 3, storage: external } + six: { kind: absolute, name: "6", value: 6, storage: external } + } + } + } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/RAD_MEMORY_MAP_FILES:READ_WRITE /subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9" + } + } + + steps: + { + expect_file: { path: "test.lib", equals_artifact: test_lib } + expect_file: { path: "entry.obj", equals_artifact: entry_obj } + } +} diff --git a/src/linker/tests/flag_conf.tst b/src/linker/tests/flag_conf.tst new file mode 100644 index 000000000..3a400df42 --- /dev/null +++ b/src/linker/tests/flag_conf.tst @@ -0,0 +1,51 @@ +test: +{ + artifacts: + { + conf_obj: + { + file_name: "conf.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".mysect", permissions: (read, execute), content: initialized_data, data: { text: "one" } } + b: { name: ".mysect", permissions: (read, write), content: initialized_data, data: { text: "two" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe conf.obj entry.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + section_count: 3 + @count(3) sections_by_index: + { + section_1: { name: ".text" } + section_2: { name: ".mysect", raw_flags: 1610612800 } + section_3: { name: ".mysect", raw_flags: 3221225536 } + } + } + } + } + } +} diff --git a/src/linker/tests/function_pad_min.tst b/src/linker/tests/function_pad_min.tst new file mode 100644 index 000000000..e9ec4fd7c --- /dev/null +++ b/src/linker/tests/function_pad_min.tst @@ -0,0 +1,30 @@ +test: +{ + artifacts: + { + funcs_obj: + { + file_name: "funcs.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".text", permissions: (read, execute), content: code, alignment: 4, data: { hex: "c3" } } + b: { name: ".text", permissions: (read, execute), content: code, alignment: 4, data: { hex: "c3" } } + c: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + a: { kind: external_function, name: "A", section: a, value: 0 } + b: { kind: external_function, name: "B", section: b, value: 0 } + c: { kind: external_function, name: "C", section: c, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:A /functionpadmin:1 /out:a.exe funcs.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: ccccccccc3ccccccc3ccc3 } } } } } } +} diff --git a/src/linker/tests/gen_rrt.tst b/src/linker/tests/gen_rrt.tst new file mode 100644 index 000000000..8f3020a4b --- /dev/null +++ b/src/linker/tests/gen_rrt.tst @@ -0,0 +1,202 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + // signature + hex: "04000000" + // duplicate in a.obj + hex: "0a0002100300000000000000" + // unique procedure type + hex: "0e000810001000000200000000000000" + // PCH ender + hex: "06001400bebafeca" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + align4: + { + concat: + { + // C13 signature and symbols subsection kind. + hex: "04000000f1000000" + size32le: + { + concat: + { + // S_OBJNAME, signature 0xcafebabe, and the terminated a.obj work path. + size16le: + { + concat: + { + hex: "0111bebafeca" + work_path: "a.obj" + zero: 1 + } + } + + // S_COMPILE3: C/x64, front-end 0.0.0.0, compiler 14.36.32537.0, TORTURE. + size16le: { hex: "3c1100000000d00000000000000000000e002400197f0000544f525455524500" } + } + } + } + } + } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + // signature + hex: "04000000" + + // PCH starter + hex: "280009150010000002000000bebafeca" + text: "corrupt-pch-file-path.obj" + zero: 1 + + // duplicate pointer type + hex: "0a0002100300000000000000" + + // unique procedure type + hex: "0e000810001000000000000000000000" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + align4: + { + concat: + { + // C13 signature and symbols subsection kind. + hex: "04000000f1000000" + size32le: + { + concat: + { + // Preserve the original use of a.obj in b.obj's S_OBJNAME record. + size16le: + { + concat: + { + hex: "0111bebafeca" + work_path: "a.obj" + zero: 1 + } + } + + // S_COMPILE3: C/x64, front-end 0.0.0.0, compiler 14.36.32537.0, TORTURE. + size16le: { hex: "3c1100000000d00000000000000000000e002400197f0000544f525455524500" } + } + } + } + } + } + } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full /rad_type_server:foo.rrt a.obj b.obj entry.obj" + } + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash a.obj foo.rrt b.obj entry.obj" + } + } + + steps: {} +} diff --git a/src/linker/tests/get_msf_stream_pages.tst b/src/linker/tests/get_msf_stream_pages.tst new file mode 100644 index 000000000..3ea2bb73c --- /dev/null +++ b/src/linker/tests/get_msf_stream_pages.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { get_msf_stream_pages: {} } +} diff --git a/src/linker/tests/ghash_basic.tst b/src/linker/tests/ghash_basic.tst new file mode 100644 index 000000000..af9017136 --- /dev/null +++ b/src/linker/tests/ghash_basic.tst @@ -0,0 +1,114 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } + } + debug_h: + { + name: ".debug$H" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "c5c9330100000200010000000000000002000000000000000300000000000000" } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } + } + debug_h: + { + name: ".debug$H" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "c5c9330100000200040000000000000005000000000000000600000000000000" } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:ghash.exe /debug:ghash entry.obj a.obj b.obj" + } + link: + { + args: "/subsystem:console /entry:entry /out:full.exe /debug:full entry.obj a.obj b.obj" + } + } + + steps: + { + expect_pdb: + { + path: "ghash.pdb" + expected: { pdb: { tpi: { leaf_count: 6 } } } + } + expect_pdb: + { + path: "full.pdb" + expected: { pdb: { tpi: { leaf_count: 3 } } } + } + } +} diff --git a/src/linker/tests/ghash_check_corrupt.tst b/src/linker/tests/ghash_check_corrupt.tst new file mode 100644 index 000000000..3d3ee80a9 --- /dev/null +++ b/src/linker/tests/ghash_check_corrupt.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { zero: 0 } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: .debug$H section is too small to contain the header*" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_check_hash_alg.tst b/src/linker/tests/ghash_check_hash_alg.tst new file mode 100644 index 000000000..0c07419f7 --- /dev/null +++ b/src/linker/tests/ghash_check_hash_alg.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "c5c9330100000100a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: mismatched .debug$H hash algorithm: got SHA1_8, expected *" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_check_magic.tst b/src/linker/tests/ghash_check_magic.tst new file mode 100644 index 000000000..527fa3492 --- /dev/null +++ b/src/linker/tests/ghash_check_magic.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "7b00000000000200a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: .debug$H contains invalid magic: got 0x7b, expected 0x133c9c5*" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_check_version.tst b/src/linker/tests/ghash_check_version.tst new file mode 100644 index 000000000..78f97f470 --- /dev/null +++ b/src/linker/tests/ghash_check_version.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "c5c93301efbe0200a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: mismatched .debug$H version: got 48879, expected 0*" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_match_debug_t.tst b/src/linker/tests/ghash_match_debug_t.tst new file mode 100644 index 000000000..c09659daa --- /dev/null +++ b/src/linker/tests/ghash_match_debug_t.tst @@ -0,0 +1,48 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + flags: (discardable) + //str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); + data: { hex: "0400000012000515000080000000000000000000000000000a0006150000800000000000" } + } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "c5c9330100000200a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: mismatched .debug$H hash count and type count: got 3 hashes for 2 types*" + } + } + steps: {} +} diff --git a/src/linker/tests/guard_cf_pulls_load_config.tst b/src/linker/tests/guard_cf_pulls_load_config.tst new file mode 100644 index 000000000..a2572f258 --- /dev/null +++ b/src/linker/tests/guard_cf_pulls_load_config.tst @@ -0,0 +1,42 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + loadcfg_lib: + { + file_name: "loadcfg.lib" + coff: { library: + { + second_linker_member: true + members: { loadcfg_obj: + { + path: "loadcfg.obj" + object: + { + machine: x64 + sections: { loadcfg: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, data: { concat: { hex: "40000000", zero: 60 } } } } + symbols: { load_config_used: { kind: external, name: "_load_config_used", section: loadcfg, value: 0 } } + } + } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe /guard:cf entry.obj loadcfg.lib", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: + { + data_directories: { load_config: { file_size: 64 } } + optional: { @bits_clear(16384) dll_characteristics } + } } } } +} diff --git a/src/linker/tests/guardsym_directive.tst b/src/linker/tests/guardsym_directive.tst new file mode 100644 index 000000000..d748e8e18 --- /dev/null +++ b/src/linker/tests/guardsym_directive.tst @@ -0,0 +1,25 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + guardsym_obj: + { + file_name: "guardsym.obj" + // MSVC link accepts GUARDSYM without treating the named symbol as /INCLUDE. + coff: { object: { machine: x64, directives: { directive: "/GUARDSYM:missing,S" } } } + } + } + + build: { link: { args: "entry.obj guardsym.obj /entry:entry /subsystem:console /out:guardsym.exe", output: none } } + steps: {} +} diff --git a/src/linker/tests/icf_associative_child_prevents_fold.tst b/src/linker/tests/icf_associative_child_prevents_fold.tst new file mode 100644 index 000000000..fe0e2d04f --- /dev/null +++ b/src/linker/tests/icf_associative_child_prevents_fold.tst @@ -0,0 +1,53 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_associative_child.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + fn_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + fn_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + handler_a: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01020304" } } + handler_b: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "04030201" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + a: { type: Addr64, offset: 0, symbol: fn_a_symbol } + b: { type: Addr64, offset: 8, symbol: fn_b_symbol } + } + } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + handler_a_definition: { kind: section_definition, section: handler_a, selection: Associative, associate: fn_a } + handler_b_definition: { kind: section_definition, section: handler_b, selection: Associative, associate: fn_b } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_associative_child.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, other_section: ".data", other_offset: 0, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_comdat_reloc_targets_fold.tst b/src/linker/tests/icf_comdat_reloc_targets_fold.tst new file mode 100644 index 000000000..74b7a3187 --- /dev/null +++ b/src/linker/tests/icf_comdat_reloc_targets_fold.tst @@ -0,0 +1,108 @@ +test: +{ + artifacts: + { + a_object: + { + file_name: "icf_comdat_reloc_a.obj" + coff: { object: + { + machine: x64 + sections: + { + fn_a: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, shared + // ret + data: { hex: "48b80000000000000000c3" } + relocations: { shared: { type: Addr64, offset: 2, symbol: shared_local_a } } + } + shared: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "00" } } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + shared_definition: { kind: section_definition, section: shared, selection: Any } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + shared_symbol: { kind: external, name: "shared", section: shared, value: 0 } + shared_local_a: { kind: static, name: "shared_local_a", section: shared, value: 0 } + } + } } + } + b_object: + { + file_name: "icf_comdat_reloc_b.obj" + coff: { object: + { + machine: x64 + sections: + { + fn_b: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, shared + // ret + data: { hex: "48b80000000000000000c3" } + relocations: { shared: { type: Addr64, offset: 2, symbol: shared_local_b } } + } + shared: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "00" } } + } + symbols: + { + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + shared_definition: { kind: section_definition, section: shared, selection: Any } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + shared_symbol: { kind: external, name: "shared", section: shared, value: 0 } + shared_local_b: { kind: static, name: "shared_local_b", section: shared, value: 0 } + } + } } + } + entry_object: + { + file_name: "icf_comdat_reloc_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + a: { type: Addr64, offset: 0, symbol: fn_a } + b: { type: Addr64, offset: 8, symbol: fn_b } + } + } + } + symbols: + { + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a: { kind: undefined_function, name: "fn_a" } + fn_b: { kind: undefined_function, name: "fn_b" } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_comdat_reloc_entry.obj icf_comdat_reloc_a.obj icf_comdat_reloc_b.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: equal } + } +} diff --git a/src/linker/tests/icf_comdat_symlink_chain.tst b/src/linker/tests/icf_comdat_symlink_chain.tst new file mode 100644 index 000000000..3fe2ef9e7 --- /dev/null +++ b/src/linker/tests/icf_comdat_symlink_chain.tst @@ -0,0 +1,83 @@ +test: +{ + artifacts: + { + leader_object: + { + file_name: "icf_chain_leader.obj" + coff: { object: + { + machine: x64 + sections: { leader: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c390" } } } + symbols: + { + leader_definition: { kind: section_definition, section: leader, selection: NoDuplicates } + leader: { kind: external_function, name: "leader", section: leader, value: 0 } + } + } } + } + duplicate_object: + { + file_name: "icf_chain_duplicate.obj" + coff: { object: + { + machine: x64 + sections: + { + duplicate: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + duplicate: { type: Addr64, offset: 0, symbol: local_duplicate } + leader: { type: Addr64, offset: 8, symbol: leader } + } + } + } + symbols: + { + duplicate_definition: { kind: section_definition, section: duplicate, selection: Largest } + duplicate: { kind: external_function, name: "dup", section: duplicate, value: 0 } + local_duplicate: { kind: static, name: "local_dup", section: duplicate, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + leader: { kind: undefined_function, name: "leader" } + } + } } + } + selected_object: + { + file_name: "icf_chain_selected.obj" + coff: { object: + { + machine: x64 + sections: { duplicate: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c390" } } } + symbols: + { + duplicate_definition: { kind: section_definition, section: duplicate, selection: Largest } + duplicate: { kind: external_function, name: "dup", section: duplicate, value: 0 } + } + } } + } + entry_object: + { + file_name: "icf_chain_entry.obj" + coff: { object: + { + machine: x64 + sections: { entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external_function, name: "entry", section: entry, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_chain_leader.obj icf_chain_duplicate.obj icf_chain_selected.obj icf_chain_entry.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: equal } + } +} diff --git a/src/linker/tests/icf_cpp_identical_functions_fold.tst b/src/linker/tests/icf_cpp_identical_functions_fold.tst new file mode 100644 index 000000000..1f859180b --- /dev/null +++ b/src/linker/tests/icf_cpp_identical_functions_fold.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + source: + { + file_name: "icf_cpp_fold.cpp" + text: { data: { concat: + { + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " __declspec(noinline) int a(void) { return 42; }" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " __declspec(noinline) int b(void) { return 42; }" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " int (* volatile pa)(void) = a;" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " int (* volatile pb)(void) = b;" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " int entry(void) { return pa == pb ? 0 : 1; }" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: cl, output: none, args: "/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_fold.obj icf_cpp_fold.cpp" } + link: { output: none, args: "/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_fold.exe /opt:ref,icf /include:pa /include:pb icf_cpp_fold.obj" } + } + steps: { run: { path: "icf_cpp_fold.exe" } } +} diff --git a/src/linker/tests/icf_cpp_multihop_functions_do_not_fold.tst b/src/linker/tests/icf_cpp_multihop_functions_do_not_fold.tst new file mode 100644 index 000000000..6a388163c --- /dev/null +++ b/src/linker/tests/icf_cpp_multihop_functions_do_not_fold.tst @@ -0,0 +1,55 @@ +test: +{ + artifacts: + { + source: + { + file_name: "icf_cpp_multihop.c" + text: { data: { concat: + { + text: "__declspec(noinline) int leaf_a(void) { return 1; }" + hex: "0a" + text: "__declspec(noinline) int leaf_b(void) { return 2; }" + hex: "0a" + text: "__declspec(noinline) int mid_a(void) { return leaf_a(); }" + hex: "0a" + text: "__declspec(noinline) int mid_b(void) { return leaf_b(); }" + hex: "0a" + text: "__declspec(noinline) int top_a(void) { return mid_a(); }" + hex: "0a" + text: "__declspec(noinline) int top_b(void) { return mid_b(); }" + hex: "0a" + text: "int (* volatile p_top_a)(void) = top_a;" + hex: "0a" + text: "int (* volatile p_top_b)(void) = top_b;" + hex: "0a" + text: "int (* volatile p_mid_a)(void) = mid_a;" + hex: "0a" + text: "int (* volatile p_mid_b)(void) = mid_b;" + hex: "0a" + text: "int (* volatile p_leaf_a)(void) = leaf_a;" + hex: "0a" + text: "int (* volatile p_leaf_b)(void) = leaf_b;" + hex: "0a" + text: "int entry(void) {" + hex: "0a" + text: " if (p_top_a == p_top_b) { return 1; }" + hex: "0a" + text: " if (p_mid_a == p_mid_b) { return 2; }" + hex: "0a" + text: " if (p_leaf_a == p_leaf_b) { return 3; }" + hex: "0a" + text: " return 0;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: cl, output: none, args: "/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_multihop.obj icf_cpp_multihop.c" } + link: { output: none, args: "/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_multihop.exe /opt:ref,icf /include:p_top_a /include:p_top_b /include:p_mid_a /include:p_mid_b /include:p_leaf_a /include:p_leaf_b icf_cpp_multihop.obj" } + } + steps: { run: { path: "icf_cpp_multihop.exe" } } +} diff --git a/src/linker/tests/icf_cyclic_icf.tst b/src/linker/tests/icf_cyclic_icf.tst new file mode 100644 index 000000000..7e28f1529 --- /dev/null +++ b/src/linker/tests/icf_cyclic_icf.tst @@ -0,0 +1,37 @@ +test: +{ + artifacts: + { + object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_b: { type: Rel32, offset: 1, symbol: b_symbol } } } + b: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_a: { type: Rel32, offset: 1, symbol: a_symbol } } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + a_symbol: { kind: external, name: "a", section: a, value: 0 } + b_symbol: { kind: static, name: "b", section: b, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /out:a.exe /entry:a /opt:icf a.obj", artifact: image } } + steps: + { + // validate output + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": + { + // a and b folded into a self-call + data: e8fbffffffc3 + } } } } } + } +} diff --git a/src/linker/tests/icf_fold_diamond.tst b/src/linker/tests/icf_fold_diamond.tst new file mode 100644 index 000000000..74b0f31c5 --- /dev/null +++ b/src/linker/tests/icf_fold_diamond.tst @@ -0,0 +1,51 @@ +test: +{ + artifacts: + { + object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + a: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_b: { type: Rel32, offset: 1, symbol: b_symbol } + call_c: { type: Rel32, offset: 6, symbol: c_symbol } + } + } + b: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_d: { type: Rel32, offset: 1, symbol: d_symbol } } } + c: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_d: { type: Rel32, offset: 1, symbol: d_symbol } } } + d: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "4831c0c3" } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + c_definition: { kind: section_definition, section: c, selection: NoDuplicates } + d_definition: { kind: section_definition, section: d, selection: NoDuplicates } + a_symbol: { kind: external, name: "a", section: a, value: 0 } + b_symbol: { kind: external, name: "b", section: b, value: 0 } + c_symbol: { kind: external, name: "c", section: c, value: 0 } + d_symbol: { kind: external, name: "d", section: d, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:a /out:a.exe /opt:icf a.obj", artifact: image } } + steps: + { + // validate output + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: e80b000000e806000000c3cccccccccce80b000000c3cccccccccccccccccccc4831c0c3 } } } } } + } +} diff --git a/src/linker/tests/icf_fold_two_funcs.tst b/src/linker/tests/icf_fold_two_funcs.tst new file mode 100644 index 000000000..8ce86de05 --- /dev/null +++ b/src/linker/tests/icf_fold_two_funcs.tst @@ -0,0 +1,81 @@ +test: +{ + artifacts: + { + object: + { + file_name: "ident_funcs.obj" + coff: { object: + { + machine: x64 + sections: + { + a: + { + name: ".text$mn" + permissions: (read, execute) + content: code + flags: (link_comdat) + // xor rax, rax + // ret + data: { hex: "4831c0c3" } + } + b: + { + name: ".text$mb" + permissions: (read, execute) + content: code + flags: (link_comdat) + // xor rax, rax + // ret + data: { hex: "4831c0c3" } + } + entry: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // call a + // call b + // ret + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + a_symbol: { kind: external_function, name: "a", section: a, value: 0 } + b_symbol: { kind: external_function, name: "b", section: b, value: 0 } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:icf ident_funcs.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: { pe: { sections: { ".text": + { + // validate .text header + virtual_offset: 4096 + @range(20, 18446744073709551615) virtual_size + file_size: 512 + // entry + // pad + // a and b folded + @starts_with(e80b000000e806000000c3cccccccccc4831c0c3) data + } } } } + } + } +} diff --git a/src/linker/tests/icf_fold_with_largest_align.tst b/src/linker/tests/icf_fold_with_largest_align.tst new file mode 100644 index 000000000..ece99c150 --- /dev/null +++ b/src/linker/tests/icf_fold_with_largest_align.tst @@ -0,0 +1,96 @@ +test: +{ + artifacts: + { + a_object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + a: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 4 + flags: (link_comdat) + // mov rax, 1 + data: { hex: "48c7c001000000c3" } + } + b: { name: ".text", permissions: (read, execute), content: code, alignment: 8, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + entry_symbol: { kind: external, name: "entry", section: entry, value: 0 } + a_symbol: { kind: static, name: "a", section: a, value: 0 } + b_symbol: { kind: static, name: "b", section: b, value: 0 } + } + } } + } + // swap sections for a and b + b_object: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + a: { name: ".text", permissions: (read, execute), content: code, alignment: 8, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + b: { name: ".text", permissions: (read, execute), content: code, alignment: 4, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + entry_symbol: { kind: external, name: "entry", section: entry, value: 0 } + a_symbol: { kind: static, name: "a", section: a, value: 0 } + b_symbol: { kind: static, name: "b", section: b, value: 0 } + } + } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:b.exe /opt:icf b.obj", artifact: b_image } + } + steps: + { + // validate output in a.exe + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".text": { data: e80b000000e806000000c3cccccccccc48c7c001000000c3 } } } } } + // validate output in b.exe + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".text": { data: e80b000000e806000000c3cccccccccc48c7c001000000c3 } } } } } + } +} diff --git a/src/linker/tests/icf_identical_bytes_different_color_spaces_do_not_fold.tst b/src/linker/tests/icf_identical_bytes_different_color_spaces_do_not_fold.tst new file mode 100644 index 000000000..661d5452f --- /dev/null +++ b/src/linker/tests/icf_identical_bytes_different_color_spaces_do_not_fold.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_color_spaces.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + text: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + vftable: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + text: { type: Addr64, offset: 0, symbol: text_symbol } + vftable: { type: Addr64, offset: 8, symbol: vftable_symbol } + } + } + } + symbols: + { + text_definition: { kind: section_definition, section: text, selection: NoDuplicates } + vftable_definition: { kind: section_definition, section: vftable, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + text_symbol: { kind: external_function, name: "text", section: text, value: 0 } + vftable_symbol: { kind: external, name: "??_7type@@6B@", section: vftable, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_color_spaces.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, other_section: ".data", other_offset: 0, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_llvm_addrsig.tst b/src/linker/tests/icf_llvm_addrsig.tst new file mode 100644 index 000000000..09a2ddb5d --- /dev/null +++ b/src/linker/tests/icf_llvm_addrsig.tst @@ -0,0 +1,36 @@ +test: +{ + artifacts: + { + source: + { + file_name: "main.c" + text: { data: { concat: + { + text: "int foo() { return 123; }" + hex: "0a" + text: "int bar() { return 123; }" + hex: "0a" + text: "int main() {" + hex: "0a" + text: "int (*fn)() = &foo;" + hex: "0a" + text: "return fn != bar;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: clang, output: "main.obj", args: "main.c -ffunction-sections -target x86_64-pc-windows-msvc" } + link: { output: none, args: "main.obj /opt:icf /out:a.exe libcmt.lib" } + link: { output: none, args: "main.obj /opt:icf /out:a_no_addrsig.exe libcmt.lib /llvm_addrsig:no" } + } + steps: + { + run: { path: "a.exe", expect_exit: 1 } + run: { path: "a_no_addrsig.exe" } + } +} diff --git a/src/linker/tests/icf_llvm_addrsig_external_symbol.tst b/src/linker/tests/icf_llvm_addrsig_external_symbol.tst new file mode 100644 index 000000000..bfac6d790 --- /dev/null +++ b/src/linker/tests/icf_llvm_addrsig_external_symbol.tst @@ -0,0 +1,46 @@ +// .llvm_addrsig can name an undefined external whose definition is in another +// object; ICF must parse and mark the resolved symbol's object, not the referrer. +test: +{ + artifacts: + { + ref_source: + { + file_name: "ref.c" + text: { data: { concat: + { + text: "extern int ext_sig();" + hex: "0a" + text: "int (*ext_sig_addr)() = &ext_sig;" + hex: "0a" + text: "int entry() { return ext_sig_addr(); }" + hex: "0a" + } } } + } + def_source: + { + file_name: "def.c" + text: { data: { concat: + { + text: "int dummy0() { return 0; }" + hex: "0a" + text: "int dummy1() { return 1; }" + hex: "0a" + text: "int dummy2() { return 2; }" + hex: "0a" + text: "int dummy3() { return 3; }" + hex: "0a" + text: "int dummy4() { return 4; }" + hex: "0a" + text: "int ext_sig() { return 0; }" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: clang, output: "ref.obj", args: "ref.c -ffunction-sections -target x86_64-pc-windows-msvc" } + compile: { tool: clang, output: "def.obj", args: "def.c -ffunction-sections -target x86_64-pc-windows-msvc" } + link: { output: none, args: "ref.obj def.obj /subsystem:console /entry:entry /opt:icf /out:addrsig_ext.exe libcmt.lib" } + } +} diff --git a/src/linker/tests/icf_multihop_reloc_target_colors_do_not_fold.tst b/src/linker/tests/icf_multihop_reloc_target_colors_do_not_fold.tst new file mode 100644 index 000000000..ccaf7b253 --- /dev/null +++ b/src/linker/tests/icf_multihop_reloc_target_colors_do_not_fold.tst @@ -0,0 +1,69 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_multihop.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + top_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: mid_a_symbol } } } + top_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: mid_b_symbol } } } + mid_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: leaf_a_symbol } } } + mid_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: leaf_b_symbol } } } + leaf_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + leaf_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c002000000c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + top_a: { type: Addr64, offset: 0, symbol: top_a_symbol } + top_b: { type: Addr64, offset: 8, symbol: top_b_symbol } + mid_a: { type: Addr64, offset: 16, symbol: mid_a_symbol } + mid_b: { type: Addr64, offset: 24, symbol: mid_b_symbol } + leaf_a: { type: Addr64, offset: 32, symbol: leaf_a_symbol } + leaf_b: { type: Addr64, offset: 40, symbol: leaf_b_symbol } + } + } + } + symbols: + { + top_a_definition: { kind: section_definition, section: top_a, selection: NoDuplicates } + top_b_definition: { kind: section_definition, section: top_b, selection: NoDuplicates } + mid_a_definition: { kind: section_definition, section: mid_a, selection: NoDuplicates } + mid_b_definition: { kind: section_definition, section: mid_b, selection: NoDuplicates } + leaf_a_definition: { kind: section_definition, section: leaf_a, selection: NoDuplicates } + leaf_b_definition: { kind: section_definition, section: leaf_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + top_a_symbol: { kind: external_function, name: "top_a", section: top_a, value: 0 } + top_b_symbol: { kind: external_function, name: "top_b", section: top_b, value: 0 } + mid_a_symbol: { kind: external_function, name: "mid_a", section: mid_a, value: 0 } + mid_b_symbol: { kind: external_function, name: "mid_b", section: mid_b, value: 0 } + leaf_a_symbol: { kind: external_function, name: "leaf_a", section: leaf_a, value: 0 } + leaf_b_symbol: { kind: external_function, name: "leaf_b", section: leaf_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_multihop.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, nonzero: true, other_section: ".data", other_offset: 24, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 24, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u64, nonzero: true, other_section: ".data", other_offset: 40, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 40, type: u64, nonzero: true } + } +} diff --git a/src/linker/tests/icf_pdata_differs_by_function_color.tst b/src/linker/tests/icf_pdata_differs_by_function_color.tst new file mode 100644 index 000000000..1cf0ea0b6 --- /dev/null +++ b/src/linker/tests/icf_pdata_differs_by_function_color.tst @@ -0,0 +1,62 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_pdata_diff.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + fn_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "b801000000c3" } } + fn_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "b802000000c3" } } + xdata_a: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + xdata_b: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + pdata_a: { name: ".pdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "000000000000000000000000" }, relocations: { first: { type: Addr32Nb, offset: 0, symbol: fn_a_symbol }, last: { type: Addr32Nb, offset: 4, symbol: fn_a_symbol }, unwind: { type: Addr32Nb, offset: 8, symbol: unwind_a } } } + pdata_b: { name: ".pdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "000000000000000000000000" }, relocations: { first: { type: Addr32Nb, offset: 0, symbol: fn_b_symbol }, last: { type: Addr32Nb, offset: 4, symbol: fn_b_symbol }, unwind: { type: Addr32Nb, offset: 8, symbol: unwind_b } } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "000000000000000000000000000000000000000000000000" } + relocations: + { + pdata_a: { type: Addr64, offset: 0, symbol: pdata_a_symbol } + pdata_b: { type: Addr64, offset: 8, symbol: pdata_b_symbol } + unwind_a: { type: Addr64, offset: 16, symbol: unwind_a } + } + } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + xdata_a_definition: { kind: section_definition, section: xdata_a, selection: NoDuplicates } + xdata_b_definition: { kind: section_definition, section: xdata_b, selection: NoDuplicates } + pdata_a_definition: { kind: section_definition, section: pdata_a, selection: NoDuplicates } + pdata_b_definition: { kind: section_definition, section: pdata_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + unwind_a: { kind: external, name: "$unwind$a", section: xdata_a, value: 0 } + unwind_b: { kind: external, name: "$unwind$b", section: xdata_b, value: 0 } + pdata_a_symbol: { kind: external, name: "$pdata$a", section: pdata_a, value: 0 } + pdata_b_symbol: { kind: external, name: "$pdata$b", section: pdata_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_diff.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true } + } +} diff --git a/src/linker/tests/icf_pdata_xdata_fold.tst b/src/linker/tests/icf_pdata_xdata_fold.tst new file mode 100644 index 000000000..3dcc4453f --- /dev/null +++ b/src/linker/tests/icf_pdata_xdata_fold.tst @@ -0,0 +1,91 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_pdata_xdata_fold.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + fn_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + fn_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + xdata_a: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + xdata_b: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + pdata_a: + { + name: ".pdata" + permissions: (read) + content: initialized_data + alignment: 4 + flags: (link_comdat) + data: { hex: "000000000000000000000000" } + relocations: + { + first: { type: Addr32Nb, offset: 0, symbol: fn_a_symbol } + last: { type: Addr32Nb, offset: 4, symbol: fn_a_symbol } + unwind: { type: Addr32Nb, offset: 8, symbol: unwind_a } + } + } + pdata_b: + { + name: ".pdata" + permissions: (read) + content: initialized_data + alignment: 4 + flags: (link_comdat) + data: { hex: "000000000000000000000000" } + relocations: + { + first: { type: Addr32Nb, offset: 0, symbol: fn_b_symbol } + last: { type: Addr32Nb, offset: 4, symbol: fn_b_symbol } + unwind: { type: Addr32Nb, offset: 8, symbol: unwind_b } + } + } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "0000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + pdata_a: { type: Addr64, offset: 0, symbol: pdata_a_symbol } + pdata_b: { type: Addr64, offset: 8, symbol: pdata_b_symbol } + unwind_a: { type: Addr64, offset: 16, symbol: unwind_a } + unwind_b: { type: Addr64, offset: 24, symbol: unwind_b } + } + } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + xdata_a_definition: { kind: section_definition, section: xdata_a, selection: NoDuplicates } + xdata_b_definition: { kind: section_definition, section: xdata_b, selection: NoDuplicates } + pdata_a_definition: { kind: section_definition, section: pdata_a, selection: NoDuplicates } + pdata_b_definition: { kind: section_definition, section: pdata_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + unwind_a: { kind: external, name: "$unwind$a", section: xdata_a, value: 0 } + unwind_b: { kind: external, name: "$unwind$b", section: xdata_b, value: 0 } + pdata_a_symbol: { kind: external, name: "$pdata$a", section: pdata_a, value: 0 } + pdata_b_symbol: { kind: external, name: "$pdata$b", section: pdata_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_xdata_fold.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: equal } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, nonzero: true, other_section: ".data", other_offset: 24, relation: equal } + } +} diff --git a/src/linker/tests/icf_readonly_non_vftable_data_policy.tst b/src/linker/tests/icf_readonly_non_vftable_data_policy.tst new file mode 100644 index 000000000..9a24492d8 --- /dev/null +++ b/src/linker/tests/icf_readonly_non_vftable_data_policy.tst @@ -0,0 +1,58 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_rdata_policy.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + data_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + data_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + data_c: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + data_d: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "0000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + a: { type: Addr64, offset: 0, symbol: data_a_symbol } + b: { type: Addr64, offset: 8, symbol: data_b_symbol } + c: { type: Addr64, offset: 16, symbol: data_c_symbol } + d: { type: Addr64, offset: 24, symbol: data_d_symbol } + } + } + } + symbols: + { + data_a_definition: { kind: section_definition, section: data_a, selection: NoDuplicates } + data_b_definition: { kind: section_definition, section: data_b, selection: NoDuplicates } + data_c_definition: { kind: section_definition, section: data_c, selection: Any } + data_d_definition: { kind: section_definition, section: data_d, selection: Any } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + data_a_symbol: { kind: external, name: "data_a", section: data_a, value: 0 } + data_b_symbol: { kind: external, name: "data_b", section: data_b, value: 0 } + data_c_symbol: { kind: external, name: "data_c", section: data_c, value: 0 } + data_d_symbol: { kind: external, name: "data_d", section: data_d, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_rdata_policy.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, nonzero: true, other_section: ".data", other_offset: 24, relation: equal } + } +} diff --git a/src/linker/tests/icf_reloc_target_symbol_types_do_not_fold.tst b/src/linker/tests/icf_reloc_target_symbol_types_do_not_fold.tst new file mode 100644 index 000000000..dab3f4f96 --- /dev/null +++ b/src/linker/tests/icf_reloc_target_symbol_types_do_not_fold.tst @@ -0,0 +1,109 @@ +// ICF must preserve identical sections with different symbol targets +test: +{ + artifacts: + { + entry_object: + { + file_name: "icf_interp_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } // ret + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "000000000000000000000000000000000000000000000000" } + relocations: + { + regular: { type: Addr64, offset: 0, symbol: fn_regular } + common: { type: Addr64, offset: 8, symbol: fn_common } + absolute: { type: Addr64, offset: 16, symbol: fn_abs } + } + } + } + symbols: + { + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + fn_regular: { kind: undefined, name: "fn_regular" } + fn_common: { kind: undefined, name: "fn_common" } + fn_abs: { kind: undefined, name: "fn_abs" } + } + } } + } + regular_object: + { + file_name: "icf_interp_regular.obj" + coff: { object: + { + machine: x64 + sections: + { + fn: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, target + // ret + data: { hex: "48c7c000000000c3" } + relocations: { target: { type: Rel32, offset: 3, symbol: target } } + } + target: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "00" } } + } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_regular", section: fn, value: 0 } + target: { kind: external, name: "target_regular", section: target, value: 0 } + } + } } + } + common_object: + { + file_name: "icf_interp_common.obj" + coff: { object: + { + machine: x64 + sections: { fn: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c000000000c3" }, relocations: { target: { type: Rel32, offset: 3, symbol: target } } } } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_common", section: fn, value: 0 } + target: { kind: common, name: "target_common", size: 8 } + } + } } + } + absolute_object: + { + file_name: "icf_interp_abs.obj" + coff: { object: + { + machine: x64 + sections: { fn: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c000000000c3" }, relocations: { target: { type: Rel32, offset: 3, symbol: target } } } } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_abs", section: fn, value: 0 } + target: { kind: absolute, name: "target_abs", value: 4660, storage: external } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_interp_entry.obj icf_interp_regular.obj icf_interp_common.obj icf_interp_abs.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 16, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, other_section: ".data", other_offset: 16, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_same_but_different.tst b/src/linker/tests/icf_same_but_different.tst new file mode 100644 index 000000000..b91d85d61 --- /dev/null +++ b/src/linker/tests/icf_same_but_different.tst @@ -0,0 +1,97 @@ +test: +{ + artifacts: + { + object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + a: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // call $ + data: { hex: "e800000000c3" } + relocations: { call_c: { type: Rel32, offset: 1, symbol: c_symbol } } + } + b: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // call $ + data: { hex: "e800000000c3" } + relocations: { call_d: { type: Rel32, offset: 1, symbol: d_symbol } } + } + c: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // mov rax, 1 + // ret + data: { hex: "48c7c001000000c3" } + } + d: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // mov rax, 2 + // ret + data: { hex: "48c7c002000000c3" } + } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + c_definition: { kind: section_definition, section: c, selection: NoDuplicates } + d_definition: { kind: section_definition, section: d, selection: NoDuplicates } + entry_symbol: { kind: external, name: "entry", section: entry, value: 0 } + a_symbol: { kind: external, name: "a", section: a, value: 0 } + b_symbol: { kind: external, name: "b", section: b, value: 0 } + c_symbol: { kind: external, name: "c", section: c, value: 0 } + d_symbol: { kind: external, name: "d", section: d, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj", artifact: image } } + steps: + { + // validate output + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": + { + // call a + // call b + // call c + // call d + // mov rax, 1 + // mov rax, 2 + data: e80b000000e816000000c3cccccccccce81b000000c3cccccccccccccccccccce81b000000c3cccccccccccccccccccc48c7c001000000c3cccccccccccccccc48c7c002000000c3 + } } } } } + } +} diff --git a/src/linker/tests/icf_unresolved_reloc_targets_do_not_fold.tst b/src/linker/tests/icf_unresolved_reloc_targets_do_not_fold.tst new file mode 100644 index 000000000..19a5c4520 --- /dev/null +++ b/src/linker/tests/icf_unresolved_reloc_targets_do_not_fold.tst @@ -0,0 +1,91 @@ +// ICF must preserve sections with unresolved target symbols (with /FORCE) +test: +{ + artifacts: + { + entry_object: + { + file_name: "icf_unresolved_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } // ret + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + undef: { type: Addr64, offset: 0, symbol: fn_undef } + weak: { type: Addr64, offset: 8, symbol: fn_weak } + } + } + } + symbols: + { + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + fn_undef: { kind: undefined, name: "fn_undef" } + fn_weak: { kind: undefined, name: "fn_weak" } + } + } } + } + undef_object: + { + file_name: "icf_unresolved_undef.obj" + coff: { object: + { + machine: x64 + sections: + { + fn: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, target + // ret + data: { hex: "48c7c000000000c3" } + relocations: { target: { type: Rel32, offset: 3, symbol: target } } + } + } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_undef", section: fn, value: 0 } + target: { kind: undefined, name: "target_undef" } + } + } } + } + weak_object: + { + file_name: "icf_unresolved_weak.obj" + coff: { object: + { + machine: x64 + sections: { fn: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c000000000c3" }, relocations: { target: { type: Rel32, offset: 3, symbol: target } } } } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_weak", section: fn, value: 0 } + fallback: { kind: absolute, name: "target_weak_fallback", value: 0, storage: external } + target: { kind: weak, name: "target_weak", fallback: fallback, search: no_library } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/force /subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_unresolved_entry.obj icf_unresolved_undef.obj icf_unresolved_weak.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, other_section: ".data", other_offset: 0, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_vftable_and_vbtable_policy.tst b/src/linker/tests/icf_vftable_and_vbtable_policy.tst new file mode 100644 index 000000000..23252aaef --- /dev/null +++ b/src/linker/tests/icf_vftable_and_vbtable_policy.tst @@ -0,0 +1,79 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_tables.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + vf_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + vf_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + vb_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + vb_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + rtti_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "01" } } + rtti_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "02" } } + vf_ref_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_a_symbol } } } + vf_ref_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_b_symbol } } } + vb_ref_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_a_symbol } } } + vb_ref_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_b_symbol } } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + vf_a: { type: Addr64, offset: 0, symbol: vf_a_symbol } + vf_b: { type: Addr64, offset: 8, symbol: vf_b_symbol } + vb_a: { type: Addr64, offset: 16, symbol: vb_a_symbol } + vb_b: { type: Addr64, offset: 24, symbol: vb_b_symbol } + vf_ref_a: { type: Addr64, offset: 32, symbol: vf_ref_a_symbol } + vf_ref_b: { type: Addr64, offset: 40, symbol: vf_ref_b_symbol } + vb_ref_a: { type: Addr64, offset: 48, symbol: vb_ref_a_symbol } + vb_ref_b: { type: Addr64, offset: 56, symbol: vb_ref_b_symbol } + } + } + } + symbols: + { + vf_a_definition: { kind: section_definition, section: vf_a, selection: NoDuplicates } + vf_b_definition: { kind: section_definition, section: vf_b, selection: NoDuplicates } + vb_a_definition: { kind: section_definition, section: vb_a, selection: NoDuplicates } + vb_b_definition: { kind: section_definition, section: vb_b, selection: NoDuplicates } + vf_ref_a_definition: { kind: section_definition, section: vf_ref_a, selection: NoDuplicates } + vf_ref_b_definition: { kind: section_definition, section: vf_ref_b, selection: NoDuplicates } + vb_ref_a_definition: { kind: section_definition, section: vb_ref_a, selection: NoDuplicates } + vb_ref_b_definition: { kind: section_definition, section: vb_ref_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + vf_a_symbol: { kind: external, name: "??_7a@@6B@", section: vf_a, value: 0 } + vf_b_symbol: { kind: external, name: "??_7b@@6B@", section: vf_b, value: 0 } + vb_a_symbol: { kind: external, name: "??_8a@@7B@", section: vb_a, value: 0 } + vb_b_symbol: { kind: external, name: "??_8b@@7B@", section: vb_b, value: 0 } + vf_ref_a_symbol: { kind: external, name: "??_7ra@@6B@", section: vf_ref_a, value: 0 } + vf_ref_b_symbol: { kind: external, name: "??_7rb@@6B@", section: vf_ref_b, value: 0 } + vb_ref_a_symbol: { kind: external, name: "??_8ra@@7B@", section: vb_ref_a, value: 0 } + vb_ref_b_symbol: { kind: external, name: "??_8rb@@7B@", section: vb_ref_b, value: 0 } + rtti_a_symbol: { kind: external, name: "rtti_a", section: rtti_a, value: 0 } + rtti_b_symbol: { kind: external, name: "rtti_b", section: rtti_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_tables.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: equal } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, other_section: ".data", other_offset: 24, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u64, other_section: ".data", other_offset: 40, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 48, type: u64, other_section: ".data", other_offset: 56, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_vftable_external_symbol_at_nonzero_offset.tst b/src/linker/tests/icf_vftable_external_symbol_at_nonzero_offset.tst new file mode 100644 index 000000000..770f42219 --- /dev/null +++ b/src/linker/tests/icf_vftable_external_symbol_at_nonzero_offset.tst @@ -0,0 +1,63 @@ +test: +{ + artifacts: + { + vftable_obj: + { + file_name: "vftable.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + ??_7B] + hex: "c3" // ret + } } + relocations: { vftable_b_ref: { type: Rel32, offset: 3, symbol: vftable_b } } + } + vftable_a: { name: ".rdata$vt", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + vftable_b: { name: ".rdata$vt", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 16 } + relocations: + { + vftable_a_ref: { type: Addr64, offset: 0, symbol: vftable_a } + vftable_b_ref: { type: Addr64, offset: 8, symbol: vftable_b } + } + } + } + symbols: + { + vftable_a_def: { kind: section_definition, section: vftable_a, selection: Any } + vftable_b_def: { kind: section_definition, section: vftable_b, selection: Any } + rtti_a: { kind: external, name: "??_R4A@@6B@", section: vftable_a, value: 0 } + vftable_a: { kind: external, name: "??_7A@@6B@", section: vftable_a, value: 8 } + rtti_b: { kind: external, name: "??_R4B@@6B@", section: vftable_b, value: 0 } + vftable_b: { kind: external, name: "??_7B@@6B@", section: vftable_b, value: 8 } + entry: { kind: external_function, name: "entry", section: entry, value: 0 } + addresses: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses vftable.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, modulo: 16, remainder: 8 } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, modulo: 16, remainder: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, other_section: ".data", other_offset: 8, other_type: u64, relation: equal } + } +} diff --git a/src/linker/tests/image_base.tst b/src/linker/tests/image_base.tst new file mode 100644 index 000000000..ff21821ce --- /dev/null +++ b/src/linker/tests/image_base.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + image_base_obj: + { + file_name: "image_base.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "488d0d00000000" // lea rcx, [__ImageBase] + hex: "48b80000000000000000" // mov rax, __ImageBase + hex: "b800000000" // mov eax, __ImageBase + hex: "c3" // ret + } + } + relocations: + { + relative: { type: Rel32, offset: 3, symbol: image_base } + absolute_64: { type: Addr64, offset: 9, symbol: image_base } + image_relative_32: { type: Addr32Nb, offset: 18, symbol: image_base } + } + } } + symbols: + { + image_base: { kind: undefined, name: "__ImageBase" } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /base:0x2000000140000000 /out:a.exe image_base.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 488d0df9efffff48b80000004001000020b800000000c3 } } } } } } +} diff --git a/src/linker/tests/import_export.tst b/src/linker/tests/import_export.tst new file mode 100644 index 000000000..8030fb360 --- /dev/null +++ b/src/linker/tests/import_export.tst @@ -0,0 +1,293 @@ +test: +{ + // write objs + artifacts: + { + import_obj: + { + file_name: "import.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 1024 } + relocations: + { + imp_foo_ref: { type: Addr32Nb, offset: 0, symbol: imp_foo } + imp_bar_ref: { type: Addr32Nb, offset: 4, symbol: imp_bar } + imp_baz_ref: { type: Addr32Nb, offset: 8, symbol: imp_baz } + imp_baf_ref: { type: Addr32Nb, offset: 12, symbol: imp_baf } + imp_ord_ref: { type: Addr32Nb, offset: 16, symbol: imp_ord } + bar_ref: { type: Addr32Nb, offset: 20, symbol: bar } + foo_ref: { type: Addr32Nb, offset: 24, symbol: foo } + ord_ref: { type: Addr32Nb, offset: 28, symbol: ord } + } + } + } + symbols: + { + imp_foo: { kind: undefined, name: "__imp_foo" } + imp_bar: { kind: undefined, name: "__imp_bar" } + imp_baz: { kind: undefined, name: "__imp_baz" } + imp_baf: { kind: undefined, name: "__imp_baf" } + imp_ord: { kind: undefined, name: "__imp_ord" } + bar: { kind: undefined, name: "bar" } + foo: { kind: undefined, name: "foo" } + ord: { kind: undefined, name: "ord" } + //"baf", + //"baz", + //"__imp_ord2", + //"__imp_ord4", + } + } } + } + + export_obj: + { + file_name: "export.obj" + coff: { object: + { + machine: x64 + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "test" } } + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "b801000000c3" } } + } + symbols: + { + entry: { kind: external, name: "_DllMainCRTStartup", section: text, value: 0 } + foo: { kind: external, name: "foo", section: data, value: 0 } + ord: { kind: external, name: "ord", section: data, value: 1 } + ord2: { kind: external, name: "ord2", section: data, value: 2 } + ord3: { kind: external, name: "ord3", section: data, value: 9 } + ord4: { kind: external, name: "ord4", section: data, value: 10 } + } + directives: + { + directive: "/export:foo=foo" + directive: "/export:bar=foo" + directive: "/export:ord,@5" + directive: "/export:ord2,@6,DATA" + directive: "/export:ord3,@7,NONAME,PRIVATE" + directive: "/export:ord4,@8,NONAME,DATA" + directive: "/export:baz=BAZ.qwe" + directive: "/export:baf=BAZ.#1" + } + } } + } + + baz_obj: + { + file_name: "baz.obj" + coff: { object: + { + machine: x64 + sections: + { + s1: { name: ".s1", permissions: (read, write), content: initialized_data, data: { text: "s1" } } + s2: { name: ".s2", permissions: (read, write), content: initialized_data, data: { text: "s2" } } + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "_DllMainCRTStartup", section: text, value: 0 } + s1: { kind: external, name: "s1", section: s1, value: 0 } + s2: { kind: external, name: "s2", section: s2, value: 0 } + } + directives: + { + directive: "/export:baf=s1" + directive: "/export:baz=s2" + } + } } + } + + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + + export_dll: { file_name: "export.dll", pe: {} } + baz_dll: { file_name: "baz.dll", pe: {} } + + loader_source: + { + file_name: "import_export_loader.c" + text: { data: { concat: + { + text: "#include " + hex: "0a" + text: "int main(void) {" + hex: "0a" + text: " if (!SetDllDirectoryA(" + hex: "22" + text: "." + hex: "22" + text: ")) return 1;" + hex: "0a" + text: " HMODULE export_dll = LoadLibraryA(" + hex: "22" + text: "export.dll" + hex: "22" + text: ");" + hex: "0a" + text: " if (!export_dll) return 2;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "bar" + hex: "22" + text: ")) return 3;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "foo" + hex: "22" + text: ")) return 4;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "ord" + hex: "22" + text: ")) return 5;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "ord2" + hex: "22" + text: ")) return 6;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(10))) return 7;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(12))) return 8;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(5))) return 9;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(6))) return 10;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(7))) return 11;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(8))) return 12;" + hex: "0a" + text: " FreeLibrary(export_dll);" + hex: "0a" + text: " return 0;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + + // link dlls + build: + { + link: { args: "/dll /nodefaultlib /out:export.dll export.obj", artifact: export_dll } // export.dll + link: { args: "/dll /out:baz.dll /export:s1,@1,NONAME /export:qwe=s2 baz.obj", artifact: baz_dll } // baz.dll + compile_link: { tool: msvc, output: "import_export_loader.exe", args: "/nologo import_export_loader.c" } + } + + steps: + { + // validate export table in export.dll + // validate export table in export.dll + expect_pe: + { + artifact: export_dll + expected: + { + pe: + { + sections: { ".data": { virtual_offset: 12288 } } + exports: + { + // validate header + flags: 0 + timestamp: 4294967295 + major_version: 0 + minor_version: 0 + ordinal_base: 5 + count: 8 + entries: + { + // validate names + export_0: { name: "baf", forwarder: "BAZ.#1", ordinal: 9 } + export_1: { name: "bar", forwarder: "", virtual_offset: 12288, ordinal: 10 } + export_2: { name: "baz", forwarder: "BAZ.qwe", ordinal: 11 } + export_3: { name: "foo", forwarder: "", virtual_offset: 12288, ordinal: 12 } + export_4: { name: "ord", forwarder: "", virtual_offset: 12289, ordinal: 5 } + export_5: { name: "ord2", forwarder: "", virtual_offset: 12290, ordinal: 6 } + export_6: { name: "", forwarder: "", virtual_offset: 12297, ordinal: 7 } + export_7: { name: "", forwarder: "", virtual_offset: 12298, ordinal: 8 } + // validate forwarders + // validate voffs + // validate ordinals + } + } + } + } + } + + // validate export table in baz.dll + expect_pe: + { + artifact: baz_dll + expected: + { + pe: + { + exports: + { + // validate header + flags: 0 + timestamp: 4294967295 + major_version: 0 + minor_version: 0 + ordinal_base: 1 + count: 4 + entries: + { + // validate names + export_0: { name: "baf", forwarder: "", virtual_offset: 12288, ordinal: 2 } + export_1: { name: "baz", forwarder: "", virtual_offset: 16384, ordinal: 3 } + export_2: { name: "qwe", forwarder: "", virtual_offset: 16384, ordinal: 4 } + export_3: { name: "", forwarder: "", virtual_offset: 12288, ordinal: 1 } + // validate forwarders + // validate voffs + // validate ordinals + } + } + } + } + } + + // test query by function name + //T_Ok(GetProcAddress(export_dll, "baf")); + //T_Ok(GetProcAddress(export_dll, "baz")); + // test query by ordinal + //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(9))); + //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(11))); + run: { path: "import_export_loader.exe" } + + //T_Ok(t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /delayload:export.dll /export:entry kernel32.Lib delayimp.lib libcmt.lib export.lib import.obj entry.obj") == 0); + // TODO: check import table + } +} diff --git a/src/linker/tests/import_kernel32.tst b/src/linker/tests/import_kernel32.tst new file mode 100644 index 000000000..05850aeaf --- /dev/null +++ b/src/linker/tests/import_kernel32.tst @@ -0,0 +1,118 @@ +test: +{ + + artifacts: + { + import_obj: + { + file_name: "import.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { hex: "7465737400" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + // sub rsp,68h ; alloc space on stack + // mov dword ptr [rsp+48h],18h ; SECURITY_ATTRIBUTES.nLength + // mov qword ptr [rsp+50h],0 ; SECURITY_ATTRIBUTES.lpSecurityDescriptor + // mov dword ptr [rsp+58h],0 ; SECURITY_ATTRIBUTES.bInheritHandle + // mov qword ptr [rsp+30h],0 ; hTemplateFile + // mov dword ptr [rsp+28h],80h ; dwFlagsAndAttributes + // mov dword ptr [rsp+20h],2 ; dwCreationDisposition + // lea r9,[rsp+48h] ; lpSecurityAttributes + // xor r8d,r8d ; dwShareMode + // mov edx,40000000h ; dwDesiredAccess + // lea rcx,[test] ; lpFileName + // call qword ptr [__imp_CreateFileA] ; call CreateFileA + // mov rcx,rax ; hObject + // call qword ptr [__imp_CloseHandle] ; call CloseHandle + // xor eax,eax ; clear result + // add rsp,68h ; dealloc stack + // ret ; return + data: { hex: "4883ec68c74424481800000048c744245000000000c74424580000000048c744243000000000c744242880000000c7442420020000004c8d4c24484533c0ba00000040488d0d00000000ff15000000004889c1ff150000000033c04883c468c3" } + relocations: + { + test_ref: { type: Rel32, offset: 70, symbol: test } + create_file_ref: { type: Rel32, offset: 76, symbol: create_file } + close_handle_ref: { type: Rel32, offset: 85, symbol: close_handle } + } + } + dead: + { + name: ".text$dead" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: { zero: 4 } + relocations: + { + compare_string_ref: { type: Addr32Nb, offset: 0, symbol: compare_string } + } + } + } + symbols: + { + dead_definition: { kind: section_definition, section: dead, selection: Any } + test: { kind: external, name: "test", section: data, value: 0 } + entry: { kind: external, name: "entry", section: text, value: 0 } + dead: { kind: external_function, name: "dead", section: dead, value: 0 } + create_file: { kind: undefined, name: "__imp_CreateFileA" } + close_handle: { kind: undefined, name: "__imp_CloseHandle" } + compare_string: { kind: undefined, name: "__imp_CompareStringW" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /fixed /opt:ref import.obj kernel32.lib" + artifact: image + } + } + + steps: + { + expect_pe: + { + artifact: image + // The two exact entries prove CreateFileA and CloseHandle are present and CompareStringW is absent. + expected: + { + pe: + { + imports: + { + count: 1 + dll_0: + { + count: 2 + entries: + { + import_0: { type: name, name: "CloseHandle" } + import_1: { type: name, name: "CreateFileA" } + } + } + } + } + } + } + run: { path: "a.exe" } + expect_file: { path: "test", nonempty: false } + } +} diff --git a/src/linker/tests/include.tst b/src/linker/tests/include.tst new file mode 100644 index 000000000..7c6a461cc --- /dev/null +++ b/src/linker/tests/include.tst @@ -0,0 +1,41 @@ +test: +{ + artifacts: + { + include_lib: + { + file_name: "include.lib" + coff: { library: { second_linker_member: true, members: { include_member: { + path: "include.obj" + object: { machine: x64, + sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "foo" } } } + symbols: { foo: { kind: external, name: "foo", section: data, value: 0 } } + } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { entry_ref: { type: Addr32Nb, offset: 0, symbol: entry } } + } }, symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // simple include test + link: { args: "/subsystem:console /entry:entry /out:a.exe /include:foo entry.obj include.lib", artifact: image } + // test unresolved include + link: { args: "/subsystem:console /entry:entry /out:a.exe /include:ewq entry.obj", expect_exit: 47 } + } + steps: + { + // validate that linker pulled-in include.obj + expect_pe: { artifact: image, expected: { pe: { sections: { ".data": { data: 666f6f } } } } } + } +} diff --git a/src/linker/tests/infer_asan.tst b/src/linker/tests/infer_asan.tst new file mode 100644 index 000000000..51786f3c1 --- /dev/null +++ b/src/linker/tests/infer_asan.tst @@ -0,0 +1,52 @@ +test: +{ + artifacts: + { + source: + { + file_name: "main.c" + text: { data: { concat: + { + text: "#include " + hex: "0a" + text: " int main(void) {" + hex: "0a" + text: "int *foo = malloc(sizeof(*foo));" + hex: "0a" + text: "free(foo);" + hex: "0a" + text: "*foo = 1;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + + build: + { + // /MD + compile: { tool: cl, output: none, args: "/MD /fsanitize=address /Z7 /c /Fo:main_md.obj main.c" } + link: { output: none, args: "main_md.obj /debug:full" } + + // /MDd + compile: { tool: cl, output: none, args: "/MDd /fsanitize=address /Z7 /c /Fo:main_mdd.obj main.c" } + link: { output: none, args: "main_mdd.obj /debug:full" } + + // /MT + compile: { tool: cl, output: none, args: "/MT /fsanitize=address /Z7 /c /Fo:main_mt.obj main.c" } + link: { output: none, args: "main_mt.obj /debug:full" } + + // /MTd + compile: { tool: cl, output: none, args: "/MT /fsanitize=address /Z7 /c /Fo:main_mtd.obj main.c" } + link: { output: none, args: "main_mtd.obj /debug:full" } + } + + steps: + { + run: { path: "main_md.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + run: { path: "main_mdd.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + run: { path: "main_mt.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + run: { path: "main_mtd.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + } +} diff --git a/src/linker/tests/invalid_bss.tst b/src/linker/tests/invalid_bss.tst new file mode 100644 index 000000000..c8b84ccf1 --- /dev/null +++ b/src/linker/tests/invalid_bss.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + bss_obj: + { + file_name: "bss.obj" + coff: { object: + { + machine: x64 + sections: { bss: { name: ".bss", permissions: (read), content: initialized_data, data: { text: "Hello, World" } } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe bss.obj entry.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + sections: + { + ".bss": + { + virtual_size: 12 + raw_flags: 1073741888 + data: 48656c6c6f2c20576f726c64 + } + } + } + } + } + } +} diff --git a/src/linker/tests/lib_member_imp_and_regular_symbol_queued_once.tst b/src/linker/tests/lib_member_imp_and_regular_symbol_queued_once.tst new file mode 100644 index 000000000..ea60c799f --- /dev/null +++ b/src/linker/tests/lib_member_imp_and_regular_symbol_queued_once.tst @@ -0,0 +1,38 @@ +test: +{ + artifacts: + { + rust_style_lib: + { + file_name: "rust_style.rlib" + coff: { library: { second_linker_member: true, members: { rust_member: { + path: "core-9f9efb2036858c45.core.78298229696da45f-cgu.0.rcgu.o" + object: { machine: x64, sections: { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + idata: { name: ".idata", permissions: (read), content: initialized_data, alignment: 8, data: { zero: 8 } } + }, symbols: { + foo: { kind: external_function, name: "foo", section: text, value: 0 } + imp_foo: { kind: external, name: "__imp_foo", section: idata, value: 0 } + } } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1 + data: { hex: "48c7c00000000048c7c100000000c3" } + relocations: { + foo_ref: { type: Addr32Nb, offset: 3, symbol: foo } + imp_foo_ref: { type: Addr32Nb, offset: 10, symbol: imp_foo } + } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + imp_foo: { kind: undefined, name: "__imp_foo" } + } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj rust_style.rlib" } } + steps: {} +} diff --git a/src/linker/tests/lib_member_reloc_apply_off_out_of_bounds.tst b/src/linker/tests/lib_member_reloc_apply_off_out_of_bounds.tst new file mode 100644 index 000000000..22de23107 --- /dev/null +++ b/src/linker/tests/lib_member_reloc_apply_off_out_of_bounds.tst @@ -0,0 +1,55 @@ +test: +{ + artifacts: + { + bad_lib: + { + file_name: "bad.lib" + coff: { library: + { + second_linker_member: true + members: { malformed: + { + path: "bad_member.obj" + object: + { + machine: x64 + sections: { data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000" } + relocations: { target: { type: Addr32, offset: 4294967295, symbol: target } } + } } + symbols: + { + target: { kind: absolute, name: "target", value: 0, storage: static } + bad: { kind: external, name: "bad", section: data, value: 0 } + } + } + } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:archive_relocation_offset_out_of_bounds.exe /include:bad entry.obj bad.lib" + expect_exit: nonzero + } + } + steps: {} +} diff --git a/src/linker/tests/lib_search_rescans_changed_symbol.tst b/src/linker/tests/lib_search_rescans_changed_symbol.tst new file mode 100644 index 000000000..de3c6b798 --- /dev/null +++ b/src/linker/tests/lib_search_rescans_changed_symbol.tst @@ -0,0 +1,71 @@ +test: +{ + // provider.lib is searched before target exists. seed.lib introduces a + // no-library weak target; promote.lib then replaces it in-place with an + // earlier search-library weak. No search-chunk entry is appended, so the + // existing entry must be revisited when provider.lib is searched again. + artifacts: + { + provider_lib: + { + file_name: "provider.lib" + coff: { library: { second_linker_member: true, members: { provider: { + path: "provider.obj" + object: { machine: x64, + sections: { target: { + name: ".target", permissions: (read), content: initialized_data, + alignment: 1, data: { text: "provider" } + } } + symbols: { target: { kind: external, name: "target", section: target, value: 0 } } + } + } } } } + } + promote_lib: + { + file_name: "promote.lib" + coff: { library: { second_linker_member: true, members: { promote: { + path: "promote.obj" + object: { machine: x64, symbols: { + load_promote: { kind: absolute, name: "load_promote", value: 1, storage: external } + fallback: { kind: absolute, name: "promote_fallback", value: 2, storage: external } + target: { kind: weak, name: "target", fallback: fallback, search: search_library } + } } + } } } } + } + seed_lib: + { + file_name: "seed.lib" + coff: { library: { second_linker_member: true, members: { seed: { + path: "seed.obj" + object: { machine: x64, symbols: { + load_seed: { kind: absolute, name: "load_seed", value: 1, storage: external } + load_promote: { kind: undefined, name: "load_promote" } + fallback: { kind: absolute, name: "seed_fallback", value: 3, storage: external } + target: { kind: weak, name: "target", fallback: fallback, search: no_library } + } } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { + name: ".text", permissions: (read, execute), content: code, + alignment: 1, data: { hex: "c3" } + } } + symbols: { + entry: { kind: external_function, name: "entry", section: text, value: 0 } + load_seed: { kind: undefined, name: "load_seed" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { + args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe provider.lib promote.lib seed.lib entry.obj" + artifact: image + } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { + ".target": { data: 70726f7669646572 } + } } } } } +} diff --git a/src/linker/tests/link_undef.tst b/src/linker/tests/link_undef.tst new file mode 100644 index 000000000..adbfb9597 --- /dev/null +++ b/src/linker/tests/link_undef.tst @@ -0,0 +1,30 @@ +test: +{ + artifacts: + { + undef_obj: { file_name: "undef.obj", coff: { object: { machine: x64, symbols: { undef: { kind: undefined, name: "undef" } } } } } + import_obj: { file_name: "import.obj", coff: { object: { machine: x64, symbols: { missing: { kind: undefined, name: "__imp_missing_dependency" } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { undef_ref: { type: Addr32Nb, offset: 0, symbol: undef } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + undef: { kind: undefined, name: "undef" } + } } } + } + } + build: + { + // try linking unresolved symbol and see if linker picks up on that + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj", expect_exit: 47 } + link: { + args: "/subsystem:console /entry:entry /out:missing_import.exe entry.obj import.obj" + expect_exit: 47 + stderr_matches: "*unresolved symbol '__imp_missing_dependency'*this is a DLL import*response file includes the import library for this module dependency*" + } + } + steps: {} +} diff --git a/src/linker/tests/link_unref_undef.tst b/src/linker/tests/link_unref_undef.tst new file mode 100644 index 000000000..0ae636f23 --- /dev/null +++ b/src/linker/tests/link_unref_undef.tst @@ -0,0 +1,20 @@ +test: +{ + artifacts: + { + undef_obj: { file_name: "undef.obj", coff: { object: { machine: x64, symbols: { undef: { kind: undefined, name: "undef" } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } + } }, symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } } } + } + } + build: + { + // try linking unreferenced unresolved symbol, this must link + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj", expect_exit: nonzero } + } + steps: {} +} diff --git a/src/linker/tests/linker_tests.c b/src/linker/tests/linker_tests.c index c0b463317..f2fd59885 100644 --- a/src/linker/tests/linker_tests.c +++ b/src/linker/tests/linker_tests.c @@ -1,19 +1,6 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) -// TODO: -// [x] defer_duplicate_imp_link -// [x] fold_two_funcs -// [x] same_but_different -// [x] fold_diamond -// [x] cyclic_icf -// [x] fold_with_largest_align -// [ ] opt_ref_comdat_undef_section -// [ ] opt_ref_weak_alias_comdat -// [ ] reloc_apply_off_out_of_bounds -// [ ] lib_member_reloc_apply_off_out_of_bounds -// [ ] relocate_undefined_section_symbol - //////////////////////////////// // Def -> COFF @@ -141,22 +128,6 @@ typedef struct T_COFF_DefLibMember *members; } T_COFF_DefLib; -typedef enum -{ - T_COFF_DefRootType_Null, - T_COFF_DefRootType_Obj, - T_COFF_DefRootType_Lib -} T_COFF_DefRootType; - -typedef struct -{ - T_COFF_DefRootType type; - union { - T_COFF_DefObj obj; - T_COFF_DefLib lib; - }; -} T_COFF_DefRoot; - internal COFF_ObjSection * t_coff_from_def_require_section(HashTable *section_ht, String8 id) { @@ -347,18 +318,6 @@ t_coff_from_def_lib(Arena *arena, T_COFF_DefLib lib) return result; } -internal String8 -t_coff_from_def_root(Arena *arena, T_COFF_DefRoot root) -{ - switch (root.type) { - case T_COFF_DefRootType_Obj: return t_coff_from_def_obj(arena, root.obj); - case T_COFF_DefRootType_Lib: return t_coff_from_def_lib(arena, root.lib); - case T_COFF_DefRootType_Null: break; - default: { NotImplemented; } break; - } - return str8_zero(); -} - internal B32 t_write_def_obj(char *path, T_COFF_DefObj obj) { @@ -368,43 +327,8 @@ t_write_def_obj(char *path, T_COFF_DefObj obj) return is_ok; } -internal B32 -t_write_def_lib(char *path, T_COFF_DefLib lib) -{ - Temp scratch = scratch_begin(0,0); - B32 is_ok = t_write_file(str8_cstring(path), t_coff_from_def_lib(scratch.arena, lib)); - scratch_end(scratch); - return is_ok; -} - //////////////////////////////// -internal String8 -t_make_sec_defn_obj(Arena *arena, String8 payload) -{ - return t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "mysect", ".mysect", payload, .flags = "r:data@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("mysect", COFF_ComdatSelect_Null), - {0} - } - }); -} - -internal String8 -t_make_obj_with_directive(Arena *arena, String8 directive) -{ - String8 directive_cstr = push_cstr(arena, directive); - return t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .directives = (char *[]){ (char *)directive_cstr.str, 0 } - }); -} - internal String8 t_make_entry_obj(Arena *arena) { @@ -431,9250 +355,21 @@ t_write_entry_obj(void) return is_ok; } -//////////////////////////////// - -TEST(machine_compat_check) -{ - // unknown.obj - T_Ok(t_write_def_obj("unknown.obj", (T_COFF_DefObj){ - .machine = &(COFF_MachineType){ COFF_MachineType_Unknown }, - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("unknown"), .flags = "rw:data" }, - {0} - } - })); - - // x64.obj - T_Ok(t_write_def_obj("x64.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("x64"), .flags = "rw:data" }, - {0} - } - })); - - // entry.obj - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - // arm64.obj - T_Ok(t_write_def_obj("arm64.obj", (T_COFF_DefObj){ - .machine = &(COFF_MachineType){ COFF_MachineType_Arm64 }, - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("arm64"), .flags = "rw:data" }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj"); - T_Ok(g_last_exit_code == 0); - - // test objs with conflicting machines - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj arm64.obj"); - T_Ok(g_last_exit_code != 0); - - // check /MACHINE switch - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe /machine:amd64 arm64.obj entry.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(simple_link_test) -{ - U8 text_payload[] = { 0xC3 }; - - String8 main_obj; - { - main_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_array_fixed(text_payload), .flags = "rx:code" }, - { "data", ".data", str8_lit("qwe"), .flags = "rw:data" }, - { "zero", ".zero", str8(0, 5), .flags = "rw:bss" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - }); - } - - String8 main_obj_name = str8_lit("main.obj"); - T_Ok(t_write_file(main_obj_name, main_obj)); - - int file_align = 512; - int virt_align = 4096; - String8 out_name = str8_lit("a.exe"); - t_invoke_linkerf("/entry:my_entry /subsystem:console /fixed /filealign:%d /align:%d /out:%S %S", file_align, virt_align, out_name, main_obj_name); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, out_name); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - T_Ok(!pe.is_pe32); - T_Ok(pe.section_count == 3); - T_Ok(pe.arch == Arch_x64); - T_Ok(pe.subsystem == PE_WindowsSubsystem_WINDOWS_CUI); - T_Ok(pe.virt_section_align == virt_align); - T_Ok(pe.file_section_align == file_align); - T_Ok(pe.symbol_count == 0); - T_Ok(pe.data_dir_count == PE_DataDirectoryIndex_COUNT); - - // check section alignment - for EachIndex(sect_idx, pe.section_count) { - COFF_SectionHeader *sect_header = §ion_table[sect_idx]; - T_Ok(AlignPadPow2(sect_header->fsize, file_align) == 0); - T_Ok(AlignPadPow2(sect_header->voff, virt_align) == 0); - } - - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_section != 0); - T_Ok(text_section->foff == file_align); - T_Ok(pe.entry_point == text_section->voff); - - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - - COFF_SectionHeader *zero_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".zero")); - T_Ok(zero_section != 0); - - String8 text_data = str8_substr(exe, rng_1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text_data, str8_array_fixed(text_payload), 0)); - - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - T_Ok(opt->sizeof_code == text_section->fsize); - T_Ok(opt->sizeof_inited_data == data_section->fsize); - T_Ok(opt->sizeof_uninited_data == 0x200); - T_Ok(opt->code_base == 0x1000); - T_Ok(opt->image_base == 0x140000000); - T_Ok(opt->major_os_ver == 6); - T_Ok(opt->minor_os_ver == 0); - T_Ok(opt->major_img_ver == 0); - T_Ok(opt->minor_img_ver == 0); - T_Ok(opt->major_subsystem_ver == 6); - T_Ok(opt->minor_subsystem_ver == 0); - T_Ok(opt->win32_version_value == 0); - T_Ok(opt->sizeof_image == 0x4000); - T_Ok(opt->sizeof_headers == 0x200); - T_Ok(opt->dll_characteristics == 0x8120); - T_Ok(opt->loader_flags == 0); -} - -TEST(map) -{ - T_Ok(t_write_def_obj("map.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("map_entry", "text", 0), - T_COFF_DefSymbol_Static("map_local", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/entry:map_entry /subsystem:console /out:mapped.exe /map:explicit.map map.obj"); - T_Ok(g_last_exit_code == 0); - - String8 map = t_read_file(arena, str8_lit("explicit.map")); - T_Ok(str8_find_needle(map, 0, str8_lit(" Timestamp is "), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Preferred load address is "), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Start Length Name Class"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Publics by Value"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit("map_entry"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" entry point at"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Static symbols"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit("map_local"), 0) < map.size); - - t_invoke_linkerf("/entry:map_entry /subsystem:console /out:default.exe /map map.obj"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_read_file(arena, str8_lit("default.map")).size > 0); - - t_invoke_linkerf("/entry:map_entry /subsystem:console /out:collision.exe /map:collision.exe map.obj"); - T_Ok(g_last_exit_code != 0); -} - - -TEST(out_of_bounds_section_number) -{ - // bad.obj - { - String8 obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "foo", ".foo", str8_lit("foo"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "foo", 0), - {0} - } - }); - { - COFF_FileHeaderInfo header = coff_file_header_info_from_data(obj); - String8 string_table = str8_substr(obj, header.string_table_range); - String8 symbol_table = str8_substr(obj, header.symbol_table_range); - COFF_ParsedSymbol symbol = coff_parse_symbol(header, string_table, symbol_table, 0); - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = 123; - } - T_Ok(t_write_file(str8_lit("bad.obj"), obj)); - } - - // entry.obj - { - String8 obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "foo"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0} - } - }); - T_Ok(t_write_file(str8_lit("entry.obj"), obj)); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj"); - T_Ok(g_last_exit_code != 0); -} - - -TEST(merge) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8_lit("hello, world"), .flags = "rw:data" }, - {0} - } - })); - - T_Ok(t_write_entry_obj()); - - // circular merge - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.test entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_CircularMerge); - } - - // circular merge with extra link - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.data /merge:.data=.test entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_CircularMerge); - } - - // merge with non-defined section - { - g_last_exit_code; - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.qwe entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - // make sure linker created .qwe and merged .test into it - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".qwe")); - T_Ok(sect != 0); - T_Ok(sect->flags == PE_DATA_SECTION_FLAGS); - String8 qwe = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(qwe, str8_lit("hello, world"),0)); - } - - // illegal merge with .reloc - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.reloc entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } - - // illegal merge with .rsrc - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.rsrc entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } - - // merge non-defined section with defined section - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.qwe=.test entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - // initialized data size includes the aligned virtual tail of merged BSS - { - T_Ok(t_write_def_obj("mixed.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit_comp("d"), .flags = "rw:data@1" }, - { "bss", ".bss", str8(0, 0x201), .flags = "rw:bss@1" }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.bss=.data entry.obj mixed.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - COFF_SectionHeader *data = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data != 0); - T_Ok(data->fsize == 0x200); - T_Ok(data->vsize == 0x202); - T_Ok(opt->sizeof_inited_data == 0x400); - } - - // merged contribution groups retain lexical order - { - T_Ok(t_write_def_obj("order.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a$m", str8_lit_comp("a"), .flags = "rw:data@1" }, - { "z", ".z$m", str8_lit_comp("z"), .flags = "rw:data@1" }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.a=.z entry.obj order.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".z")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + 2)); - T_Ok(str8_match(data, str8_lit("az"), 0)); - } - - // merge .test -> .qwe -> .data - { - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.qwe /merge:.qwe=.data entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - // make sure linker merged .test into .data - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".data")); - T_Ok(sect != 0); - T_Ok(sect->flags == PE_DATA_SECTION_FLAGS); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("hello, world"),0)); - } -} - -TEST(section_directive_read_only_grouped_section) -{ - T_Ok(t_write_entry_obj()); - - T_Ok(t_write_def_obj("prot.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "prot_a", "prot$a", str8_lit_comp("A"), .flags = "rw:data@1" }, - { "prot_mem", "prot$mem", str8_lit_comp("mem"), .flags = "rw:data@1" }, - { "prot_z", "prot$z", str8_lit_comp("Z"), .flags = "rw:data@1" }, - {0} - }, - .directives = (char *[]){ "/SECTION:prot,R", 0 }, - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj prot.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("prot")); - T_Ok(sect != 0); - T_Ok(sect->flags == (COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead)); -} - -TEST(section_directive_align_grouped_section) -{ - if (t_id_linker() != Linker_radlink) { return; } - - T_Ok(t_write_entry_obj()); - - T_Ok(t_write_def_obj("prot.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "prot_mem", "prot$mem", str8_lit_comp("mem"), .flags = "rw:data@1" }, - {0} - }, - .directives = (char *[]){ "/SECTION:prot,R,ALIGN=8192", 0 }, - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj prot.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("prot")); - T_Ok(sect != 0); - T_Ok(sect->flags == (COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_Align8192Bytes)); -} - -TEST(link_undef) -{ - T_Ok(t_write_def_obj("undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("undef"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "undef"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("undef"), - {0} - } - })); - - // try linking unresolved symbol and see if linker picks up on that - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj"); - T_Ok(g_last_exit_code == LNK_Error_UnresolvedSymbol); -} - -TEST(link_unref_undef) -{ - T_Ok(t_write_def_obj("undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("undef"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - // try linking unreferenced unresolved symbol, this must link - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(weak_lib_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); - - // linker must pick weak symbol from a.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x111; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_lib_vs_weak_nolib) +internal String8 +data_from_pdb(Arena *arena, PDB_Context *pdb) { - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} + TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("foo")); + TP_Arena *tp_arena = tp_arena_alloc(tp); + pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0); -TEST(weak_lib_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "q"), - {0} - } - })); + AssertAlways(msf_build(pdb->msf) == MSF_Error_OK); + String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf); + AssertAlways(t_write_file_list(str8_lit("test.pdb"), raw_msf_list)); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); + String8 data = str8_list_join(arena, &raw_msf_list, 0); - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} + tp_arena_release(&tp_arena); + tp_release(tp); -TEST(weak_lib_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); - - // linker must pick weak symbol from a.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_alias_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("qwe", 0x111), - T_COFF_DefSymbol_Weak("sym", COFF_WeakExt_SearchAlias, "qwe"), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("ewq", 0x222), - T_COFF_DefSymbol_Weak("sym", COFF_WeakExt_SearchAlias, "ewq"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "sym"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("sym"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} - -TEST(weak_alias_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_alias_vs_weak_nolib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_alias_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_nolib_vs_weak_nolib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_nolib_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_nolib_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} - -TEST(weak_nolib_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - // linker must pick weak symbol from a.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x111; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_nolib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x111; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_vs_common) -{ - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "a", ".a", str8_lit("a"), .flags = "rw:data" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Static("_a", "a", 0), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "_a"), - {0} - } - })); - T_Ok(t_write_def_obj("common.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Common("w", 2), {0} } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "w"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("w"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe common.obj weak.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe weak.obj common.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *bss = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".bss")); - T_Ok(bss); - T_Ok(bss->fsize == 0); - T_Ok(bss->vsize == 2); -} - -TEST(abs_vs_weak) -{ - U32 abs_value = 0x123; - U8 text_code[] = { 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3 }; - - T_Ok(t_write_def_obj("abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("foo", abs_value), - {0}, - } - })); - - T_Ok(t_write_def_obj("text.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "mydata", ".mydata", str8_lit("mydata"), .flags = "rx:code@1" }, - { - "text", ".text", str8_array_fixed(text_code), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 2, "foo"), - {0}, - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("mydata", "mydata", 0), - T_COFF_DefSymbol_Weak("foo", COFF_WeakExt_NoLibrary, "mydata"), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe abs.obj text.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe text.obj abs.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_section != 0); - - String8 text_data = str8_substr(exe, rng_1u64(text_section->foff, text_section->foff + text_section->fsize)); - String8 inst = str8_prefix(text_data, 2); - T_Ok(str8_match(inst, str8_array(text_code, 2), 0)); - - String8 imm = str8_prefix(str8_skip(text_data, 2), 8); - U64 expected_imm = abs_value; - T_Ok(str8_match(imm, str8_struct(&expected_imm), 0)); -} - -TEST(abs_vs_regular) -{ - String8 shared_symbol_name = str8_lit("foo"); - - U8 regular_payload[] = { 0xC0, 0xFF, 0xEE }; - String8 regular_obj_name = str8_lit("regular.obj"); - - T_Ok(t_write_def_obj("regular.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "data", ".data", str8_array_fixed(regular_payload), .flags = "rw:data" }, {0}, }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("foo", "data", 0), {0}, } - })); - - String8 abs_obj_name = str8_lit("abs.obj"); - T_Ok(t_write_def_obj("abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 0x1234), {0}, } - })); - - U8 entry_text[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, // mov rax, $imm - 0xC3 // ret - }; - String8 entry_obj_name = str8_lit("entry.obj"); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "foo"), - {0}, - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0}, - } - })); - - // TODO: validate that linker issues multiply defined symbol error - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe abs.obj regular.obj entry.obj"); - // linker should complain about multiply defined symbol - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe regular.obj abs.obj entry.obj"); - // linker should complain even in case regular is before abs - T_Ok(g_last_exit_code != 0); -} - -TEST(abs_vs_common) -{ - String8 shared_symbol_name = str8_lit("foo"); - - String8 common_obj_name = str8_lit("common.obj"); - - T_Ok(t_write_def_obj("common.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Common("foo", 321), {0}, } - })); - - String8 abs_obj_name = str8_lit("abs.obj"); - T_Ok(t_write_def_obj("abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 0x1234), {0}, } - })); - - U8 entry_text[] = { 0xC3 }; - String8 entry_obj_name = str8_lit("entry.obj"); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, {0}, }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("my_entry", "text", 0), {0}, } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe abs.obj common.obj entry.obj"); - if (g_last_exit_code == 0) { - // TODO: validate that linker issues multiply defined symbol error - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe common.obj abs.obj entry.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); - } else { - T_Ok(g_last_exit_code != 0); - } - } -} - -TEST(abs_vs_abs) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 'a'), {0}, }, - })); - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 'b'), {0}, }, - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} - -TEST(undef_weak_lib) -{ - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("b", 0xc3000000), - T_COFF_DefSymbol_Weak("a", COFF_WeakExt_SearchLibrary, "b"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "a"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("a"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0}, - } - })); - - // undefined symbol must always replace weak symbol with search library - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:entry entry.obj weak.obj"); - T_Ok(g_last_exit_code == LNK_Error_UnresolvedSymbol); -} - -TEST(undef_weak_search_alias) -{ - Temp scratch = scratch_begin(0,0); - - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "data", ".data", str8_lit_comp("\xde\xad\xbe\xef"), .flags = "rw:data" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("ptr"), - T_COFF_DefSymbol_Weak("foo", COFF_WeakExt_SearchAlias, "ptr"), - {0} - } - })); - - T_Ok(t_write_def_obj("ptr.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("entry"), - T_COFF_DefSymbol_Weak("ptr", COFF_WeakExt_SearchAlias, "entry"), - {0} - } - })); - - T_Ok(t_write_def_obj("undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "foo"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Undef("foo"), {0} } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("entry", "text", 0), {0} } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe weak.obj entry.obj ptr.obj undef.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(weak_cycle) -{ - String8 ab_obj_name = str8_lit("ab.obj"); - - T_Ok(t_write_def_obj("ab.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("B"), - T_COFF_DefSymbol_Weak("A", COFF_WeakExt_SearchAlias, "B"), - {0}, - } - })); - - String8 ba_obj_name = str8_lit("ba.obj"); - T_Ok(t_write_def_obj("ba.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("A"), - T_COFF_DefSymbol_Weak("B", COFF_WeakExt_SearchAlias, "A"), - {0}, - } - })); - - String8 entry_obj_name = str8_lit("entry.obj"); - U8 entry_payload[] = { 0xC3 }; - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_array_fixed(entry_payload), .flags = "rx:code@1" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("my_entry", "text", 0), {0} } - })); - - U64 timeout = now_time_us() + 3 * 1000 * 1000; // give a generous 3 seconds - t_invoke_linker_timeoutf(timeout, "/subsystem:console /entry:my_entry %S %S %S", entry_obj_name, ab_obj_name, ba_obj_name); -} - -TEST(weak_tag) -{ - U32 weak_tag_expected_value = 0x12345678; - String8 weak_tag_obj_name = str8_lit("weak_tag.obj"); - - T_Ok(t_write_def_obj("weak_tag.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32, 0, "strong_second"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsStatic("abs", weak_tag_expected_value), - T_COFF_DefSymbol_Weak("strong_first", COFF_WeakExt_SearchAlias, "abs"), - T_COFF_DefSymbol_Weak("strong_second", COFF_WeakExt_SearchAlias, "strong_first"), - {0} - } - })); - - String8 entry_name = str8_lit("my_entry"); - U8 entry_text[] = { 0xC3 }; - String8 entry_obj_name = str8_lit("entry.obj"); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern((char *)entry_name.str, "text", 0), {0} } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe %S %S", weak_tag_obj_name, entry_obj_name); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - String8 data = str8_substr(exe, rng_1u64(data_section->foff, data_section->foff + data_section->vsize)); - T_Ok(data_section); - T_Ok(data_section->vsize == 4); - T_Ok(str8_match(data, str8_struct(&weak_tag_expected_value), 0)); -} - -TEST(undef_section) -{ - U8 payload[] = { 1, 2, 3 }; - String8 sec_defn_obj = t_make_sec_defn_obj(arena, str8_array_fixed(payload)); - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, ".mysect"), - {0} - } - }, - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_UndefSec(".mysect", COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - t_write_file(str8_lit("sec_defn.obj"), sec_defn_obj); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj sec_defn.obj"); - if (g_last_exit_code == 0) { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *mysect_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".mysect")); - if (data_section && mysect_section) { - if (data_section->vsize == 4 && mysect_section->vsize == 3) { - String8 addr32nb = str8_substr(exe, rng_1u64(data_section->foff, data_section->foff + data_section->vsize)); - String8 expected_voff = str8_struct(&mysect_section->voff); - T_Ok(str8_match(addr32nb, expected_voff, 0)); - } - } - } -} - -TEST(sect_symbol) -{ - String8 sect_payload = str8_lit("hello, world"); - T_Ok(t_write_def_obj("sect.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "mysect1", ".mysect$1", sect_payload, .flags = "rw:data@1" }, - {0} - }, - .directives = (char *[]){ "/merge:.mysect=.data", 0 } - })); - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, ".mysect$2222"), - {0} - } - }, - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_UndefSec(".mysect$2222", t_coff_section_flags_from_cstr("rw:data")), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj sect.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - - T_Ok(sect != 0); - - String8 sect_data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - - String8 addr_data = str8_substr(sect_data, rng_1u64(0, sizeof(U64))); - T_Ok(addr_data.size == sizeof(U64)); - - U64 addr = *(U64 *)addr_data.str; - T_Ok(addr - (pe.image_base + sect->voff) == 8); - - String8 payload_got = str8_substr(sect_data, rng_1u64(8, sect_data.size)); - T_Ok(str8_match(payload_got, sect_payload, 0)); -} - -TEST(undef_reloc_section) -{ - U8 payload[] = { 1, 2, 3 }; - String8 sec_defn_obj = t_make_sec_defn_obj(arena, str8_array_fixed(payload)); - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code" }, - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, ".reloc"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - T_COFF_DefSymbol_UndefSec(".reloc", PE_RELOC_SECTION_FLAGS), - {0} - } - })); - T_Ok(t_write_file(str8_lit("sec_defn.obj"), sec_defn_obj)); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj sec_defn.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_SectRefsDiscardedMemory); - } else { - T_Ok(g_last_exit_code != 0); - } -} - -TEST(find_merged_pdata) -{ - U8 foobar_payload[] = { - 0x40, 0x57, 0x48, 0x81, 0xEC, 0x00, 0x02, 0x00, 0x00, 0x48, 0x8B, 0x05, 0x00, 0x00, 0x00, 0x00, - 0x48, 0x33, 0xC4, 0x48, 0x89, 0x84, 0x24, 0xF0, 0x01, 0x00, 0x00, 0x48, 0x8D, 0x04, 0x24, 0x48, - 0x8B, 0xF8, 0x33, 0xC0, 0xB9, 0xEC, 0x01, 0x00, 0x00, 0xF3, 0xAA, 0xB8, 0x04, 0x00, 0x00, 0x00, - 0x48, 0x6B, 0xC0, 0x02, 0x8B, 0x04, 0x04, 0x48, 0x8B, 0x8C, 0x24, 0xF0, 0x01, 0x00, 0x00, 0x48, - 0x33, 0xCC, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x48, 0x81, 0xC4, 0x00, 0x02, 0x00, 0x00, 0x5F, 0xC3, - 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, - 0x48, 0x83, 0xEC, 0x28, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x48, 0x83, 0xC4, 0x28, 0xC3 - }; - U8 xdata_payload[] = { - 0x19, 0x1B, 0x03, 0x00, 0x09, 0x01, 0x40, 0x00, 0x02, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xF0, 0x01, 0x00, 0x00, 0x01, 0x04, 0x01, 0x00, 0x04, 0x42, 0x00, 0x00 - }; - PE_IntelPdata intel_pdata = {0}; - U8 text_payload[] = { 0xC3 }; - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "xdata", ".xdata", str8_array_fixed(xdata_payload), .flags = "r:data@4" }, - { - "pdata", ".pdata", str8_struct(&intel_pdata), .flags = "r:data@4", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$foobar"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "foobar"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "foobar"), - {0} - } - }, - { "foobar", ".foobar", str8_array_fixed(foobar_payload), .flags = "rx:code@1" }, - { "text", ".text", str8_array_fixed(text_payload), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Static("foobar", "foobar", 0), - T_COFF_DefSymbol_Secdef("xdata", COFF_ComdatSelect_Null), - T_COFF_DefSymbol_Static("$unwind$foobar", "xdata", 0), - T_COFF_DefSymbol_Secdef("pdata", COFF_ComdatSelect_Null), - T_COFF_DefSymbol_Static("$pdata$foobar", "pdata", 0), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj /merge:.pdata=.rdata"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - T_Ok(dim_1u64(pe.data_dir_franges[PE_DataDirectoryIndex_EXCEPTIONS]) == 0xC); -} - -TEST(guard_cf_pulls_load_config) -{ - U8 load_config_data[0x40] = {0}; - U32 load_config_size = sizeof(load_config_data); - MemoryCopy(load_config_data, &load_config_size, sizeof(load_config_size)); - - T_Ok(t_write_entry_obj()); - T_Ok(t_write_def_lib("loadcfg.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("loadcfg.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "loadcfg", ".rdata", str8_array_fixed(load_config_data), .flags = "r:data@8" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("_load_config_used", "loadcfg", 0), - {0} - } - } - }, - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe /guard:cf entry.obj loadcfg.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - T_Ok(dim_1u64(pe.data_dir_franges[PE_DataDirectoryIndex_LOAD_CONFIG]) == load_config_size); - - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - if (t_id_linker() == Linker_radlink) { - T_Ok(!(opt->dll_characteristics & PE_DllCharacteristic_GUARD_CF)); - } -} - -TEST(section_sort) -{ - COFF_SectionFlags data_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemRead|COFF_SectionFlag_Align1Bytes; - T_Ok(t_write_def_obj("data.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "idata_2", ".idata$2", str8_lit("last"), .flags = "rw:data@1" }, - { "idata_5", ".idata$5", str8_lit("first"), .flags = "rw:data@1" }, - { "rdata", ".rdata", str8_lit("middle"), .flags = "r:data@1" }, - { "data_z", ".data$z", str8_lit("five"), .raw_flags = data_flags }, - { "data_a", ".data$a", str8_lit("three"), .raw_flags = data_flags }, - { "data_bbbbb", ".data$bbbbb", str8_lit("four"), .raw_flags = data_flags }, - { "data_empty", ".data$", str8_lit("two"), .raw_flags = data_flags }, - { "data", ".data", str8_lit("one"), .raw_flags = data_flags }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe data.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section); - - String8 data = str8_substr(exe, rng_1u64(data_section->foff, data_section->foff + data_section->vsize)); - String8 expected_data = str8_lit("onetwothreefourfive"); - T_Ok(str8_match(data, expected_data, 0)); - - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - T_Ok(rdata_section); - - String8 rdata = str8_substr(exe, rng_1u64(rdata_section->foff, rdata_section->foff + 15)); - T_Ok(str8_match(rdata, str8_lit("firstmiddlelast"), 0)); -} - -TEST(flag_conf) -{ - COFF_SectionFlags my_sect0_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemExecute; - COFF_SectionFlags my_sect1_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemWrite; - T_Ok(t_write_def_obj("conf.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".mysect", str8_lit("one"), .raw_flags = my_sect0_flags }, - { "b", ".mysect", str8_lit("two"), .raw_flags = my_sect1_flags }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe conf.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeaderArray my_sects = coff_section_header_array_from_name(arena, string_table, section_table, pe.section_count, str8_lit(".mysect")); - T_Ok(my_sects.count == 2); - - COFF_SectionHeader *my_sect0 = &my_sects.v[0]; - COFF_SectionHeader *my_sect1 = &my_sects.v[1]; - T_Ok(my_sect0->flags == my_sect0_flags); - T_Ok(my_sect1->flags == my_sect1_flags); -} - -TEST(invalid_bss) -{ - COFF_SectionFlags bss_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead; - String8 bss_data = str8_lit("Hello, World"); - T_Ok(t_write_def_obj("bss.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "bss", ".bss", bss_data, .raw_flags = bss_flags }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe bss.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *bss_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".bss")); - T_Ok(bss_sect != 0); - T_Ok(bss_sect->vsize == 0xC); - T_Ok(bss_sect->flags == bss_flags); - String8 data = str8_substr(exe, rng_1u64(bss_sect->foff, bss_sect->foff + bss_sect->vsize)); - T_Ok(str8_match(data, bss_data, 0)); -} - -TEST(common_block) -{ - U8 a_data[6] = {0}; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_array_fixed(a_data), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 0, "A"), - {0} - } - }, - { "bss", ".bss", str8(0, 1), .flags = "rw:bss" }, // shift common block's initial position - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("A", 3), - {0} - } - })); - - U8 b_data[9] = { 0 }; - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_array_fixed(b_data), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "B"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("B", 6), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe /fixed /largeaddressaware:no /merge:.bss=.comm a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *comm_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".comm")); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(comm_sect != 0); - T_Ok(data_sect != 0); - - // blocks must be sorted in descending order to reduce alignment padding - T_Ok(comm_sect->vsize == 0x13); - - // ensure linker correctly patched addresses for symbols pointing into common block - String8 data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->fsize)); - U32 *a_addr = (U32 *)data.str; - U64 *b_addr = (U64 *)(data.str + sizeof(a_data)); - T_Ok(*a_addr == (pe.image_base + comm_sect->voff + 0x10)); - T_Ok(*b_addr == (pe.image_base + comm_sect->voff + 0x8)); -} - -TEST(base_relocs) -{ - // main.obj - String8 entry_name = str8_lit("my_entry"); - U64 mov_func_name64 = 2; - U64 mov_func_name32 = 16; - U8 main_text[] = { - 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov rax, func_name - 0xff, 0xd0, // call rax - 0x48, 0x31, 0xc0, // xor rax, rax - 0xb8, 0x00, 0x00, 0x00, 0x00, // mov eax, func_name - 0xff, 0xd0, // call rax - 0xc3 // ret - }; - - // func.obj - String8 func_name = str8_lit("foo"); - U8 func_text[] = { 0xc3 }; - - // linker must not produce base relocations for absolute symbol - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(main_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, mov_func_name64, "foo"), - T_COFF_DefReloc(X64_Addr32, mov_func_name32, "foo"), - {0} - } - }, - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 0, "abs"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("foo"), - T_COFF_DefSymbol_AbsStatic("abs", 0x12345678), - T_COFF_DefSymbol_Extern((char *)entry_name.str, "text", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("func.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_array_fixed(func_text), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern((char *)func_name.str, "text", 0), - {0} - } - })); - - String8 out_name = str8_lit("a.exe"); - t_invoke_linkerf("/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe main.obj func.obj"); - T_Ok(g_last_exit_code == 0); - - // it is illegal to merge .reloc with other sections - t_invoke_linkerf("/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.reloc=.rdata main.obj func.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } else { - T_Ok(g_last_exit_code != 0); - } - - // the other way around is illegal too - t_invoke_linkerf("/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.rdata=.reloc main.obj func.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } else { - T_Ok(g_last_exit_code != 0); - } -} - -TEST(simple_lib_test) -{ - String8 test_payload = str8_lit("The quick brown fox jumps over the lazy dog"); - - T_Ok(t_write_def_lib("test.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("test.obj"), - .machine = T_COFF_DefSetMachine(Unknown), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8(test_payload.str, test_payload.size+1), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("test", "data", 0), - {0} - } - } - }, - {0} - } - })); - - U8 entry_text[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, - 0xC3 - }; - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "test"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("test"), - T_COFF_DefSymbol_Extern("my_entry", "text", 7), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe entry.obj test.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->fsize)); - String8 data_data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->fsize)); - - // was test payload linked? - String8 data_string = str8_cstring_capped(data_data.str, data_data.str + data_data.size); - T_Ok(str8_match(data_string, test_payload, 0)); - - // do we have enough bytes to read text? - T_Ok(text_data.size >= sizeof(entry_text)); - - // linker must pull-in test.obj and patch relocation for "test" symbol - U32 *data_addr32nb = (U32 *)(text_data.str+3); - T_Ok(*data_addr32nb == data_sect->voff); -} - -TEST(lib_member_imp_and_regular_symbol_queued_once) -{ - T_Ok(t_write_def_lib("rust_style.rlib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("core-9f9efb2036858c45.core.78298229696da45f-cgu.0.rcgu.o"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code@1" }, - { "idata", ".idata", str8_lit_comp("\x00\x00\x00\x00\x00\x00\x00\x00"), .flags = "r:data@8" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("foo", "text", 0), - T_COFF_DefSymbol_Extern("__imp_foo", "idata", 0), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\x48\xC7\xC1\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "foo"), - T_COFF_DefReloc(X64_Addr32Nb, 10, "__imp_foo"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - T_COFF_DefSymbol_Undef("__imp_foo"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj rust_style.rlib"); - T_Ok(g_last_exit_code == 0); -} - -#if OS_WINDOWS -TEST(import_export) -{ - // write objs - T_Ok(t_write_def_obj("import.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - .id = "data", - .name = ".data", - .flags = "rw:data", - .data = str8_array_fixed((U8[1024]){0}), - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0*4, "__imp_foo"), - T_COFF_DefReloc(X64_Addr32Nb, 1*4, "__imp_bar"), - T_COFF_DefReloc(X64_Addr32Nb, 2*4, "__imp_baz"), - T_COFF_DefReloc(X64_Addr32Nb, 3*4, "__imp_baf"), - T_COFF_DefReloc(X64_Addr32Nb, 4*4, "__imp_ord"), - T_COFF_DefReloc(X64_Addr32Nb, 5*4, "bar"), - T_COFF_DefReloc(X64_Addr32Nb, 6*4, "foo"), - T_COFF_DefReloc(X64_Addr32Nb, 7*4, "ord"), - {0}, - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__imp_foo"), - T_COFF_DefSymbol_Undef("__imp_bar"), - T_COFF_DefSymbol_Undef("__imp_baz"), - T_COFF_DefSymbol_Undef("__imp_baf"), - T_COFF_DefSymbol_Undef("__imp_ord"), - T_COFF_DefSymbol_Undef("bar"), - T_COFF_DefSymbol_Undef("foo"), - T_COFF_DefSymbol_Undef("ord"), - //"baf", - //"baz", - //"__imp_ord2", - //"__imp_ord4", - {0}, - }, - })); - - T_Ok(t_write_def_obj("export.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "data", 0), - T_COFF_DefSymbol_Extern("ord", "data", 1), - T_COFF_DefSymbol_Extern("ord2", "data", 2), - T_COFF_DefSymbol_Extern("ord3", "data", 9), - T_COFF_DefSymbol_Extern("ord4", "data", 10), - {0} - }, - .directives = (char*[]){ - "/export:foo=foo", - "/export:bar=foo", - "/export:ord,@5", - "/export:ord2,@6,DATA", - "/export:ord3,@7,NONAME,PRIVATE", - "/export:ord4,@8,NONAME,DATA", - "/export:baz=BAZ.qwe", - "/export:baf=BAZ.#1", - 0, - } - })); - - T_Ok(t_write_def_obj("baz.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "s1", ".s1", str8_lit("s1"), .flags = "rw:data" }, - { "s2", ".s2", str8_lit("s2"), .flags = "rw:data" }, - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("_DllMainCRTStartup", "text", 0), - T_COFF_DefSymbol_Extern("s1", "s1", 0), - T_COFF_DefSymbol_Extern("s2", "s2", 0), - {0} - }, - .directives = (char*[]){ - "/export:baf=s1", - "/export:baz=s2", - 0, - } - })); - - T_Ok(t_write_entry_obj()); - - // link dlls - t_invoke_linkerf("/dll /out:export.dll libcmt.lib export.obj"); // export.dll - T_Ok(g_last_exit_code == 0); - t_invoke_linkerf("/dll /out:baz.dll /export:s1,@1,NONAME /export:qwe=s2 baz.obj"); // baz.dll - T_Ok(g_last_exit_code == 0); - - // validate export table in export.dll - if (t_id_linker() == Linker_radlink) { - // validate export table in export.dll - { - String8 dll = t_read_file(arena, str8_lit("export.dll")); - PE_BinInfo pe = pe_bin_info_from_data(arena, dll); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(dll, pe.section_table_range).str; - PE_ParsedExportTable export_table = pe_exports_from_data(arena, pe.section_count, section_table, dll, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - COFF_SectionHeader *data_sect = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".data")); - - // validate header - T_Ok(export_table.flags == 0); - T_Ok(export_table.time_stamp == COFF_TimeStamp_Max); - T_Ok(export_table.major_ver == 0); - T_Ok(export_table.minor_ver == 0); - T_Ok(export_table.ordinal_base == 5); - T_Ok(export_table.export_count == 8); - - // validate names - T_Ok(str8_match(export_table.exports[0].name, str8_lit("baf"), 0)); - T_Ok(str8_match(export_table.exports[1].name, str8_lit("bar"), 0)); - T_Ok(str8_match(export_table.exports[2].name, str8_lit("baz"), 0)); - T_Ok(str8_match(export_table.exports[3].name, str8_lit("foo"), 0)); - T_Ok(str8_match(export_table.exports[4].name, str8_lit("ord"), 0)); - T_Ok(str8_match(export_table.exports[5].name, str8_lit("ord2"), 0)); - T_Ok(export_table.exports[6].name.size == 0); - T_Ok(export_table.exports[7].name.size == 0); - - // validate forwarders - T_Ok(str8_match(export_table.exports[0].forwarder, str8_lit("BAZ.#1"), 0)); - T_Ok(export_table.exports[1].forwarder.size == 0); - T_Ok(str8_match(export_table.exports[2].forwarder, str8_lit("BAZ.qwe"), 0)); - T_Ok(export_table.exports[3].forwarder.size == 0); - T_Ok(export_table.exports[4].forwarder.size == 0); - T_Ok(export_table.exports[5].forwarder.size == 0); - T_Ok(export_table.exports[6].forwarder.size == 0); - T_Ok(export_table.exports[7].forwarder.size == 0); - - // validate voffs - T_Ok(export_table.exports[1].voff == data_sect->voff + 0x0); - T_Ok(export_table.exports[3].voff == data_sect->voff + 0x0); - T_Ok(export_table.exports[4].voff == data_sect->voff + 0x1); - T_Ok(export_table.exports[5].voff == data_sect->voff + 0x2); - T_Ok(export_table.exports[6].voff == data_sect->voff + 0x9); - T_Ok(export_table.exports[7].voff == data_sect->voff + 0xa); - - // validate ordinals - T_Ok(export_table.exports[0].ordinal == 9); - T_Ok(export_table.exports[1].ordinal == 10); - T_Ok(export_table.exports[2].ordinal == 11); - T_Ok(export_table.exports[3].ordinal == 12); - T_Ok(export_table.exports[4].ordinal == 5); - T_Ok(export_table.exports[5].ordinal == 6); - T_Ok(export_table.exports[6].ordinal == 7); - T_Ok(export_table.exports[7].ordinal == 8); - } - - // validate export table in baz.dll - { - String8 dll = t_read_file(arena, str8_lit("baz.dll")); - PE_BinInfo pe = pe_bin_info_from_data(arena, dll); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(dll, pe.section_table_range).str; - PE_ParsedExportTable export_table = pe_exports_from_data(arena, pe.section_count, section_table, dll, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - - // validate header - T_Ok(export_table.flags == 0); - T_Ok(export_table.time_stamp == COFF_TimeStamp_Max); - T_Ok(export_table.major_ver == 0); - T_Ok(export_table.minor_ver == 0); - T_Ok(export_table.ordinal_base == 1); - T_Ok(export_table.export_count == 4); - - // validate names - T_Ok(str8_match(export_table.exports[0].name, str8_lit("baf"), 0)); - T_Ok(str8_match(export_table.exports[1].name, str8_lit("baz"), 0)); - T_Ok(str8_match(export_table.exports[2].name, str8_lit("qwe"), 0)); - T_Ok(str8_match(export_table.exports[3].name, str8_zero(), 0)); - - // validate forwarders - T_Ok(str8_match(export_table.exports[0].forwarder, str8_zero(), 0)); - T_Ok(str8_match(export_table.exports[1].forwarder, str8_zero(), 0)); - T_Ok(str8_match(export_table.exports[2].forwarder, str8_zero(), 0)); - T_Ok(str8_match(export_table.exports[3].forwarder, str8_zero(), 0)); - - // validate voffs - T_Ok(export_table.exports[0].voff == 0x3000); - T_Ok(export_table.exports[1].voff == 0x4000); - T_Ok(export_table.exports[2].voff == 0x4000); - T_Ok(export_table.exports[3].voff == 0x3000); - - // validate ordinals - T_Ok(export_table.exports[0].ordinal == 2); - T_Ok(export_table.exports[1].ordinal == 3); - T_Ok(export_table.exports[2].ordinal == 4); - T_Ok(export_table.exports[3].ordinal == 1); - } - } - - { - T_Ok(SetDllDirectoryA((LPCSTR)g_wdir.str)); - HANDLE export_dll = LoadLibrary("export.dll"); - T_Ok(export_dll); - - // test query by function name - //T_Ok(GetProcAddress(export_dll, "baf")); - T_Ok(GetProcAddress(export_dll, "bar")); - //T_Ok(GetProcAddress(export_dll, "baz")); - T_Ok(GetProcAddress(export_dll, "foo")); - T_Ok(GetProcAddress(export_dll, "ord")); - T_Ok(GetProcAddress(export_dll, "ord2")); - - // test query by ordinal - //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(9))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(10))); - //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(11))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(12))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(5))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(6))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(7))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(8))); - } - - //T_Ok(t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /delayload:export.dll /export:entry kernel32.Lib delayimp.lib libcmt.lib export.lib import.obj entry.obj") == 0); - // TODO: check import table -} - -TEST(def_file_full) -{ - if (t_id_linker() == Linker_lld) { return; } - - T_Ok(t_write_def_obj("def_full.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_lit("test"), .flags = "rw:data" }, - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - }, - })); - - T_Ok(t_write_file(str8_lit("full.def"), str8_lit( - "; leading comment\n" - "NAME \"def full.exe\" BASE=0x140020000\n" - "VERSION 7.8\n" - "HEAPSIZE 0x30000, 0x4000\n" - "STACKSIZE 0x50000,0x6000\n" - "SECTIONS .rdata READ\n" - "EXPORTS foo @ 2 DATA\n"))); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:full.def def_full.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("def full.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - - T_Ok(opt->image_base == 0x140020000); - T_Ok(opt->major_img_ver == 7); - T_Ok(opt->minor_img_ver == 8); - T_Ok(opt->sizeof_heap_reserve == 0x30000); - T_Ok(opt->sizeof_heap_commit == 0x4000); - T_Ok(opt->sizeof_stack_reserve == 0x50000); - T_Ok(opt->sizeof_stack_commit == 0x6000); - - COFF_SectionHeader *rdata = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - T_Ok(rdata != 0); - T_Ok(rdata->flags == (COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead)); - - PE_ParsedExportTable export_table = pe_exports_from_data(arena, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - T_Ok(export_table.export_count == 1); - T_Ok(str8_match(export_table.exports[0].name, str8_lit("foo"), 0)); - T_Ok(export_table.exports[0].ordinal == 2); - - T_Ok(t_write_file(str8_lit("bad_base_space.def"), str8_lit( - "NAME bad_base_space BASE = 0x140020000\n" - "EXPORTS foo @2 DATA\n"))); - T_Ok(t_write_file(str8_lit("bad_base_colon.def"), str8_lit( - "NAME bad_base_colon BASE:0x140020000\n" - "EXPORTS foo @2 DATA\n"))); - T_Ok(t_write_file(str8_lit("bad_section_align.def"), str8_lit( - "NAME bad_section_align\n" - "SECTIONS .rdata READ ALIGN=8192\n" - "EXPORTS foo @2 DATA\n"))); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:bad_base_space.def /out:bad_base_space.exe def_full.obj"); - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:bad_base_colon.def /out:bad_base_colon.exe def_full.obj"); - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:bad_section_align.def /out:bad_section_align.exe def_full.obj"); - T_Ok(g_last_exit_code != 0); - - T_Ok(t_write_def_obj("def_full_dll.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("_DllMainCRTStartup", "text", 0), - T_COFF_DefSymbol_Extern("dll_foo", "data", 0), - {0} - }, - })); - - T_Ok(t_write_file(str8_lit("full_dll.def"), str8_lit( - "LIBRARY folded BASE=0x180020000\n" - "EXPORTS\n" - " dll_foo DATA\n"))); - - t_invoke_linkerf("/dll /subsystem:console /def:full_dll.def def_full_dll.obj"); - T_Ok(g_last_exit_code == 0); - - String8 dll = t_read_file(arena, str8_lit("folded.dll")); - PE_BinInfo dll_pe = pe_bin_info_from_data(arena, dll); - COFF_SectionHeader *dll_section_table = (COFF_SectionHeader *)str8_substr(dll, dll_pe.section_table_range).str; - PE_OptionalHeader32Plus *dll_opt = str8_deserial_get_raw_ptr(dll, dll_pe.optional_header_off, sizeof(*dll_opt)); - PE_ParsedExportTable dll_export_table = pe_exports_from_data(arena, dll_pe.section_count, dll_section_table, dll, dll_pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], dll_pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - - T_Ok(dll_opt->image_base == 0x180020000); - T_Ok(dll_export_table.export_count == 1); - T_Ok(str8_match(dll_export_table.exports[0].name, str8_lit("dll_foo"), 0)); -} - -TEST(utf16_rsp) -{ - T_Ok(t_write_entry_obj()); - - String8 rsp_text = str8_lit("/subsystem:console /entry:entry /out:a.exe entry.obj\n"); - String16 rsp16 = str16_from_8(arena, rsp_text); - String8 rsp_file = str8_cat(arena, str8_lit("\xff\xfe"), str8_array(rsp16.str, rsp16.size)); - T_Ok(t_write_file(str8_lit("args.rsp"), rsp_file)); - - t_invoke_linkerf("@args.rsp"); - T_Ok(g_last_exit_code == 0); -} -#endif - -TEST(image_base) -{ - T_Ok(t_write_def_obj("image_base.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", - ".text", - str8_lit_comp( - "\x48\x8D\x0D\x00\x00\x00\x00" // lea rcx, [__ImageBase] - "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x00" // mov rax, __ImageBase - "\xB8\x00\x00\x00\x00" // mov eax, __ImageBase - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "__ImageBase"), - T_COFF_DefReloc(X64_Addr64, 9, "__ImageBase"), - T_COFF_DefReloc(X64_Addr32Nb, 18, "__ImageBase"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__ImageBase"), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /base:0x2000000140000000 /out:a.exe image_base.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_section); - - U8 expected_text[] = { - 0x48, 0x8D, 0x0D, 0xF9, 0xEF, 0xFF, 0xFF, - 0x48, 0xB8, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x20, - 0xB8, 0x00, 0x00, 0x00, 0x00, - 0xC3 - }; - String8 text_data = str8_substr(exe, rng_1u64(text_section->foff, text_section->foff + sizeof(expected_text))); - T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0)); -} - -TEST(comdat_any) -{ - T_Ok(t_write_def_obj("1.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test$mn", str8_lit("1"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("test", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("2.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test$mn", str8_lit("2"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("test", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - { - t_invoke_linkerf("/subsystem:console /entry:entry /out:1.exe 1.obj 2.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - String8 exe = t_read_file(arena, str8_lit("1.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".test")); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("1"), 0)); - } - - { - t_invoke_linkerf("/subsystem:console /entry:entry /out:2.exe 2.obj 1.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - String8 exe = t_read_file(arena, str8_lit("2.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".test")); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("2"), 0)); - } -} - -// MSVC vftables use COMDAT sections whose public symbol can start past the -// section definition symbol; references to a replaced copy must target the winner. -TEST(comdat_external_symbol_at_nonzero_offset) -{ - U8 data[16] = {0}; - U8 ptr[8] = {0}; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + foo] - 0xC3 - }; - - T_Ok(t_write_def_obj("leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 8), - {0} - } - })); - - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "foo"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "foo"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 8), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff + 8; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff + 8; - T_Ok(actual_lea_target == expected_lea_target); -} - -TEST(comdat_external_symbol_at_zero_offset) -{ - U8 data[8] = {0}; - U8 ptr[8] = {0}; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + foo] - 0xC3 - }; - - T_Ok(t_write_def_obj("leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "foo"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "foo"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 0), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff; - T_Ok(actual_lea_target == expected_lea_target); -} - -// Duplicate COMDAT sections can have identical bytes while their symbol tables -// disagree about where a same-named public symbol points inside the section. -// This mirrors MSVC vftable COMDATs: the selected copy may have leading RTTI data -// at offset 0 and the vftable symbol at offset 8, while a discarded copy's -// vftable symbol is at offset 0. Relocations against the discarded symbol must -// use the selected symbol's value, not just the selected section contribution -// plus the discarded symbol's original offset. -TEST(comdat_external_symbol_uses_leader_offset) -{ - U8 leader_data[16] = {0}; - U8 ref_data[16] = {0}; - U8 ptr[8] = {0}; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + ??_7X] - 0xC3 - }; - - T_Ok(t_write_def_obj("leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(leader_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 8), - {0} - } - })); - - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(ref_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7X@@6B@"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "??_7X@@6B@"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 0), - T_COFF_DefSymbol_ExternFunc("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff + 8; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff + 8; - T_Ok(actual_lea_target == expected_lea_target); -} - -// Chromium has duplicate vftable COMDATs where the referencing copy is an -// IMAGE_COMDAT_SELECT_ANY section with the public vftable at offset 0, while -// the selected IMAGE_COMDAT_SELECT_LARGEST copy has the same public symbol at -// offset 8. Relocations in the discarded object must resolve to the selected -// public symbol, not to the discarded section. -TEST(comdat_largest_external_symbol_uses_selected_offset) -{ - U8 discarded_data[16] = {0}; - U8 selected_data[24] = {0}; - U8 ptr[16] = {0}; - U8 selected_text[] = { 0xC3 }; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + ??_7X] - 0xC3 - }; - - T_Ok(t_write_def_obj("discarded.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(discarded_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7X@@6B@"), - T_COFF_DefReloc(X64_Addr64, 8, "force_selected"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "??_7X@@6B@"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 0), - T_COFF_DefSymbol_Undef("force_selected"), - T_COFF_DefSymbol_ExternFunc("entry", "text", 0), - {0} - } - })); - - T_Ok(t_write_def_lib("selected.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("selected.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(selected_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "force_text", ".text", str8_array_fixed(selected_text), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 8), - T_COFF_DefSymbol_ExternFunc("force_selected", "force_text", 0), - {0} - } - } - }, - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,noicf discarded.obj selected.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff + 8; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff + 8; - T_Ok(actual_lea_target == expected_lea_target); -} - -TEST(icf_vftable_external_symbol_at_nonzero_offset) -{ - U8 entry_text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + ??_7B] - 0xC3, // ret - }; - U8 vftable_data[16] = {0}; - U8 addresses[16] = {0}; - - T_Ok(t_write_def_obj("vftable.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "??_7B@@6B@"), - {0} - }}, - { "vftable_a", ".rdata$vt", str8_array_fixed(vftable_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vftable_b", ".rdata$vt", str8_array_fixed(vftable_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7A@@6B@"), - T_COFF_DefReloc(X64_Addr64, 8, "??_7B@@6B@"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("vftable_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_R4A@@6B@", "vftable_a", 0), - T_COFF_DefSymbol_Extern("??_7A@@6B@", "vftable_a", 8), - T_COFF_DefSymbol_Extern("??_R4B@@6B@", "vftable_b", 0), - T_COFF_DefSymbol_Extern("??_7B@@6B@", "vftable_b", 8), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses vftable.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 a_vftable_ptr = 0; - U64 b_vftable_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff + 0, &a_vftable_ptr); - str8_deserial_read_struct(exe, data_section->foff + 8, &b_vftable_ptr); - T_Ok(a_vftable_ptr != 0); - T_Ok(b_vftable_ptr != 0); - T_Ok((a_vftable_ptr - pe.image_base - rdata_section->voff) % sizeof(vftable_data) == 8); - T_Ok((b_vftable_ptr - pe.image_base - rdata_section->voff) % sizeof(vftable_data) == 8); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - T_Ok(actual_lea_target == b_vftable_ptr - pe.image_base); -} - -// A referenced zero-sized COMDAT symbol is meaningful enough for -// relocations, even though the COMDAT contributes no bytes to the image. -TEST(zero_length_comdat_referenced_by_reloc) -{ - U8 data[8] = {0}; - - T_Ok(t_write_entry_obj()); - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "pad", ".rdata$a", str8_lit("xy"), .flags = "r:data@1" }, - { "empty", ".rdata$b", str8_zero(), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data", ".data", str8_array_fixed(data), .flags = "rw:data", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "EMPTY"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("empty", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("EMPTY", "empty", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref entry.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *rdata_section = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".data")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - - U64 empty_va = 0; - str8_deserial_read_struct(exe, data_section->foff, &empty_va); - B32 empty_after_pad = empty_va >= pe.image_base + rdata_section->voff + 2; - T_Ok(empty_after_pad); - - U64 empty_off_in_rdata = empty_va - pe.image_base - rdata_section->voff; - T_Ok(rdata_section->foff + empty_off_in_rdata <= exe.size); - T_Ok(exe.str[rdata_section->foff + empty_off_in_rdata - 2] == 'x'); - T_Ok(exe.str[rdata_section->foff + empty_off_in_rdata - 1] == 'y'); -} - -TEST(zero_length_static_comdat_referenced_by_reloc) -{ - if (t_id_linker() != Linker_radlink) { return; } - - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + EMPTY] - 0xC3 - }; - - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "empty", ".rdata", str8_zero(), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "EMPTY"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("empty", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Static("EMPTY", "empty", 0), - T_COFF_DefSymbol_Secdef("text", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,noicf test.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(comdat_no_duplicates) -{ - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "a"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("a"), - {0} - } - })); - - String8 test_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8_lit("a"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("test", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("a", "test", 0), - {0} - } - }); - T_Ok(t_write_file(str8_lit("a.obj"), test_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), test_obj)); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".test")); - T_Ok(sect); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("a"), 0)); -} - -TEST(comdat_same_size) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_SameSize), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("b"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_SameSize), - T_COFF_DefSymbol_Extern("TEST", "b", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("c.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "c", ".c", str8_lit("cc"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_SameSize), - T_COFF_DefSymbol_Extern("TEST", "c", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("a"), 0)); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe a.obj b.obj c.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); } -} - -TEST(comdat_exact_match) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_ExactMatch), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("a2.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a2", ".a2", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a2", COFF_ComdatSelect_ExactMatch), - T_COFF_DefSymbol_Extern("TEST", "a2", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("b"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_ExactMatch), - T_COFF_DefSymbol_Extern("TEST", "b", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe entry.obj a2.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a2")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("a"), 0)); - } -} - -TEST(comdat_largest) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("bb"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "b", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("c.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "c", ".c", str8_lit("c"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "c", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:entry entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *discard_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - T_Ok(discard_sect == 0); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("bb"), 0)); - } - - t_invoke_linkerf("/subsystem:console /out:b.exe /entry:entry entry.obj c.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".c")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("c"), 0)); - } -} - -TEST(comdat_associative) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", "a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aa", "aa", str8_lit("aa"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - T_COFF_DefSymbol_Associative("aa", "a"), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "bb", "bb", str8_lit("bb"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", "b", str8_lit("b"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "bbb", "bbb", str8_lit("bbb"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("bb", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Associative("b", "bb"), - T_COFF_DefSymbol_Associative("bbb", "bb"), - T_COFF_DefSymbol_Extern("TEST", "bb", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *a = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("a")); - COFF_SectionHeader *aa = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("aa")); - COFF_SectionHeader *b = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("b")); - COFF_SectionHeader *bb = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("bb")); - COFF_SectionHeader *bbb = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("bbb")); - T_Ok(a == 0); - T_Ok(aa == 0); - T_Ok(b != 0); - T_Ok(bb != 0); - T_Ok(bbb != 0); - String8 b_data = str8_substr(exe, rng_1u64(b->foff, b->foff + b->vsize)); - String8 bb_data = str8_substr(exe, rng_1u64(bb->foff, bb->foff + bb->vsize)); - String8 bbb_data = str8_substr(exe, rng_1u64(bbb->foff, bbb->foff + bbb->vsize)); - T_Ok(str8_match(b_data, str8_lit("b"), 0)); - T_Ok(str8_match(bb_data, str8_lit("bb"), 0)); - T_Ok(str8_match(bbb_data, str8_lit("bbb"), 0)); -} - -TEST(comdat_associative_loop) -{ - T_Ok(t_write_def_obj("loop.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "aaaa", ".aaaa", str8_lit("aaaa"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aa", ".aa", str8_lit("aa"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "a", ".a", str8_lit("a"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aaa", ".aaa", str8_lit("aaa"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Associative("aaa", "aa"), - T_COFF_DefSymbol_Associative("aaaa", "aaa"), - T_COFF_DefSymbol_Associative("a", "aa"), - T_COFF_DefSymbol_Associative("aa", "a"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe loop.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_AssociativeLoop); } -} - -TEST(comdat_associative_non_comdat) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data" }, - { "b", ".b", str8_lit("b"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "a", 0), - T_COFF_DefSymbol_Associative("b", "a"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a != 0); - T_Ok(b != 0); - String8 a_data = str8_substr(exe, rng_1u64(a->foff, a->foff + a->vsize)); - String8 b_data = str8_substr(exe, rng_1u64(b->foff, b->foff + b->vsize)); - T_Ok(str8_match(a_data, str8_lit("a"), 0)); - T_Ok(str8_match(b_data, str8_lit("b"), 0)); -} - -TEST(comdat_associative_out_of_bounds) -{ - { - String8 obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aa", ".aa", str8_lit("aa"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - T_COFF_DefSymbol_Associative("aa", "a"), - {0} - } - }); - { - COFF_FileHeaderInfo header = coff_file_header_info_from_data(obj); - String8 string_table = str8_substr(obj, header.string_table_range); - String8 symbol_table = str8_substr(obj, header.symbol_table_range); - COFF_ParsedSymbol symbol = coff_parse_symbol(header, string_table, symbol_table, 3); - AssertAlways(str8_match(symbol.name, str8_lit(".aa"), 0)); - AssertAlways(symbol.aux_symbol_count == 1); - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - COFF_SymbolSecDef *secdef = (COFF_SymbolSecDef *)(symbol16 + 1); - secdef->number_lo = 321; - } - T_Ok(t_write_file(str8_lit("bad.obj"), obj)); - } - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_IllData); } -} - -TEST(comdat_with_offset) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "rdata", ".rdata", str8_lit_cstr("1Hello, World!"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("rdata", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "rdata", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "rdata", ".rdata", str8_lit_cstr("Hello, World!"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("rdata", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "rdata", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(reloc_against_removed_comdat) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "rdata", ".rdata", str8_lit_cstr("1Hello, World!"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("rdata", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "rdata", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "comdat", ".rdata", str8_lit_cstr("H"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "regular", ".rdata", str8_lit_comp("\x00\x00\x00\x00"), .flags = "r:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "STATIC"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("comdat", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "comdat", 1), - T_COFF_DefSymbol_Static("STATIC", "comdat", 2), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(sect_align) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "shift", ".a", str8_lit("q"), .flags = "rw:data" }, - { "none", ".a", str8_lit("abc"), .flags = "rw:data" }, - { "a1", ".a", str8_lit("wr"), .flags = "rw:data@1" }, - { "a2", ".a", str8_lit("e"), .flags = "rw:data@2" }, - { "a4", ".a", str8_lit("ttttt"), .flags = "rw:data@4" }, - { "a8", ".a", str8_lit("g"), .flags = "rw:data@8" }, - { "a16", ".a", str8_lit("o"), .flags = "rw:data@16" }, - { "a32", ".a", str8_lit("p"), .flags = "rw:data@32" }, - { "a64", ".a", str8_lit("f"), .flags = "rw:data@64" }, - { "a128", ".a", str8_lit("x"), .flags = "rw:data@128" }, - { "a256", ".a", str8_lit("c"), .flags = "rw:data@256" }, - { "a512", ".a", str8_lit("v"), .flags = "rw:data@512" }, - { "a1024", ".a", str8_lit("b"), .flags = "rw:data@1024" }, - { "a2048", ".a", str8_lit("n"), .flags = "rw:data@2048" }, - { "a4096", ".a", str8_lit("m"), .flags = "rw:data@4096" }, - { "a8192", ".a", str8_lit("z"), .flags = "rw:data@8192" }, - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe /align:8192 test.obj"); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".a")); - T_Ok(sect); - String8 sect_data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - - String8 shift = str8_substr(sect_data, rng_1u64(0, 1)); - T_Ok(str8_match(shift, str8_lit("q"), 0)); - String8 a_none = str8_substr(sect_data, rng_1u64(16, 16 + 3)); - T_Ok(str8_match(a_none, str8_lit("abc"), 0)); - String8 a_1 = str8_substr(sect_data, rng_1u64(19, 21)); - T_Ok(str8_match(a_1, str8_lit("wr"), 0)); - String8 a_2 = str8_substr(sect_data, rng_1u64(22, 23)); - T_Ok(str8_match(a_2, str8_lit("e"), 0)); - String8 a_4 = str8_substr(sect_data, rng_1u64(24, 29)); - T_Ok(str8_match(a_4, str8_lit("ttttt"), 0)); - String8 a_8 = str8_substr(sect_data, rng_1u64(32, 33)); - T_Ok(str8_match(a_8, str8_lit("g"), 0)); - String8 a_16 = str8_substr(sect_data, rng_1u64(48, 49)); - T_Ok(str8_match(a_16, str8_lit("o"), 0)); - String8 a_32 = str8_substr(sect_data, rng_1u64(64, 65)); - T_Ok(str8_match(a_32, str8_lit("p"), 0)); - String8 a_64 = str8_substr(sect_data, rng_1u64(128, 129)); - T_Ok(str8_match(a_64, str8_lit("f"), 0)); - String8 a_128 = str8_substr(sect_data, rng_1u64(256, 257)); - T_Ok(str8_match(a_128, str8_lit("x"), 0)); - String8 a_256 = str8_substr(sect_data, rng_1u64(512, 513)); - T_Ok(str8_match(a_256, str8_lit("c"), 0)); - String8 a_512 = str8_substr(sect_data, rng_1u64(1024, 1025)); - T_Ok(str8_match(a_512, str8_lit("v"), 0)); - String8 a_1024 = str8_substr(sect_data, rng_1u64(2048, 2049)); - T_Ok(str8_match(a_1024, str8_lit("b"), 0)); - String8 a_2048 = str8_substr(sect_data, rng_1u64(4096, 4097)); - T_Ok(str8_match(a_2048, str8_lit("n"), 0)); - String8 a_4096 = str8_substr(sect_data, rng_1u64(8192, 8193)); - T_Ok(str8_match(a_4096, str8_lit("m"), 0)); - String8 a_8192 = str8_substr(sect_data, rng_1u64(16384, 16385)); - T_Ok(str8_match(a_8192, str8_lit("z"), 0)); -} - -TEST(alt_name) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("test", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("foo.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("foo"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "foo"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0} - } - })); - - // basic alternate name test - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:foo=test test.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - // linker should not chase alt name links - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe /alternatename:foo=bar /alternatename:bar=test test.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // alt name conflict - t_invoke_linkerf("/subsystem:console /entry:entry /out:c.exe /alternatename:foo=test /alternatename:foo=qwe test.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:d.exe /alternatename:foo foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:e.exe /alternatename:foo-oof foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:foo=test=bar foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:foo= foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:= foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename: foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // TODO: check that RAD Linker prints these warnings - - // warn about alt name to self alt name? - t_invoke_linkerf("/subsystem:console /entry:entry /out:f.exe /alternatename:foo=foo foo.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - // warn about alt name to unknown symbol? - t_invoke_linkerf("/subsystem:console /entry:entry /out:g.exe /alternatename:qwe=ewq foo.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(include) -{ - T_Ok(t_write_def_lib("include.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("include.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("foo"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "entry"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - // simple include test - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /include:foo entry.obj include.lib"); - T_Ok(g_last_exit_code == 0); - - // validate that linker pulled-in include.obj - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *foo_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(foo_sect != 0); - String8 foo_data = str8_substr(exe, rng_1u64(foo_sect->foff, foo_sect->foff + foo_sect->vsize)); - T_Ok(str8_match(foo_data, str8_lit("foo"), 0)); - } - - // test unresolved include - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /include:ewq entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_UnresolvedSymbol); } -} - -TEST(communal_var_vs_regular) -{ - T_Ok(t_write_def_obj("communal.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("TEST", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("defn.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - // linker should replace communal TEST with .data TEST - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe communal.obj defn.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe defn.obj communal.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - char *exes[] = { "a.exe", "b.exe" }; - for EachElement(i, exes) { - String8 exe = t_read_file(arena, str8_cstring(exes[i])); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_sect); - String8 data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->vsize)); - T_Ok(str8_match(data, str8_lit("test"), 0)); - } -} - -TEST(communal_var_vs_regular_comdat) -{ - T_Ok(t_write_def_obj("communal.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("TEST", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("large.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - // linker should replace communal TEST with .data TEST - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe communal.obj large.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe large.obj communal.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - char *exes[] = { "a.exe", "b.exe" }; - for EachElement(i, exes) { - String8 exe = t_read_file(arena, str8_cstring(exes[i])); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_sect); - String8 data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->vsize)); - T_Ok(str8_match(data, str8_lit("test"), 0)); - } -} - -#if OS_WINDOWS -TEST(import_kernel32) -{ - T_Ok(t_write_def_obj("import.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit_cstr("test"), .flags = "rw:data" }, - { - "text", ".text", - str8_lit_comp( - "\x48\x83\xEC\x68" // sub rsp,68h ; alloc space on stack - "\xC7\x44\x24\x48\x18\x00\x00\x00" // mov dword ptr [rsp+48h],18h ; SECURITY_ATTRIBUTES.nLength - "\x48\xC7\x44\x24\x50\x00\x00\x00\x00" // mov qword ptr [rsp+50h],0 ; SECURITY_ATTRIBUTES.lpSecurityDescriptor - "\xC7\x44\x24\x58\x00\x00\x00\x00" // mov dword ptr [rsp+58h],0 ; SECURITY_ATTRIBUTES.bInheritHandle - "\x48\xC7\x44\x24\x30\x00\x00\x00\x00" // mov qword ptr [rsp+30h],0 ; hTemplateFile - "\xC7\x44\x24\x28\x80\x00\x00\x00" // mov dword ptr [rsp+28h],80h ; dwFlagsAndAttributes - "\xC7\x44\x24\x20\x02\x00\x00\x00" // mov dword ptr [rsp+20h],2 ; dwCreationDisposition - "\x4C\x8D\x4C\x24\x48" // lea r9,[rsp+48h] ; lpSecurityAttributes - "\x45\x33\xC0" // xor r8d,r8d ; dwShareMode - "\xBA\x00\x00\x00\x40" // mov edx,40000000h ; dwDesiredAccess - "\x48\x8D\x0D\x00\x00\x00\x00" // lea rcx,[test] ; lpFileName - "\xFF\x15\x00\x00\x00\x00" // call qword ptr [__imp_CreateFileA] ; call CreateFileA - "\x48\x89\xC1" // mov rcx,rax ; hObject - "\xFF\x15\x00\x00\x00\x00" // call qword ptr [__imp_CloseHandle] ; call CloseHandle - "\x33\xC0" // xor eax,eax ; clear result - "\x48\x83\xC4\x68" // add rsp,68h ; dealloc stack - "\xC3" // ret ; return - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 70, "test"), - T_COFF_DefReloc(X64_Rel32, 76, "__imp_CreateFileA"), - T_COFF_DefReloc(X64_Rel32, 85, "__imp_CloseHandle"), - {0}, - } - }, - { - "dead", ".text$dead", str8(0, 4), - .flags = "rx:code@1", - .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "__imp_CompareStringW"), - {0}, - } - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("dead", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("test", "data", 0), - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_ExternFunc("dead", "dead", 0), - T_COFF_DefSymbol_Undef("__imp_CreateFileA"), - T_COFF_DefSymbol_Undef("__imp_CloseHandle"), - T_COFF_DefSymbol_Undef("__imp_CompareStringW"), - {0}, - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /fixed /opt:ref import.obj kernel32.lib"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - PE_ParsedStaticImportTable imports = pe_static_imports_from_data(arena, pe.is_pe32, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); - B32 found_create = 0; - B32 found_close = 0; - B32 found_dead = 0; - for EachIndex(dll_idx, imports.count) { - PE_ParsedStaticDLLImport *dll = &imports.v[dll_idx]; - for EachIndex(import_idx, dll->import_count) { - PE_ParsedImport *import = &dll->imports[import_idx]; - if (import->type == PE_ParsedImport_Name) { - found_create |= str8_match(import->u.name.string, str8_lit("CreateFileA"), 0); - found_close |= str8_match(import->u.name.string, str8_lit("CloseHandle"), 0); - found_dead |= str8_match(import->u.name.string, str8_lit("CompareStringW"), 0); - } - } - } - T_Ok(found_create); - T_Ok(found_close); - T_Ok(!found_dead); - - String8 test_file_path = push_str8f(arena, "%S/test", g_wdir); - delete_file_at_path(test_file_path); - - ProcessLaunchParams launch_opts = {0}; - launch_opts.inherit_env = 0; - launch_opts.path = g_wdir; - str8_list_pushf(arena, &launch_opts.cmd_line, "%S/a.exe", g_wdir); - Process handle = process_launch(&launch_opts); - AssertAlways(!process_match(handle, process_zero())); - U64 exit_code = max_U64; - process_join(handle, max_U64, &exit_code); - T_Ok(exit_code == 0); - T_Ok(file_path_exists(test_file_path)); - } -} - -TEST(delay_import) -{ - { - U8 return_0[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, // mov rax, 0 - 0xc3 // ret - }; - U8 return_1[] = { - 0x48, 0xC7, 0xC0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3 // ret - }; - U8 return_2[] = { - 0x48, 0xC7, 0xC0, 0x02, 0x00, 0x00, 0x00, // mov rax, 2 - 0xc3 // ret - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "return_0", ".text", str8_array_fixed(return_0), .flags = "rx:code@1" }, - { "return_1", ".text", str8_array_fixed(return_1), .flags = "rx:code@1" }, - { "return_2", ".text", str8_array_fixed(return_2), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("return_1", "return_1", 0), - T_COFF_DefSymbol_ExternFunc("return_2", "return_2", 0), - {0} - } - })); - } - - { - U8 return_0[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, // mov rax, 0 - 0xc3 // ret - }; - U8 return_123[] = { - 0x48, 0xC7, 0xC0, 0x7B, 0x00, 0x00, 0x00, // mov rax, 123 - 0xc3 // ret - }; - U8 return_321[] = { - 0x48, 0xC7, 0xC0, 0x41, 0x01, 0x00, 0x00, // mov rax, 321 - 0xc3 // ret - }; - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "return_0", ".text", str8_array_fixed(return_0), .flags = "rx:code@1" }, - { "return_123", ".text", str8_array_fixed(return_123), .flags = "rx:code@1" }, - { "return_321", ".text", str8_array_fixed(return_321), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("return_123", "return_123", 0), - T_COFF_DefSymbol_ExternFunc("return_321", "return_321", 0), - {0} - } - })); - } - - { - U8 text[] = { - 0x56, // push rsi - 0x57, // push rdi - 0x48, 0x83, 0xEC, 0x28, // sub rsp,28h - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_1 - 0x89, 0xC6, // mov esi,eax - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_2 - 0x89, 0xC7, // mov edi,eax - 0x01, 0xF7, // add edi,esi - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_123 - 0x89, 0xC6, // mov esi,eax - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_321 - 0x01, 0xF0, // add eax,esi - 0x01, 0xF8, // add eax,edi - 0x48, 0x83, 0xC4, 0x28, // add rsp,28h - 0x5F, // pop rdi - 0x5E, // pop rsi - 0xC3, // ret - }; - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 7, "return_1"), - T_COFF_DefReloc(X64_Rel32, 14, "return_2"), - T_COFF_DefReloc(X64_Rel32, 23, "return_123"), - T_COFF_DefReloc(X64_Rel32, 30, "return_321"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("return_1"), - T_COFF_DefSymbol_Undef("return_2"), - T_COFF_DefSymbol_Undef("return_123"), - T_COFF_DefSymbol_Undef("return_321"), - {0} - } - })); - } - - t_invoke_linkerf("/dll /implib:a.lib /export:return_1 /export:return_2 /rad_time_stamp:0x69EB0E28 a.obj libcmt.lib"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/dll /implib:b.lib /export:return_123 /export:return_321 /rad_time_stamp:0x69EB0E28 b.obj libcmt.lib"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /fixed /debug:full /rad_time_stamp:0x69EB0E28 main.obj a.lib b.lib kernel32.lib delayimp.lib libcmt.lib /delayload:a.dll /delayload:b.dll"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - PE_ParsedDelayImportTable delay_import_table = pe_delay_imports_from_data(arena, pe.is_pe32, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_DELAY_IMPORT]); - - PE_ParsedDelayDLLImport *a_import = &delay_import_table.v[0]; - T_Ok(a_import->attributes == 1); - T_Ok(str8_match(a_import->name, str8_lit("a.dll"), 0)); - T_Ok(a_import->module_handle_voff != 0); - T_Ok(a_import->name_table_voff != 0); - T_Ok(a_import->bound_table_voff != 0); - T_Ok(a_import->unload_table_voff != 0); - T_Ok(a_import->time_stamp == 0); - T_Ok(a_import->bound_table_count == 2); - T_Ok(a_import->unload_table_count == 2); - T_Ok(a_import->import_count == 2); - - PE_ParsedImport *return_1 = &a_import->imports[0]; - T_Ok(return_1->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_1->u.name.string, str8_lit("return_1"), 0)); - T_Ok(return_1->u.name.hint == 0); - - PE_ParsedImport *return_2 = &a_import->imports[1]; - T_Ok(return_2->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_2->u.name.string, str8_lit("return_2"), 0)); - T_Ok(return_2->u.name.hint == 1); - - PE_ParsedDelayDLLImport *b_import = &delay_import_table.v[1]; - T_Ok(b_import->attributes == 1); - T_Ok(str8_match(b_import->name, str8_lit("b.dll"), 0)); - T_Ok(b_import->module_handle_voff != 0); - T_Ok(b_import->name_table_voff != 0); - T_Ok(b_import->bound_table_voff != 0); - T_Ok(b_import->unload_table_voff != 0); - T_Ok(b_import->time_stamp == 0); - T_Ok(b_import->bound_table_count == 2); - T_Ok(b_import->unload_table_count == 2); - T_Ok(b_import->import_count == 2); - - PE_ParsedImport *return_123 = &b_import->imports[0]; - T_Ok(return_123->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_123->u.name.string, str8_lit("return_123"), 0)); - T_Ok(return_123->u.name.hint == 0); - - PE_ParsedImport *return_321 = &b_import->imports[1]; - T_Ok(return_321->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_321->u.name.string, str8_lit("return_321"), 0)); - T_Ok(return_321->u.name.hint == 1); -} - -TEST(delay_import_user32) -{ - { - U64 msg_off = 0; - U64 caption_off = 5; - String8 str_payload = str8_lit_comp("test\0foo\0"); - U8 text[] = { - 0x48, 0x83, 0xEC, 0x28, // sub rsp,28h - 0x45, 0x33, 0xC9, // xor r9d,r9d - 0x4C, 0x8D, 0x05, 0x00, 0x00, 0x00, 0x00, // lea r8,[msg] - 0x48, 0x8D, 0x15, 0x00, 0x00, 0x00, 0x00, // lea rdx,[caption] - 0x33, 0xC9, // xor ecx,ecx - 0xFF, 0x15, 0x00, 0x00, 0x00, 0x00, // call qword ptr [__imp_MessageBoxA] - 0x33, 0xC0, // xor eax,eax - 0x48, 0x83, 0xC4, 0x28, // add rsp,28h - 0xC3, // ret - }; - T_Ok(t_write_def_obj("delay_import.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "str", ".str", str_payload, .flags = "rw:data" }, - { - "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 10, "msg"), - T_COFF_DefReloc(X64_Rel32, 17, "caption"), - T_COFF_DefReloc(X64_Rel32, 25, "__imp_MessageBoxA"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("msg", "str", msg_off), - T_COFF_DefSymbol_Extern("caption", "str", caption_off), - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("__imp_MessageBoxA"), - {0} - } - })); - } - - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:entry /fixed /delayload:user32.dll kernel32.lib user32.lib libcmt.lib delayimp.lib delay_import.obj /debug:full"); - T_Ok(g_last_exit_code == 0); -} - -#endif - -TEST(empty_section) -{ - T_Ok(t_write_def_obj("empty_section.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8(0,0), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("TEST"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe empty_section.obj entry.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(removed_section) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8_lit_comp("\xC3"), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkRemove }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("TEST"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe test.obj entry.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(function_pad_min) -{ - T_Ok(t_write_def_obj("funcs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@4" }, - { "b", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@4" }, - { "c", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("A", "a", 0), - T_COFF_DefSymbol_ExternFunc("B", "b", 0), - T_COFF_DefSymbol_ExternFunc("C", "c", 0), - {0}, - } - })); - - t_invoke_linkerf("/subsystem:console /entry:A /functionpadmin:1 /out:a.exe funcs.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - - U8 expected_text[] = { - 0xcc, 0xcc, 0xcc, 0xcc, 0xc3, - 0xcc, 0xcc, 0xcc, 0xc3, - 0xcc, 0xc3, - }; - T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0)); -} - -TEST(first_member_header) -{ - T_Ok(t_write_def_lib("test.lib", (T_COFF_DefLib){ - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("obj.obj"), - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("8", 0x8), - T_COFF_DefSymbol_AbsExtern("1", 0x1), - T_COFF_DefSymbol_AbsExtern("9", 0x9), - T_COFF_DefSymbol_AbsExtern("7", 0x7), - T_COFF_DefSymbol_AbsExtern("4", 0x4), - T_COFF_DefSymbol_AbsExtern("5", 0x5), - T_COFF_DefSymbol_AbsExtern("2", 0x2), - T_COFF_DefSymbol_AbsExtern("3", 0x3), - T_COFF_DefSymbol_AbsExtern("6", 0x6), - {0} - } - } - }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - String8 lib_before = t_read_file(arena, str8_lit("test.lib")); - String8 obj_before = t_read_file(arena, str8_lit("entry.obj")); - - t_invoke_linkerf("/RAD_MEMORY_MAP_FILES:READ_WRITE /subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9"); - T_Ok(g_last_exit_code == 0); - T_Ok(str8_match(lib_before, t_read_file(arena, str8_lit("test.lib")), 0)); - T_Ok(str8_match(obj_before, t_read_file(arena, str8_lit("entry.obj")), 0)); -} - -TEST(second_member_header) -{ - T_Ok(t_write_def_lib("test.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("obj.obj"), - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("8", 0x8), - T_COFF_DefSymbol_AbsExtern("1", 0x1), - T_COFF_DefSymbol_AbsExtern("9", 0x9), - T_COFF_DefSymbol_AbsExtern("7", 0x7), - T_COFF_DefSymbol_AbsExtern("4", 0x4), - T_COFF_DefSymbol_AbsExtern("5", 0x5), - T_COFF_DefSymbol_AbsExtern("2", 0x2), - T_COFF_DefSymbol_AbsExtern("3", 0x3), - T_COFF_DefSymbol_AbsExtern("6", 0x6), - {0} - } - } - }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9"); - T_Ok(g_last_exit_code == 0); -} - -TEST(defer_impl_link_to_second_search_pass) -{ - T_Ok(t_write_def_obj("imp_ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "text", 0), - T_COFF_DefSymbol_Undef("__imp_foo"), - T_COFF_DefSymbol_Undef("func"), - {0} - } - })); - - T_Ok(t_write_def_lib("impl_ref.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("impl_ref.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("func", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_lib("foo.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Import, - .import = { - "foo.dll", "foo", COFF_ImportBy_Name, COFF_ImportHeader_Code, - .hit_or_ordinal = 0, - .time_stamp = T_COFF_DefSetTimeStamp(~0u), - .machine = T_COFF_DefSetMachine(X64) - } - }, - {0} - } - })); - - T_Ok(t_write_def_lib("foo2.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Import, - .import = { - "foo.dll", "foo", COFF_ImportBy_Name, COFF_ImportHeader_Code, - .hit_or_ordinal = 0, - .time_stamp = T_COFF_DefSetTimeStamp(~0u), - .machine = T_COFF_DefSetMachine(X64) - } - }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe foo.lib foo2.lib imp_ref.obj impl_ref.lib"); - T_Ok(g_last_exit_code == 0); -} - -TEST(defer_duplicate_imp_link) -{ - T_COFF_DefLib bar_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "bar.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "bar.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\xff\x25\x00\x00\x00\x00"), .flags = "rx:code" - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__imp_bar"), - T_COFF_DefSymbol_ExternFunc("qwe", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - T_COFF_DefLib foo_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "foo.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "foo.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\xff\x25\x00\x00\x00\x00"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 2, "bar"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("bar"), - T_COFF_DefSymbol_Undef("qwe"), - T_COFF_DefSymbol_ExternFunc("thunk", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - String8 bar_lib = t_coff_from_def_lib(arena, bar_lib_any); - String8 foo_lib = t_coff_from_def_lib(arena, foo_lib_any); - - T_Ok(t_write_file(str8_lit("bar.lib"), bar_lib)); - T_Ok(t_write_file(str8_lit("foo.lib"), foo_lib)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe bar.lib foo.lib entry.obj /include:thunk"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - PE_ParsedStaticImportTable static_imptab = pe_static_imports_from_data(arena, pe.is_pe32, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); - COFF_SectionHeader *data_sect = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".data")); - - T_Ok(static_imptab.count == 1); - - PE_ParsedStaticDLLImport *dll = &static_imptab.v[0]; - T_Ok(str8_match(dll->name, str8_lit("foo.dll"), StringMatchFlag_CaseInsensitive)); - - T_Ok(dll->import_count == 1); - PE_ParsedImport *imp = &dll->imports[0]; - T_Ok(imp->type == PE_ParsedImport_Name); - T_Ok(str8_match(imp->u.name.string, str8_lit("bar"), 0)); - - U64 iat_foff = pe_foff_from_voff(exe, &pe, dll->import_address_table_voff); - U64 ilt_foff = pe_foff_from_voff(exe, &pe, dll->import_name_table_voff); - - U64 bar_idx = 0; - U64 bar_iat_addr = 0; - U64 bar_ilt_addr = 0; - str8_deserial_read_struct(exe, iat_foff + bar_idx * sizeof(U64), &bar_iat_addr); - str8_deserial_read_struct(exe, ilt_foff + bar_idx * sizeof(U64), &bar_ilt_addr); - - T_Ok(bar_iat_addr != 0); - T_Ok(bar_ilt_addr != 0); - T_Ok(bar_iat_addr == bar_ilt_addr); - T_Ok(data_sect->voff <= bar_iat_addr && bar_iat_addr < data_sect->voff + data_sect->vsize); -} - -TEST(opt_ref_dangling_section) -{ - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "f"), - {0} - } - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("f"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_cstr("A0000"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "q"), - {0} - } - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("q"), - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("f", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_lib("b.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("b.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "q", ".q", str8_lit_comp("\x1\x2\x3\x4"), .flags = "rw:data" }, - { - "data", ".data", str8_lit_cstr("BBBBBBBBBBBBBBB"), - .flags = "rw:data", - .raw_flags = COFF_SectionFlag_LnkCOMDAT - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("q", "q", 0), - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("f", "data", 0), - {0}, - } - } - }, - {0}, - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".q")); - T_Ok(sect != 0); -} - -// TODO: relocations against undefined section symbols -#if 1 -TEST(relocate_undefined_section_symbol) -{ - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "caller"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("caller"), - {0} - } - })); - - U64 target_count = 25; - - String8 caller_data = str8( - push_array(arena, U8, target_count * 4), target_count * 4); - T_COFF_DefReloc *caller_relocs = push_array(arena, T_COFF_DefReloc, target_count + 1); - T_COFF_DefSymbol *caller_symbols = push_array(arena, T_COFF_DefSymbol, target_count + 3); - - caller_symbols[0] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("caller", COFF_ComdatSelect_Any); - caller_symbols[1] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Extern("caller", "caller", 0); - - String8List targets = {0}; - - for EachIndex(i, target_count) { - char *obj_name = (char *)str8f(arena, "target%u.obj", i).str; - char *target_name = (char *)str8f(arena, "target%u", i).str; - char *sect_name = (char *)str8f(arena, ".target%u", i).str; - T_Ok(t_write_def_obj(obj_name, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { target_name, sect_name, str8_cstring(target_name), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef(target_name, COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern(target_name, target_name, 0), - {0} - } - })); - - caller_relocs[i] = (T_COFF_DefReloc)T_COFF_DefReloc(X64_Addr32Nb, i * 4, sect_name); - caller_symbols[i + 2] = (T_COFF_DefSymbol)T_COFF_DefSymbol_UndefSec(sect_name, COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead); - - str8_list_pushf(arena, &targets, obj_name); - } - - T_Ok(t_write_def_obj("caller.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "caller", ".caller", caller_data, .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = caller_relocs}, - {0} - }, - .symbols = caller_symbols, - })); - - String8 target_paths = str8_list_join(arena, &targets, &(StringJoin){.sep=str8_lit(" ")}); - T_Ok(t_invoke_linkerf("/subsystem:console /entry:entry /opt:ref /out:a.exe entry.obj caller.obj %S", target_paths) && g_last_exit_code == 0); - - String8 output = g_errors; - U64 match_count = 0; - while (output.size) { - if (t_match_line(&output, str8_lit("WARNING(*): *: relocation * in section *+* points to an undefined symbol *(*)"))) { - match_count += 1; - } - } - //T_Ok(match_count > 0); -} -#endif - -#if 1 -TEST(weak_alias_comdat_duplicate_fallback) -{ - U64 dummy_count = 64; - - for EachIndex(obj_idx, 2) { - char prefix = obj_idx == 0 ? 'a' : 'b'; - T_COFF_DefSection *sections = push_array(arena, T_COFF_DefSection, dummy_count + 5); - T_COFF_DefSymbol *symbols = push_array(arena, T_COFF_DefSymbol, dummy_count*2 + 11); - U64 section_idx = 0; - U64 symbol_idx = 0; - - for EachIndex(i, dummy_count) { - char *id = (char *)str8f(arena, "dummy_%c_%I64u", prefix, i).str; - char *name = (char *)str8f(arena, "?dummy_%c_%I64u@@YAXXZ", prefix, i).str; - sections[section_idx++] = (T_COFF_DefSection){ id, ".text", str8_lit_comp("\xC3"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef(id, COFF_ComdatSelect_Any); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc(name, id, 0); - } - - sections[section_idx++] = (T_COFF_DefSection){ "text0", ".text", str8_lit_comp("\x33\xC0"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - sections[section_idx++] = (T_COFF_DefSection){ "text1", ".text", str8_lit_comp("\x33\xC0"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - sections[section_idx++] = (T_COFF_DefSection){ "xdata0", ".xdata", str8_lit_comp("\x01\x00\x00\x00"), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - sections[section_idx++] = (T_COFF_DefSection){ "text2", ".text", str8_lit_comp("\x33\xC0\xC3"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text0", COFF_ComdatSelect_Any); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G__shared_count@__Cr@std@@MEAAPEAXI@Z", "text0", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text1", COFF_ComdatSelect_Any); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", "text1", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Associative("xdata0", "text0"); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text2", COFF_ComdatSelect_Any); - - if (obj_idx == 0) { - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", "text2", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z"); - } else { - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", "text2", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z"); - } - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E__shared_count@__Cr@std@@MEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G__shared_count@__Cr@std@@MEAAPEAXI@Z"); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z"); - - T_Ok(t_write_def_obj(obj_idx == 0 ? "a.obj" : "b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = sections, - .symbols = symbols, - })); - } - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("??_E__shared_count@__Cr@std@@MEAAPEAXI@Z"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe b.obj a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} -#endif - -#if 1 -TEST(opt_ref_weak_alias_comdat) -{ - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "target", ".target", str8_lit("target"), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("target", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("target", "target", 0), - T_COFF_DefSymbol_Weak("weak_target", COFF_WeakExt_SearchAlias, "target"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "weak_target"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("weak_target"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /opt:ref /out:a.exe entry.obj weak.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *target_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".target")); - T_Ok(target_sect != 0); -} -#endif - - -TEST(fail_if_mismatch) -{ - T_Ok(t_write_entry_obj()); - - // ------------------------------------------------------------ - // try linking two objs with mismatching directives - - String8 a1 = t_make_obj_with_directive(arena, str8_lit("/FAILIFMISMATCH:a=1")); - String8 a2 = t_make_obj_with_directive(arena, str8_lit("/FAILIFMISMATCH:a=2")); - T_Ok(t_write_file(str8_lit("a1.obj"), a1)); - T_Ok(t_write_file(str8_lit("a2.obj"), a2)); - - t_invoke_linkerf("entry.obj a1.obj a2.obj /entry:entry /subsystem:console /out:a2.exe"); - if (t_id_linker() == Linker_radlink) T_Ok(g_last_exit_code == LNK_Error_FailIfMismatch); - else T_Ok(g_last_exit_code != 0); - - // ------------------------------------------------------------ - // happy case - - T_Ok(t_write_file(str8_lit("a1_copy.obj"), a1)); - - t_invoke_linkerf("entry.obj a1.obj a1_copy.obj /entry:entry /subsystem:console /out:a1.exe"); - T_Ok(g_last_exit_code == 0); - - // ------------------------------------------------------------ - // test conflicting directives in obj - - String8 conf_dirs = t_make_obj_with_directive(arena, str8_lit("/FAILIFMISMATCH:a=1 /FAILIFMISMATCH:a=2")); - T_Ok(t_write_file(str8_lit("conf_dirs.obj"), conf_dirs)); - - t_invoke_linkerf("entry.obj conf_dirs.obj /entry:entry /subsystem:console /out:conf_dirs.exe"); - if (t_id_linker() == Linker_radlink) T_Ok(g_last_exit_code == LNK_Error_FailIfMismatch); - else T_Ok(g_last_exit_code != 0); - - // ------------------------------------------------------------ - // passing switch on command line - - t_invoke_linkerf("entry.obj a1.obj /FAILIFMISMATCH:a=2 /out:cmddir.exe"); - if (t_id_linker() == Linker_radlink) T_Ok(g_last_exit_code == LNK_Error_FailIfMismatch); - else T_Ok(g_last_exit_code != 0); -} - -TEST(guardsym_directive) -{ - T_Ok(t_write_entry_obj()); - - // MSVC link accepts GUARDSYM without treating the named symbol as /INCLUDE. - String8 guardsym = t_make_obj_with_directive(arena, str8_lit("/GUARDSYM:missing,S")); - T_Ok(t_write_file(str8_lit("guardsym.obj"), guardsym)); - - t_invoke_linkerf("entry.obj guardsym.obj /entry:entry /subsystem:console /out:guardsym.exe"); - T_Ok(g_last_exit_code == 0); -} - -TEST(long_section_name) -{ - Arch arch = Arch_x64; - - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code" }, - { "debug_info", ".debug_info", str8_lit("DEBUG_INFO"), .flags = "rw:data" }, - { "debug_abbrev", ".debug_abbrev", str8_lit("DEBUG_ABBREV"), .flags = "rw:data" }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - // link test.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe test.obj"); - T_Ok(g_last_exit_code == 0); - - // load linked exe - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *debug_info = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".debug_info")); - T_Ok(debug_info); - - COFF_SectionHeader *debug_abbrev = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".debug_abbrev")); - T_Ok(debug_abbrev); -} - -TEST(debug_p_sig_mismatch) -{ - String8 a_obj_name = str8_lit("a.obj"); - String8 b_obj_name = str8_lit("b.obj"); - U32 a_sig = 0xCAFEBABE; - U32 b_sig = 0xDEADBEEF; - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = a_sig; - String8 obj_name_string = a_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - String8 a_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = a_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - a_debug_p = str8_serial_end(arena, &srl); - } - - String8 b_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = b_sig; - String8 obj_name_string = a_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - b_debug_s = str8_serial_end(arena, &srl); - } - - String8 b_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = b_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + a_obj_name.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, a_obj_name); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - b_debug_t = str8_serial_end(arena, &srl); - } - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", a_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", b_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", b_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - B32 found_error = 0; - String8 a_obj_path = t_make_file_path(arena, str8_lit("a.obj")); - String8 b_obj_path = t_make_file_path(arena, str8_lit("b.obj")); - String8 expected_line = str8f(arena, "Error(%03d): %S: PCH signature mismatch, expected 0x%x got 0x%x; PCH obj %S", LNK_Error_PrecompSigMismatch, b_obj_path, b_sig, a_sig, a_obj_path); - - String8 output = g_errors; - while (output.size) { - String8 line = t_chop_line(&output); - found_error = str8_match(line, expected_line, StringMatchFlag_CaseInsensitive|StringMatchFlag_SlashInsensitive); - if (found_error) { break; } - } - T_Ok(found_error); -} - -TEST(pch_sig_fallback) -{ - String8 a_obj_file_path = t_make_file_path(arena, str8_lit("a.obj")); - String8 b_obj_file_path = t_make_file_path(arena, str8_lit("b.obj")); - U32 a_sig = 0xCAFEBABE; - U32 b_sig = 0xCAFEBABE; - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = a_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - String8 a_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = a_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - a_debug_p = str8_serial_end(arena, &srl); - } - - String8 b_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = b_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - b_debug_s = str8_serial_end(arena, &srl); - } - - String8 b_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - String8 corrupt_pch_path = str8_lit("corrupt-pch-file-path.obj"); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = b_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + corrupt_pch_path.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, corrupt_pch_path); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - b_debug_t = str8_serial_end(arena, &srl); - } - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", a_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", b_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", b_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(write_data_to_file_path(a_obj_file_path, a_obj)); - T_Ok(write_data_to_file_path(b_obj_file_path, b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(gen_rrt) -{ - String8 a_obj_file_path = t_make_file_path(arena, str8_lit("a.obj")); - String8 b_obj_file_path = t_make_file_path(arena, str8_lit("b.obj")); - U32 a_sig = 0xCAFEBABE; - U32 b_sig = 0xCAFEBABE; - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = a_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - String8 a_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = a_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - a_debug_p = str8_serial_end(arena, &srl); - } - - String8 b_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = b_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - b_debug_s = str8_serial_end(arena, &srl); - } - - String8 b_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - String8 corrupt_pch_path = str8_lit("corrupt-pch-file-path.obj"); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = b_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + corrupt_pch_path.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, corrupt_pch_path); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - b_debug_t = str8_serial_end(arena, &srl); - } - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", a_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", b_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", b_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(write_data_to_file_path(a_obj_file_path, a_obj)); - T_Ok(write_data_to_file_path(b_obj_file_path, b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full /rad_type_server:foo.rrt a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash a.obj foo.rrt b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(debug_p_and_debug_t_in_obj) -{ - U32 pch_sig = 0xCAFEBABE; - String8 pch_obj_name = str8_lit("pch.obj"); - - String8 pch_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = pch_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - pch_debug_s = str8_serial_end(arena, &srl); - } - - String8 pch_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - pch_debug_p = str8_serial_end(arena, &srl); - } - - String8 pch_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 1, sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + pch_obj_name.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, pch_obj_name); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - pch_debug_t = str8_serial_end(arena, &srl); - } - - T_Ok(t_write_def_obj("pch.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", pch_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_t", ".debug$T", pch_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", pch_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full pch.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 output = g_errors; - B32 found_warning = 0; - String8 pch_obj_path = t_make_file_path(arena, str8_lit("pch.obj")); - String8 expected_line = str8f(arena, "Warning(%03d): %S: multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections", LNK_Warning_MultipleDebugTAndDebugP, pch_obj_path); - while (output.size) { - String8 line = t_chop_line(&output); - found_warning = str8_match(line, expected_line, StringMatchFlag_CaseInsensitive|StringMatchFlag_SlashInsensitive); - if (found_warning) { break; } - } - T_Ok(found_warning); -} - -TEST(merge_duplicate_types) -{ - { - U32 pch_sig = 0xCAFEBABE; - String8 pch_obj_name = str8_lit("pch.obj"); - String8 a_obj_name = str8_lit("a.obj"); - String8 c_obj_name = str8_lit("c.obj"); - - String8 pch_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = pch_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - pch_debug_s = str8_serial_end(arena, &srl); - } - String8 debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - debug_p = str8_serial_end(arena, &srl); - } - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = a_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - - String8 debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + pch_obj_name.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, pch_obj_name); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - debug_t = str8_serial_end(arena, &srl); - } - - String8 c_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPointer ptr = { .itype = CV_BasicType_SHORT }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - c_debug_t = str8_serial_end(arena, &srl); - } - - String8 c_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - // S_OBJNAME - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = c_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - // S_COMPILE3 - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - // S_LPROC32 - U16 *foo_size = str8_serial_push_size(arena, &srl, sizeof(*foo_size)); - U64 foo_off = srl.total_size; - str8_serial_push_u16(arena, &srl, CV_SymKind_LPROC32); - CV_SymProc32 *foo_proc = str8_serial_push_size(arena, &srl, sizeof(*foo_proc)); - foo_proc->itype = 0x1001; - foo_proc->sec = 1; - foo_proc->len = 1; - str8_serial_push_cstr(arena, &srl, str8_lit("foo")); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - *foo_size = srl.total_size - foo_off; - - // S_PROC_ID_END - str8_serial_push_u16(arena, &srl, 2); - str8_serial_push_u16(arena, &srl, CV_SymKind_END); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - // $$Symbols header - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - c_debug_s = str8_serial_end(arena, &srl); - } - - String8 pch_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", pch_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 c_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp(""), .flags = "rx:code" }, - { "debug_t", ".debug$T", c_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", c_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 entry_obj = t_make_entry_obj(arena); - - T_Ok(t_write_file(str8_lit("entry.obj"), entry_obj)); - T_Ok(t_write_file(str8_lit("pch.obj"), pch_obj)); - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - T_Ok(t_write_file(str8_lit("c.obj"), c_obj)); - - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe pch.obj a.obj b.obj c.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - } - - // load msf - String8 pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, pdb); - // find named streams - String8 info_data = msf_data_from_stream(msf, PDB_FixedStream_Info); - PDB_Info *pdb_info = pdb_info_from_data(arena, info_data); - PDB_NamedStreamTable *named_streams = pdb_named_stream_table_from_info(arena, pdb_info); - // find string table - MSF_StreamNumber strtbl_sn = named_streams->sn[PDB_NamedStream_StringTable]; - String8 strtbl_data = msf_data_from_stream(msf, strtbl_sn); - PDB_Strtbl *strtbl = pdb_strtbl_from_data(arena, strtbl_data); - // find TPI - String8 tpi_data = msf_data_from_stream(msf, PDB_FixedStream_Tpi); - PDB_TpiParsed *tpi = pdb_tpi_from_data(arena, tpi_data); - - U64 type_count = tpi->itype_opl - tpi->itype_first; - T_Ok(type_count == 5); - - CV_DebugT debug_t = cv_debug_t_from_data(arena, pdb_leaf_data_from_tpi(tpi), 4); - T_Ok(debug_t.count == type_count); - - { - CV_Leaf ptr_leaf = cv_debug_t_get_leaf(&debug_t, 0); - T_Ok(ptr_leaf.kind == CV_LeafKind_POINTER); - T_Ok(ptr_leaf.data.size == sizeof(CV_LeafPointer)); - - CV_LeafPointer *ptr = (CV_LeafPointer *)ptr_leaf.data.str; - T_Ok(ptr->itype == CV_BasicType_VOID); - T_Ok(ptr->attribs == 0); - } - - { - CV_Leaf proc_leaf = cv_debug_t_get_leaf(&debug_t, 1); - T_Ok(proc_leaf.kind == CV_LeafKind_PROCEDURE); - T_Ok(proc_leaf.data.size == sizeof(CV_LeafProcedure)); - - CV_LeafProcedure *proc = (CV_LeafProcedure *)proc_leaf.data.str; - T_Ok(proc->ret_itype == 0x1000); - T_Ok(proc->call_kind == CV_CallKind_NearPascal); - } - - { - CV_Leaf proc_leaf = cv_debug_t_get_leaf(&debug_t, 2); - T_Ok(proc_leaf.kind == CV_LeafKind_PROCEDURE); - T_Ok(proc_leaf.data.size == sizeof(CV_LeafProcedure)); - - CV_LeafProcedure *proc = (CV_LeafProcedure *)proc_leaf.data.str; - T_Ok(proc->ret_itype == 0x1000); - T_Ok(proc->call_kind == CV_CallKind_NearC); - } - - { - CV_Leaf ptr_leaf = cv_debug_t_get_leaf(&debug_t, 3); - T_Ok(ptr_leaf.kind == CV_LeafKind_POINTER); - T_Ok(ptr_leaf.data.size == sizeof(CV_LeafPointer)); - - CV_LeafPointer *ptr = (CV_LeafPointer *)ptr_leaf.data.str; - T_Ok(ptr->itype == CV_BasicType_SHORT); - T_Ok(ptr->attribs == 0); - } - - { - CV_Leaf proc_leaf = cv_debug_t_get_leaf(&debug_t, 4); - T_Ok(proc_leaf.kind == CV_LeafKind_PROCEDURE); - T_Ok(proc_leaf.data.size == sizeof(CV_LeafProcedure)); - - CV_LeafProcedure *proc = (CV_LeafProcedure *)proc_leaf.data.str; - T_Ok(proc->ret_itype == 0x1003); - T_Ok(proc->call_kind == CV_CallKind_NearC); - } -} - -TEST(cyclic_type) -{ - String8List *debug_t = push_array(arena, String8List, 1); - str8_serial_begin(arena, debug_t); - str8_serial_push_u32(arena, debug_t, CV_Signature_C13); - str8_serial_push_string(arena, debug_t, cv_make_leaf(arena, CV_LeafKind_POINTER, str8_struct((&(CV_LeafPointer){ .itype = 0x1001 })), CV_LeafAlign)); - str8_serial_push_string(arena, debug_t, cv_make_leaf(arena, CV_LeafKind_POINTER, str8_struct((&(CV_LeafPointer){ .itype = 0x1000 })), CV_LeafAlign)); - - CV_DebugS debug_s = {0}; - str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_make_symbol(arena, CV_SymKind_GPROC32, cv_make_proc32(arena, (CV_SymProc32){ .itype = 0x1001 }, str8_lit("foo")))); - String8List debug_s_string = cv_data_from_debug_s_c13(arena, &debug_s, 1); - - String8 raw_coff = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", str8_serial_end(arena, debug_t), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", str8_list_join(arena, &debug_s_string, 0), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_entry_obj()); - T_Ok(t_write_file(str8_lit("cycle.obj"), raw_coff)); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full /rad_ignore:-%u cycle.obj entry.obj", LNK_Error_InvalidTypeIndex); - T_Ok(g_last_exit_code == 0); - - String8 output = g_errors; - B32 is_cycle_detected = 0; - while (output.size && !is_cycle_detected) { - String8 line = str8_chop_line(&output); - is_cycle_detected = str8_match_wildcard(line, str8_lit("Error(*): *: LF_POINTER(type_index: *) forward refs member type index * (leaf struct offset: *)"), StringMatchFlag_CaseInsensitive); - t_chop_line(&output); - } - T_Ok(is_cycle_detected); -} - -TEST(get_msf_stream_pages) -{ - MSF_Context *msf = msf_alloc(MSF_DEFAULT_PAGE_SIZE, MSF_DEFAULT_FPM); - - { - U64 stream_size = MB(150) + 1; - - MSF_StreamNumber sn = msf_stream_alloc_ex(msf, stream_size); - - U8 *test = push_array(arena, U8, stream_size); - MemorySet(test, 0xca, stream_size/2); - MemorySet(test + stream_size/2, 0xbe, stream_size/2); - - String8List stream_data = msf_data_from_sn(arena, msf, sn); - T_Ok(stream_data.total_size == stream_size); - T_Ok(stream_data.node_count == 12); - - String8Array a = str8_array_from_list(arena, &stream_data); - T_Ok(a.v[0].size == 0xffd000); - T_Ok(a.v[1].size == 0xffe000); - T_Ok(a.v[2].size == 0xffe000); - T_Ok(a.v[3].size == 0xffe000); - T_Ok(a.v[4].size == 0xffe000); - T_Ok(a.v[5].size == 0xffe000); - T_Ok(a.v[6].size == 0xffe000); - T_Ok(a.v[7].size == 0xffd000); - T_Ok(a.v[8].size == 0x1000); - T_Ok(a.v[9].size == 0xffe000); - T_Ok(a.v[10].size == 0x613000); - T_Ok(a.v[11].size == 1); - - String8Node buf = *stream_data.first; - U64 buf_pos = 0; - str8_buffer_write(&buf, &buf_pos, str8(test, stream_size)); - - String8 cmp = msf_stream_read_block(arena, msf, sn, stream_size); - T_Ok(cmp.size == stream_size); - T_Ok(MemoryCompare(cmp.str, test, stream_size) == 0); - } - - { - MSF_StreamNumber sn = msf_stream_alloc_ex(msf, 1); - String8List stream_data = msf_data_from_sn(arena, msf, sn); - T_Ok(stream_data.node_count == 1); - T_Ok(stream_data.total_size == 1); - T_Ok(stream_data.first->string.size == 1); - } - - msf_release(msf); -} - -internal String8 -data_from_pdb(Arena *arena, PDB_Context *pdb) -{ - TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("foo")); - TP_Arena *tp_arena = tp_arena_alloc(tp); - pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0); - - AssertAlways(msf_build(pdb->msf) == MSF_Error_OK); - String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf); - AssertAlways(t_write_file_list(str8_lit("test.pdb"), raw_msf_list)); - - String8 data = str8_list_join(arena, &raw_msf_list, 0); - - tp_arena_release(&tp_arena); - tp_release(tp); - - return data; -} - -TEST(validate_info_stream) -{ - COFF_TimeStamp time_stamp = 123; - U32 age = 1; - Guid guid = { .data1 = max_U32, .data2 = max_U16 - 1, .data3 = max_U16 - 2, .data4 = { 1, 2, 3, 4, 5, 6, 7, 8 } }; - PDB_Context *pdb = pdb_alloc(MSF_DEFAULT_PAGE_SIZE, COFF_MachineType_X64, time_stamp, age, guid); - - char *stream_names[] = { "one", "two", "three", "four", "five" }; - MSF_StreamNumber stream_numbers[ArrayCount(stream_names)] = {0}; - - for EachElement(i, stream_names) { - stream_numbers[i] = pdb_push_named_stream(&pdb->info->named_stream_ht, pdb->msf, str8_cstring(stream_names[i])); - T_Ok(stream_numbers[i] != MSF_INVALID_STREAM_NUMBER); - } - - String8 raw_msf = data_from_pdb(arena, pdb); - MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_msf); - String8 info_data = msf_data_from_stream(msf_parsed, PDB_FixedStream_Info); - -#if 0 - fprintf(stderr, "\n"); - for EachIndex(i, info_data.size) { - fprintf(stderr, "0x%02x, ", info_data.str[i]); - if (i % 19 == 18 && i > 0) { fprintf(stderr, "\n"); } - } -#endif - U8 expected_info_data[] = { - 0x94, 0x2e, 0x31, 0x01, 0x7b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xfd, - 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x22, 0x00, 0x00, 0x00, 0x6f, 0x6e, 0x65, 0x00, 0x74, 0x77, - 0x6f, 0x00, 0x74, 0x68, 0x72, 0x65, 0x65, 0x00, 0x66, 0x6f, 0x75, 0x72, 0x00, 0x66, 0x69, 0x76, 0x65, 0x00, 0x2f, - 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x13, - 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x51, 0x33, 0x01, - }; - T_Ok(str8_match(info_data, str8_array_fixed(expected_info_data), 0)); - - pdb_release(pdb); -} - -TEST(validate_gsi) -{ - String8 raw_symbols[] = { - cv_make_symbol(arena, CV_SymKind_CONSTANT, cv_make_const(arena, (CV_SymConstant){ .itype = 0 }, 263, str8_lit("CV_SymKind_BLOCK16"))), - cv_make_symbol(arena, CV_SymKind_GDATA32, cv_make_data32(arena, (CV_SymData32) { .itype = 0, .off = 0x25440, .sec = 2 }, str8_lit("__newclmap"))), - cv_make_symbol(arena, CV_SymKind_GDATA32, cv_make_data32(arena, (CV_SymData32) { .itype = 0, .off = 123, .sec = 1 }, str8_lit("coffeebabe"))), - cv_make_symbol(arena, CV_SymKind_GDATA32, cv_make_data32(arena, (CV_SymData32) { .itype = 0, .off = 123, .sec = 1 }, str8_lit("deadbeef"))), - }; - - CV_Symbol symbols[ArrayCount(raw_symbols)] = {0}; - for EachElement(i, raw_symbols) { symbols[i] = cv_symbol_from_ptr(raw_symbols[i].str); } - - CV_DebugS debug_s = {0}; - for EachElement(i, symbols) { str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbols[i], CV_SymbolAlign)); } - String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); - String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); - - T_Ok(t_write_def_obj("debug.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - String8 gsi_data = msf_data_from_stream(msf, dbi->gsi_sn); - PDB_GsiParsed *gsi = pdb_gsi_from_data(arena, gsi_data); - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - - for EachElement(i, symbols) { - CV_Symbol symbol = symbols[i]; - - String8 string = cv_name_from_symbol(symbols[i].kind, symbols[i].data); - U64 symbol_off = pdb_gsi_symbol_from_string(gsi, symbol_data, string); - T_Ok(symbol_off < symbol_data.size); - - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - T_Ok(test_symbol_size > 0); - - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(str8_match(test_symbol_name, string, 0)); - } -} - -TEST(validate_gsi_procs_and_typedefs) -{ - String8 raw_symbols[] = { - cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), - cv_make_symbol(arena, CV_SymKind_GPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("global_proc"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("global_typedef"))), - - cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), - cv_make_symbol(arena, CV_SymKind_LPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("local_proc"))), - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("local_typedef"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - }; - - CV_Symbol symbols[ArrayCount(raw_symbols)] = {0}; - for EachElement(i, raw_symbols) { symbols[i] = cv_symbol_from_ptr(raw_symbols[i].str); } - - CV_DebugS debug_s = {0}; - for EachElement(i, symbols) { str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbols[i], CV_SymbolAlign)); } - String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); - String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); - - T_Ok(t_write_def_obj("debug.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - String8 gsi_data = msf_data_from_stream(msf, dbi->gsi_sn); - PDB_GsiParsed *gsi = pdb_gsi_from_data(arena, gsi_data); - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - - struct { - char *name; - B32 is_global; - U64 offset; - U64 imod; - } procs[] = { - { "global_proc", 1, 0x18, 2, }, - { "local_proc", 0, 0x64, 2, }, - }; - for EachElement(i, procs) { - U64 symbol_off = pdb_gsi_symbol_from_string(gsi, symbol_data, str8_cstring(procs[i].name)); - T_Ok(symbol_off < symbol_data.size); - - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(test_symbol_size > 0); - T_Ok(str8_match(str8_cstring(procs[i].name), test_symbol_name, 0)); - T_Ok(test_symbol.kind == (procs[i].is_global ? CV_SymKind_PROCREF : CV_SymKind_LPROCREF)); - - CV_SymRef2 *proc_ref = str8_deserial_get_raw_ptr(test_symbol.data, 0, sizeof(*proc_ref)); - T_Ok(proc_ref->suc_name == 0); - T_Ok(proc_ref->sym_off == procs[i].offset); - T_Ok(proc_ref->imod == procs[i].imod); - } - - struct { - char *name; - B32 is_global; - } typedefs[] = { - { "global_typedef", 1 }, - { "local_typedef", 0 }, - }; - for EachElement(i, typedefs) { - U64 symbol_off = pdb_gsi_symbol_from_string(gsi, symbol_data, str8_cstring(typedefs[i].name)); - if (typedefs[i].is_global) { - T_Ok(symbol_off < symbol_data.size); - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(test_symbol_size > 0); - T_Ok(str8_match(str8_cstring(typedefs[i].name), test_symbol_name, 0)); - T_Ok(test_symbol.kind == CV_SymKind_UDT); - } else { - T_Ok(symbol_off >= symbol_data.size); - } - } -} - -TEST(validate_psi) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit("FOOBAR"), .flags = "rx:code" }, - { "data", ".data", str8_lit("QWE"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("global_func", "text", 1), - T_COFF_DefSymbol_Extern("global_var", "data", 1), - T_COFF_DefSymbol_Static("static_var", "data", 1), - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - String8 psi_data = msf_data_from_stream(msf, dbi->psi_sn); - String8 psi_gsi_data = str8_range(psi_data.str + sizeof(PDB_PsiHeader), psi_data.str+psi_data.size); - PDB_GsiParsed *psi = pdb_gsi_from_data(arena, psi_gsi_data); - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - - struct { - char *name; - B32 is_global; - CV_Pub32Flags flags; - } pubs[] = { - { "global_func", 1, CV_Pub32Flag_Function }, - { "global_var", 1, 0 }, - { "static_var", 0, 0 }, - }; - for EachElement(i, pubs) { - U64 symbol_off = pdb_gsi_symbol_from_string(psi, symbol_data, str8_cstring(pubs[i].name)); - if (pubs[i].is_global) { - T_Ok(symbol_off < symbol_data.size); - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(test_symbol_size > 0); - T_Ok(str8_match(str8_cstring(pubs[i].name), test_symbol_name, 0)); - T_Ok(test_symbol.kind == CV_SymKind_PUB32); - CV_SymPub32 *pub32 = str8_deserial_get_raw_ptr(test_symbol.data, 0, sizeof(*pub32)); - T_Ok(pub32->sec > 0); - T_Ok(pubs[i].flags == pub32->flags); - } else { - T_Ok(symbol_off >= symbol_data.size); - } - } - - U64 pub32_count = 0; - for EachElement(i, psi->buckets) { - PDB_GsiBucket bucket = psi->buckets[i]; - for EachIndex(k, bucket.count) { - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, bucket.offs[k]), 0, 1, &test_symbol); - T_Ok(test_symbol_size >= sizeof(CV_SymPub32)); - T_Ok(test_symbol.kind == CV_SymKind_PUB32); - pub32_count += 1; - } - } - T_Ok(pub32_count > 0); -} - -TEST(psi_addr_map_radix_sort) -{ - String8 names[] = { - str8_lit("alpha"), - str8_lit("bravo"), - str8_lit("charlie"), - str8_lit("delta"), - }; - U64 address_count = (1 << 15) + 1; - U64 record_count = address_count * ArrayCount(names); - - PDB_GsiSortRecord *records = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); - PDB_GsiSortRecord *expected = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); - for EachIndex(i, record_count) { - U64 address_idx = address_count - 1 - i / ArrayCount(names); - records[i].isect_off.isect = 1 + address_idx % 257; - records[i].isect_off.off = address_idx / 257; - records[i].name = names[ArrayCount(names) - 1 - i % ArrayCount(names)]; - records[i].offset = i * sizeof(U32); - } - MemoryCopyTyped(expected, records, record_count); - radsort(expected, record_count, psi_addr_map_compar_is_before); - - TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("psi addr map sort test")); - U32 *addr_map = psi_addr_map_from_gsi_records(tp, arena, records, record_count); - - for EachIndex(i, record_count) { - T_Ok(addr_map[i] == expected[i].offset); - } - tp_release(tp); -} - -TEST(u64_array_radix_sort_parallel) -{ - TP_Context *tp = tp_alloc(arena, 1, 1, str8_zero()); - - U64 small_values[] = { max_U64, 0, 7, 1, 7, 0x100000000ull, 2 }; - u64_array_sort_radix_parallel(tp, ArrayCount(small_values), small_values); - for (U64 i = 1; i < ArrayCount(small_values); i += 1) { - T_Ok(small_values[i-1] <= small_values[i]); - } - - U64 count = 200003; - U64 *values = push_array_no_zero(arena, U64, count); - U64 state = 0x9e3779b97f4a7c15ull; - U64 sum_before = 0; - U64 xor_before = 0; - for EachIndex(i, count) { - state ^= state >> 12; - state ^= state << 25; - state ^= state >> 27; - values[i] = state * 0x2545f4914f6cdd1dull; - if ((i % 97) == 0) { values[i] = 0; } - if ((i % 193) == 0) { values[i] = max_U64; } - if ((i % 389) == 0) { values[i] = 0x100000001ull; } - sum_before += values[i]; - xor_before ^= values[i]; - } - - u64_array_sort_radix_parallel(tp, count, values); - U64 sum_after = 0; - U64 xor_after = 0; - B32 is_sorted = 1; - for EachIndex(i, count) { - if (i > 0 && values[i-1] > values[i]) { is_sorted = 0; } - sum_after += values[i]; - xor_after ^= values[i]; - } - T_Ok(is_sorted); - T_Ok(sum_before == sum_after); - T_Ok(xor_before == xor_after); - tp_release(tp); -} - -TEST(pdbstripped) -{ - String8 debug_obj; - { - String8 raw_symbols[] = { - cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), - cv_make_symbol(arena, CV_SymKind_GPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("global_proc"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("global_typedef"))), - - cv_make_symbol(arena, CV_SymKind_LPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("local_proc"))), - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("local_typedef"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - }; - - CV_Symbol symbols[ArrayCount(raw_symbols)] = {0}; - for EachElement(i, raw_symbols) { symbols[i] = cv_symbol_from_ptr(raw_symbols[i].str); } - - CV_DebugS debug_s = {0}; - for EachElement(i, symbols) { str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbols[i], CV_SymbolAlign)); } - String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); - String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); - - debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - } - - String8 pub_obj; - { - pub_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit("FOOBAR"), .flags = "rx:code" }, - { "data", ".data", str8_lit("QWE"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("global_func", "text", 1), - T_COFF_DefSymbol_Extern("global_var", "data", 1), - T_COFF_DefSymbol_Static("static_var", "data", 1), - {0} - } - }); - } - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_file(str8_lit("pub.obj"), pub_obj)); - T_Ok(t_write_entry_obj()); - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe /pdbstripped:a.stripped.pdb entry.obj pub.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.stripped.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - - String8 mods_data = pdb_data_from_dbi_range(dbi, PDB_DbiRange_ModuleInfo); - PDB_CompUnitArray *mods = pdb_comp_unit_array_from_data(arena, mods_data); - T_Ok(mods->count > 0); - - // modules must contain only stubs for static procs - for EachIndex(i, mods->count) { - PDB_CompUnit *mod = mods->units[i]; - U64 sym_data_size = mod->range_off[PDB_DbiCompUnitRange_Symbols + 1] - mod->range_off[PDB_DbiCompUnitRange_Symbols]; - U64 c11_data_size = mod->range_off[PDB_DbiCompUnitRange_C11 + 1] - mod->range_off[PDB_DbiCompUnitRange_C11]; - U64 c13_data_size = mod->range_off[PDB_DbiCompUnitRange_C13 + 1] - mod->range_off[PDB_DbiCompUnitRange_C13]; - T_Ok(c11_data_size == 0); - T_Ok(c13_data_size == 0); - if (str8_match(str8_skip_last_slash(mod->obj_name), str8_lit("debug.obj"), 0)) { - T_Ok(sym_data_size > 0); - } else { - T_Ok(sym_data_size == 0); - } - - String8 mod_data = msf_data_from_stream(msf, mod->sn); - String8 sym_data = str8_substr(mod_data, r1u64(mod->range_off[PDB_DbiCompUnitRange_Symbols], mod->range_off[PDB_DbiCompUnitRange_Symbols + 1])); - for (U64 cursor = 0; cursor < sym_data_size; ) { - CV_Symbol symbol = {0}; - U64 read_size = cv_read_symbol(sym_data, cursor, PDB_SYMBOL_ALIGN, &symbol); - T_Ok(read_size > 0); - cursor += read_size; - T_Ok(symbol.kind == CV_SymKind_LPROC32 || symbol.kind == CV_SymKind_END); - } - } - - // global symbol stream must have public and references to the static stubs - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - for (U64 cursor = 0; cursor < symbol_data.size; ) { - CV_Symbol symbol = {0}; - U64 read_size = cv_read_symbol(symbol_data, cursor, PDB_SYMBOL_ALIGN, &symbol); - T_Ok(read_size > 0); - cursor += read_size; - T_Ok(symbol.kind == CV_SymKind_PUB32 || - symbol.kind == CV_SymKind_LPROCREF); - } - - // types must be stripped - String8 tpi = msf_data_from_stream(msf, PDB_FixedStream_Tpi); - T_Ok(tpi.size == sizeof(PDB_TpiHeader)); - String8 ipi = msf_data_from_stream(msf, PDB_FixedStream_Ipi); - T_Ok(ipi.size == sizeof(PDB_TpiHeader)); -} - -TEST(ghash_check_corrupt) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: .debug$H section is too small to contain the header"); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_check_magic) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - -#if 0 - U64 *hashes = push_array(arena, U64, t.node_count); - U64 i = 0; - for EachNode(n, String8Node, t.first->next) { - blake3(&hashes[i], sizeof(hashes[i]), n->string.str, n->string.size); - i += 1; - } -#endif - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = 123, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: .debug$H contains invalid magic: got 0x7b, expected 0x%x", LLVM_GHash_Magic); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_check_version) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - -#if 0 - U64 *hashes = push_array(arena, U64, t.node_count); - U64 i = 0; - for EachNode(n, String8Node, t.first->next) { - blake3(&hashes[i], sizeof(hashes[i]), n->string.str, n->string.size); - i += 1; - } -#endif - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = 0xbeef })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: mismatched .debug$H version: got %u, expected %u", 0xbeef, LLVM_GHash_CurrentVersion); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_check_hash_alg) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_SHA1_8, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: mismatched .debug$H hash algorithm: got SHA1_8, expected *"); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_match_debug_t) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - //str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - String8 output = {0}; - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: mismatched .debug$H hash count and type count: got 3 hashes for 2 types"); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_basic) -{ - String8 a_obj; - { - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 1); - str8_serial_push_u64(arena, &h, 2); - str8_serial_push_u64(arena, &h, 3); - String8 debug_h = str8_serial_end(arena, &h); - - a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - } - - String8 b_obj; - { - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 4); - str8_serial_push_u64(arena, &h, 5); - str8_serial_push_u64(arena, &h, 6); - String8 debug_h = str8_serial_end(arena, &h); - - b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - } - - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_pdb); - String8 raw_tpi = msf_data_from_stream(msf_parsed, PDB_FixedStream_Tpi); - PDB_TpiParsed *tpi = pdb_tpi_from_data(arena, raw_tpi); - U64 leaf_count = tpi->itype_opl - tpi->itype_first; - T_Ok(leaf_count == 6); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_pdb); - String8 raw_tpi = msf_data_from_stream(msf_parsed, PDB_FixedStream_Tpi); - PDB_TpiParsed *tpi = pdb_tpi_from_data(arena, raw_tpi); - U64 leaf_count = tpi->itype_opl - tpi->itype_first; - T_Ok(leaf_count == 3); - } -} - -TEST(patch_cv_symbol_tree) -{ - String8List raw_symbols = {0}; - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("foo.obj"), 123))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_GPROC32, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("Proc")))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE, cv_make_inline_site(arena, (CV_SymInlineSite){0}, str8_zero()))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE_END, cv_make_inline_site_end(arena))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_END, cv_make_end(arena))); - - U64 tree_size = cv_patch_symbol_tree_offsets(raw_symbols, sizeof(CV_Signature), 4); - T_Ok(tree_size == 84); - - { - String8Node buf = *raw_symbols.first; - U64 buf_pos = 0; - - CV_SymbolHeader obj_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(obj_header), &obj_header) == sizeof(obj_header)); - T_Ok(obj_header.kind == CV_SymKind_OBJNAME); - T_Ok(str8_buffer_skip(&buf, &buf_pos, obj_header.size - sizeof(CV_SymKind))); - - CV_SymbolHeader proc_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(proc_header), &proc_header) == sizeof(proc_header)); - T_Ok(proc_header.kind == CV_SymKind_GPROC32); - - CV_SymProc32 proc; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(proc), &proc) == sizeof(proc)); - T_Ok(proc.end == 0x54); - T_Ok(str8_buffer_skip(&buf, &buf_pos, proc_header.size - sizeof(CV_SymKind) - sizeof(proc))); - - CV_SymbolHeader inline_site_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site_header), &inline_site_header) == sizeof(inline_site_header)); - T_Ok(inline_site_header.kind == CV_SymKind_INLINESITE); - - CV_SymInlineSite inline_site; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site), &inline_site)); - T_Ok(inline_site.parent == 0x14); - T_Ok(inline_site.end == 0x50); - T_Ok(str8_buffer_skip(&buf, &buf_pos, inline_site_header.size - sizeof(CV_SymKind) - sizeof(inline_site))); - - CV_SymbolHeader inline_end_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(inline_end_header), &inline_end_header) == sizeof(inline_end_header)); - T_Ok(inline_end_header.kind == CV_SymKind_INLINESITE_END); - - CV_SymbolHeader proc_end_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(proc_end_header), &proc_end_header) == sizeof(proc_end_header)); - T_Ok(proc_end_header.kind == CV_SymKind_END); - - T_Ok(buf.string.size == 0); - T_Ok(buf.string.str == 0); - T_Ok(buf_pos == 0); - } -} - -TEST(whole_archive) -{ - T_Ok(t_write_entry_obj()); - T_Ok(t_write_def_lib("a.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("a.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data" }, - {0} - } - } - }, - {0} - } - })); - T_Ok(t_write_def_lib("b.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("b.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("b"), .flags = "rw:data" }, - {0} - } - } - }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:all_libs.exe entry.obj /wholearchive a.lib b.lib"); - T_Ok(g_last_exit_code == 0); - { - String8 exe = t_read_file(arena, str8_lit("all_libs.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a_sect != 0); - T_Ok(b_sect != 0); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:only_a.exe entry.obj /wholearchive:a.lib a.lib b.lib"); - T_Ok(g_last_exit_code == 0); - { - String8 exe = t_read_file(arena, str8_lit("only_a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a_sect != 0); - T_Ok(b_sect == 0); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:only_b.exe /wholearchive:b.lib a.lib b.lib entry.obj"); - T_Ok(g_last_exit_code == 0); - { - String8 exe = t_read_file(arena, str8_lit("only_b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a_sect == 0); - T_Ok(b_sect != 0); - } -} -#if OS_WINDOWS - -internal B32 -t_radlink_validate_asan_out(String8 obj_name) -{ - Temp scratch = scratch_begin(0,0); - B32 is_ok = 0; - - t_invoke_linkerf("%S /debug:full", obj_name); - if (g_last_exit_code != 0) { goto exit; } - - String8 exe_path = t_make_file_path(scratch.arena, str8f(scratch.arena, "%S.exe", str8_chop_last_dot(obj_name))); - - char *old_path_cstr = getenv("PATH"); - String8List env = {0}; - str8_list_pushf(scratch.arena, &env, "PATH=%S;%S", str8_chop_last_slash(t_cl_path()), str8_cstring(old_path_cstr)); - t_invoke_env(exe_path, str8_zero(), env, max_U64); - - String8 s = g_errors; - - String8 header = t_chop_line(&s); - if ( ! str8_match(header, str8_lit("================================================================="), 0)) { - goto exit; - } - - String8 cause = t_chop_line(&s); - if ( str8_find_needle(cause, 0, str8_lit("AddressSanitizer: heap-use-after-free on address"), 0) >= cause.size) { - goto exit; - } - - is_ok = 1; - exit:; - scratch_end(scratch); - return is_ok; -} - -TEST(infer_asan) -{ - char *program = - "#include \n" - " int main(void) {\n" - "int *foo = malloc(sizeof(*foo));\n" - "free(foo);\n" - "*foo = 1;\n" - "}\n" - ; - - // /MD - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MD /fsanitize=address /Z7 /c /Fo:main_md.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_md.obj"))); - } - - // /MDd - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MDd /fsanitize=address /Z7 /c /Fo:main_mdd.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_mdd.obj"))); - } - - // /MT - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MT /fsanitize=address /Z7 /c /Fo:main_mt.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_mt.obj"))); - } - - // /MTd - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MT /fsanitize=address /Z7 /c /Fo:main_mtd.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_mtd.obj"))); - } -} - -#endif - -#if OS_WINDOWS -TEST(determ_test) -{ - // compile the test target (torture) - t_invoke_cl("/fsanitize=address /c /Z7 /Fo:test.obj -I%S /Zc:preprocessor %S/torture/torture_main.c", t_src_path(), t_src_path()); - T_Ok(g_last_exit_code == 0); - - U64 run_count = 25; - T_Ok(run_count > 1); - String8 test_path = t_make_file_path(arena, str8_lit("test.obj")); - - // single-threaded link - t_invoke_linkerf("%S /debug:full /rad_time_stamp:0 /rad_workers:1 /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74 /out:main.exe", test_path); - T_Ok(g_last_exit_code == 0); - - // read b - String8 main_exe = t_read_file(arena, str8_lit("main.exe")); - String8 main_pdb = t_read_file(arena, str8_lit("main.pdb")); - - // multi-threaded links - ProcessList linkers = {0}; - for EachIndex(i, run_count) { - String8 out_path = t_make_file_path(arena, str8f(arena, "%llu.exe", i)); - String8 cmdl = str8f(arena, "%S %S /debug:full /rad_time_stamp:0 /rad_imagealtpath:main.exe /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74 /out:%S", t_radlink_path(), test_path, out_path); - Process process_handle = launch_cmd_line(cmdl); - T_Ok(!process_match(process_zero(), process_handle)); - process_list_push(arena, &linkers, process_handle); - } - - // wait for linkers - for EachNode(n, ProcessNode, linkers.first) { process_join(n->v, max_U64, 0); } - - for EachIndex(i, run_count) { - Temp temp = temp_begin(arena); - String8 exe = t_read_file(temp.arena, str8f(temp.arena, "%llu.exe", i)); - String8 pdb = t_read_file(temp.arena, str8f(temp.arena, "%llu.pdb", i)); - T_Ok(exe.size); - T_Ok(pdb.size); - T_Ok(str8_match(main_exe, exe, 0)); - T_Ok(str8_match(main_pdb, pdb, 0)); - temp_end(temp); - } -} - -#endif - -internal B32 t_read_exe_data_vaddrs(Arena *arena, String8 exe_path, U64 *vaddrs, U64 count); - -#if OS_WINDOWS -TEST(ms_link_icfs_identical_comdats) -{ - U8 same_text[] = { - 0x48, 0x31, 0xc0, // xor rax, rax - 0xc3 // ret - }; - U8 entry_text[] = { - 0xc3, // ret - }; - U8 addresses[] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - }; - - T_Ok(t_write_def_obj("ms_icf.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text$mn", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "a", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "a"), - T_COFF_DefReloc(X64_Addr64, 8, "b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("a", "a", 0), - T_COFF_DefSymbol_ExternFunc("b", "b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - B32 is_invoke_ok = t_invoke(str8_lit("link.exe"), str8_lit("/nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf.exe /opt:ref,icf /include:a /include:b /include:addresses ms_icf.obj"), max_U64); - T_Ok(is_invoke_ok); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("ms_icf.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 a_addr = 0; - U64 b_addr = 0; - str8_deserial_read_struct(data, 0, &a_addr); - str8_deserial_read_struct(data, 8, &b_addr); - T_Ok(a_addr != 0); - T_Ok(a_addr == b_addr); // COMDAT are folded -} - -#if 0 -TEST(ms_link_icf_section_flag_eligibility) -{ - U8 ret_text[] = { - 0xc3, // ret - }; - U8 data_bytes[] = { - 1, 2, 3, 4, 5, 6, 7, 8, - }; - U8 addresses[] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - }; - - T_Ok(t_write_def_obj("ms_icf_flags.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "code_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 8, "code_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 16, "code_plain_a"), - T_COFF_DefReloc(X64_Addr64, 24, "code_plain_b"), - T_COFF_DefReloc(X64_Addr64, 32, "rdata_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 40, "rdata_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 48, "wdata_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 56, "wdata_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 64, "ro_code_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 72, "ro_code_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 80, "rw_code_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 88, "rw_code_comdat_b"), - {0} - } - }, - { "code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "code_plain_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { "code_plain_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { "ro_code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "r:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ro_code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "r:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rw_code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rw:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rw_code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rw:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rdata_comdat_a", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rdata_comdat_b", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "wdata_comdat_a", ".data$mn", str8_array_fixed(data_bytes), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "wdata_comdat_b", ".data$mn", str8_array_fixed(data_bytes), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("code_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("code_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rdata_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rdata_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("wdata_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("wdata_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("ro_code_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("ro_code_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rw_code_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rw_code_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("code_comdat_a", "code_comdat_a", 0), - T_COFF_DefSymbol_ExternFunc("code_comdat_b", "code_comdat_b", 0), - T_COFF_DefSymbol_ExternFunc("code_plain_a", "code_plain_a", 0), - T_COFF_DefSymbol_ExternFunc("code_plain_b", "code_plain_b", 0), - T_COFF_DefSymbol_ExternFunc("ro_code_comdat_a", "ro_code_comdat_a", 0), - T_COFF_DefSymbol_ExternFunc("ro_code_comdat_b", "ro_code_comdat_b", 0), - T_COFF_DefSymbol_ExternFunc("rw_code_comdat_a", "rw_code_comdat_a", 0), - T_COFF_DefSymbol_ExternFunc("rw_code_comdat_b", "rw_code_comdat_b", 0), - T_COFF_DefSymbol_Extern("rdata_comdat_a", "rdata_comdat_a", 0), - T_COFF_DefSymbol_Extern("rdata_comdat_b", "rdata_comdat_b", 0), - T_COFF_DefSymbol_Extern("wdata_comdat_a", "wdata_comdat_a", 0), - T_COFF_DefSymbol_Extern("wdata_comdat_b", "wdata_comdat_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - B32 is_invoke_ok = t_invoke(str8_lit("link.exe"), str8_lit("/nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf_flags.exe /opt:ref,icf /include:addresses ms_icf_flags.obj"), max_U64); - T_Ok(is_invoke_ok); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("ms_icf_flags.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 code_comdat_a_addr = 0; - U64 code_comdat_b_addr = 0; - U64 code_plain_a_addr = 0; - U64 code_plain_b_addr = 0; - U64 rdata_comdat_a_addr = 0; - U64 rdata_comdat_b_addr = 0; - U64 wdata_comdat_a_addr = 0; - U64 wdata_comdat_b_addr = 0; - U64 ro_code_comdat_a_addr = 0; - U64 ro_code_comdat_b_addr = 0; - U64 rw_code_comdat_a_addr = 0; - U64 rw_code_comdat_b_addr = 0; - str8_deserial_read_struct(data, 0, &code_comdat_a_addr); - str8_deserial_read_struct(data, 8, &code_comdat_b_addr); - str8_deserial_read_struct(data, 16, &code_plain_a_addr); - str8_deserial_read_struct(data, 24, &code_plain_b_addr); - str8_deserial_read_struct(data, 32, &rdata_comdat_a_addr); - str8_deserial_read_struct(data, 40, &rdata_comdat_b_addr); - str8_deserial_read_struct(data, 48, &wdata_comdat_a_addr); - str8_deserial_read_struct(data, 56, &wdata_comdat_b_addr); - str8_deserial_read_struct(data, 64, &ro_code_comdat_a_addr); - str8_deserial_read_struct(data, 72, &ro_code_comdat_b_addr); - str8_deserial_read_struct(data, 80, &rw_code_comdat_a_addr); - str8_deserial_read_struct(data, 88, &rw_code_comdat_b_addr); - - T_Ok(code_comdat_a_addr != 0); - T_Ok(code_comdat_a_addr == code_comdat_b_addr); // executable code COMDATs fold - T_Ok(ro_code_comdat_a_addr == ro_code_comdat_b_addr); // read-only code COMDATs fold - T_Ok(code_plain_a_addr != code_plain_b_addr); // non-COMDAT code does not fold - T_Ok(wdata_comdat_a_addr != wdata_comdat_b_addr); // writable data COMDATs do not fold - T_Ok(rw_code_comdat_a_addr != rw_code_comdat_b_addr); // writable code COMDATs do not - T_Ok(rdata_comdat_a_addr == rdata_comdat_b_addr); // read-only data COMDATs fold -} -#endif - -#endif - -#if 1 - -internal B32 -t_read_exe_data_vaddrs(Arena *arena, String8 exe_path, U64 *vaddrs, U64 count) -{ - B32 result = 0; - String8 exe = t_read_file(arena, exe_path); - if (exe.size) { - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - if (data_section != 0 && data_section->foff + count*sizeof(U64) <= exe.size) { - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + count*sizeof(U64))); - result = str8_deserial_read_array(data, 0, vaddrs, count); - } - } - return result; -} - -TEST(icf_fold_two_funcs) -{ - U8 same_text[] = { - 0x48, 0x31, 0xc0, // xor rax, rax - 0xc3 // ret - }; - U8 entry_text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, // call a - 0xe8, 0x00, 0x00, 0x00, 0x00, // call b - 0xc3, // ret - }; - T_Ok(t_write_def_obj("ident_funcs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text$mb", str8_array_fixed(same_text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("a", "a", 0), - T_COFF_DefSymbol_ExternFunc("b", "b", 0), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf ident_funcs.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".text")); - - // validate .text header - T_Ok(text_sect->voff == 0x1000); - T_Ok(text_sect->vsize >= 0x14); - T_Ok(text_sect->fsize == 0x200); - - T_Ok(text_sect->foff + text_sect->vsize <= exe.size); - String8 text_data = str8_substr(exe, r1u64(text_sect->foff, text_sect->foff + 0x14)); - - U8 expected_text[] = { - // entry - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xe8, 0x06, 0x00, 0x00, 0x00, - 0xc3, - - // pad - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - - // a and b folded - 0x48, 0x31, 0xc0, - 0xc3, - }; - T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0)); -} - -TEST(icf_associative_child_prevents_fold) -{ - U8 ret_text[] = { 0xc3 }; - U8 handler_a[] = { 1, 2, 3, 4 }; - U8 handler_b[] = { 4, 3, 2, 1 }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_associative_child.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { "fn_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "fn_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "handler_a", ".xdata", str8_array_fixed(handler_a), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "handler_b", ".xdata", str8_array_fixed(handler_b), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "fn_a"), - T_COFF_DefReloc(X64_Addr64, sizeof(U64), "fn_b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Associative("handler_a", "fn_a"), - T_COFF_DefSymbol_Associative("handler_b", "fn_b"), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), - T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_associative_child.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[1] != 0); - T_Ok(vaddrs[0] != vaddrs[1]); -} - -TEST(icf_comdat_reloc_targets_fold) -{ - U8 fn_text[] = { - 0x48, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, // mov rax, shared - 0xc3, // ret - }; - U8 shared_data[] = { 0 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_comdat_reloc_a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn_a", ".text$mn", str8_array_fixed(fn_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 2, "shared_local_a"), {0} } }, - { "shared", ".rdata", str8_array_fixed(shared_data), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("shared", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), - T_COFF_DefSymbol_Extern("shared", "shared", 0), - T_COFF_DefSymbol_Static("shared_local_a", "shared", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_comdat_reloc_b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn_b", ".text$mn", str8_array_fixed(fn_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 2, "shared_local_b"), {0} } }, - { "shared", ".rdata", str8_array_fixed(shared_data), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("shared", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("shared", "shared", 0), - T_COFF_DefSymbol_Static("shared_local_b", "shared", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_comdat_reloc_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "fn_a"), T_COFF_DefReloc(X64_Addr64, sizeof(U64), "fn_b"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_UndefFunc("fn_a"), - T_COFF_DefSymbol_UndefFunc("fn_b"), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_comdat_reloc_entry.obj icf_comdat_reloc_a.obj icf_comdat_reloc_b.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); -} - -TEST(icf_same_but_different) -{ - U8 text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, // call $ - 0xc3 - }; - U8 return_1[] = { - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3 // ret - }; - U8 return_2[] = { - 0x48, 0xc7, 0xc0, 0x02, 0x00, 0x00, 0x00, // mov rax, 2 - 0xc3 // ret - }; - U8 call_a_and_b[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "entry", ".text", str8_array_fixed(call_a_and_b), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - { - "a", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "c"), - {0} - } - }, - { - "b", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "d"), - {0} - } - }, - { "c", ".text", str8_array_fixed(return_1), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "d", ".text", str8_array_fixed(return_2), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("d", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "entry", 0), - T_COFF_DefSymbol_Extern("a", "a", 0), - T_COFF_DefSymbol_Extern("b", "b", 0), - T_COFF_DefSymbol_Extern("c", "c", 0), - T_COFF_DefSymbol_Extern("d", "d", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - // validate output - { - U8 expected_text[] = { - 0xe8, 0x0b, 0x00, 0x00, 0x00, // call a - 0xe8, 0x16, 0x00, 0x00, 0x00, // call b - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xe8, 0x1b, 0x00, 0x00, 0x00, // call c - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xe8, 0x1b, 0x00, 0x00, 0x00, // call d - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0x48, 0xc7, 0xc0, 0x02, 0x00, 0x00, 0x00, // mov rax, 2 - 0xc3, - }; - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -TEST(icf_fold_diamond) -{ - U8 call_b_and_c[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - U8 call_and_return[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - U8 clear_and_return[] = { - 0x48, 0x31, 0xc0, - 0xc3 - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "a", ".text", str8_array_fixed(call_b_and_c), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "b"), - T_COFF_DefReloc(X64_Rel32, 6, "c"), - {0} - } - }, - { - "b", ".text", str8_array_fixed(call_and_return), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "d"), - {0} - } - }, - { - "c", ".text", str8_array_fixed(call_and_return), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "d"), - {0} - } - }, - { "d", ".text", str8_array_fixed(clear_and_return), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("d", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("a", "a", 0), - T_COFF_DefSymbol_Extern("b", "b", 0), - T_COFF_DefSymbol_Extern("c", "c", 0), - T_COFF_DefSymbol_Extern("d", "d", 0), - {0} - } - })); - t_invoke_linkerf("/subsystem:console /entry:a /out:a.exe /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - // validate output - { - U8 expected_text[] = { - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xe8, 0x06, 0x00, 0x00, 0x00, - 0xc3, - - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xc3, - - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - - 0x48, 0x31, 0xc0, - 0xc3 - }; - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -TEST(icf_cyclic_icf) -{ - U8 text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "a", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "b"), - {0} - } - }, - { - "b", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("a", "a", 0), - T_COFF_DefSymbol_Static("b", "b", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:a /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - // validate output - { - U8 expected_text[] = { - 0xe8, 0xfb, 0xff, 0xff, 0xff, // a and b folded into a self-call - 0xc3, - }; - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -// ICF must preserve identical sections with different symbol targets -TEST(icf_reloc_target_symbol_types_do_not_fold) -{ - U8 reloc_text[] = { - 0x48, 0xc7, 0xc0, 0x00, 0x00, 0x00, 0x00, // mov rax, target - 0xc3 // ret - }; - U8 ret_text[] = { - 0xc3, // ret - }; - U8 target_data[] = { - 0x00, - }; - U8 addresses[3 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_interp_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "fn_regular"), - T_COFF_DefReloc(X64_Addr64, 8, "fn_common"), - T_COFF_DefReloc(X64_Addr64, 16, "fn_abs"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - T_COFF_DefSymbol_Undef("fn_regular"), - T_COFF_DefSymbol_Undef("fn_common"), - T_COFF_DefSymbol_Undef("fn_abs"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_interp_regular.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_regular"), {0} } }, - { "target", ".rdata$mn", str8_array_fixed(target_data), .flags = "r:data@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_regular", "fn", 0), - T_COFF_DefSymbol_Extern("target_regular", "target", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_interp_common.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_common"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_common", "fn", 0), - T_COFF_DefSymbol_Common("target_common", 8), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_interp_abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_abs"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_abs", "fn", 0), - T_COFF_DefSymbol_AbsExtern("target_abs", 0x1234), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_interp_entry.obj icf_interp_regular.obj icf_interp_common.obj icf_interp_abs.obj"); - T_Ok(g_last_exit_code == 0); - - U64 fn_vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), fn_vaddrs, ArrayCount(fn_vaddrs))); - - for EachElement(i, fn_vaddrs) { - for (U64 j = i + 1; j < ArrayCount(fn_vaddrs); j += 1) { - T_Ok(fn_vaddrs[i] != fn_vaddrs[j]); - } - } -} - -// ICF must preserve sections with unresolved target symbols (with /FORCE) -TEST(icf_unresolved_reloc_targets_do_not_fold) -{ - U8 reloc_text[] = { - 0x48, 0xc7, 0xc0, 0x00, 0x00, 0x00, 0x00, // mov rax, target - 0xc3 // ret - }; - U8 ret_text[] = { - 0xc3, // ret - }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_unresolved_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "fn_undef"), - T_COFF_DefReloc(X64_Addr64, 8, "fn_weak"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - T_COFF_DefSymbol_Undef("fn_undef"), - T_COFF_DefSymbol_Undef("fn_weak"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_unresolved_undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_undef"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_undef", "fn", 0), - T_COFF_DefSymbol_Undef("target_undef"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_unresolved_weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_weak"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_weak", "fn", 0), - T_COFF_DefSymbol_AbsExtern("target_weak_fallback", 0), - T_COFF_DefSymbol_Weak("target_weak", COFF_WeakExt_NoLibrary, "target_weak_fallback"), - {0} - } - })); - - t_invoke_linkerf("/force /subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_unresolved_entry.obj icf_unresolved_undef.obj icf_unresolved_weak.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 undef_vaddr = 0; - U64 weak_vaddr = 0; - str8_deserial_read_struct(data, 0, &undef_vaddr); - str8_deserial_read_struct(data, 8, &weak_vaddr); - T_Ok(undef_vaddr != 0); - T_Ok(weak_vaddr != 0); - T_Ok(undef_vaddr != weak_vaddr); -} - -TEST(icf_fold_with_largest_align) -{ - U8 text[] = { - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3 - }; - U8 call_a_and_b[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "entry", ".text", str8_array_fixed(call_a_and_b), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - { "a", ".text", str8_array_fixed(text), .flags = "rx:code@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text", str8_array_fixed(text), .flags = "rx:code@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "entry", 0), - T_COFF_DefSymbol_Static("a", "a", 0), - T_COFF_DefSymbol_Static("b", "b", 0), - {0} - } - }); - - // swap sections for a and b - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "entry", ".text", str8_array_fixed(call_a_and_b), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - { "a", ".text", str8_array_fixed(text), .flags = "rx:code@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text", str8_array_fixed(text), .flags = "rx:code@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "entry", 0), - T_COFF_DefSymbol_Static("a", "a", 0), - T_COFF_DefSymbol_Static("b", "b", 0), - {0} - } - }); - - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe /opt:icf b.obj"); - T_Ok(g_last_exit_code == 0); - - U8 expected_text[] = { - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xe8, 0x06, 0x00, 0x00, 0x00, - 0xc3, - 0xcc, - 0xcc, - 0xcc, - 0xcc, - 0xcc, - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, - 0x00, - 0xc3, - }; - - // validate output in a.exe - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } - - // validate output in b.exe - { - String8 exe = t_read_file(arena, str8_lit("b.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -TEST(icf_identical_bytes_different_color_spaces_do_not_fold) -{ - U8 same_bytes[] = { - 0xc3, - }; - U8 entry_text[] = { - 0xc3, - }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_color_spaces.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "text", ".text$mn", str8_array_fixed(same_bytes), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vftable", ".rdata$mn", str8_array_fixed(same_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "text"), - T_COFF_DefReloc(X64_Addr64, 8, "??_7type@@6B@"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("text", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("text", "text", 0), - T_COFF_DefSymbol_Extern("??_7type@@6B@", "vftable", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_color_spaces.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 text_vaddr = 0; - U64 vftable_vaddr = 0; - str8_deserial_read_struct(data, 0, &text_vaddr); - str8_deserial_read_struct(data, 8, &vftable_vaddr); - T_Ok(text_vaddr != 0); - T_Ok(vftable_vaddr != 0); - T_Ok(text_vaddr != vftable_vaddr); -} - -TEST(icf_multihop_reloc_target_colors_do_not_fold) -{ - U8 call_text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3, - }; - U8 return_1[] = { - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, - 0xc3, - }; - U8 return_2[] = { - 0x48, 0xc7, 0xc0, 0x02, 0x00, 0x00, 0x00, - 0xc3, - }; - U8 entry_text[] = { - 0xc3, - }; - U8 addresses[6 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_multihop.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { - "top_a", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "mid_a"), {0} } - }, - { - "top_b", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "mid_b"), {0} } - }, - { - "mid_a", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "leaf_a"), {0} } - }, - { - "mid_b", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "leaf_b"), {0} } - }, - { "leaf_a", ".text$mn", str8_array_fixed(return_1), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "leaf_b", ".text$mn", str8_array_fixed(return_2), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "top_a"), - T_COFF_DefReloc(X64_Addr64, 8, "top_b"), - T_COFF_DefReloc(X64_Addr64, 16, "mid_a"), - T_COFF_DefReloc(X64_Addr64, 24, "mid_b"), - T_COFF_DefReloc(X64_Addr64, 32, "leaf_a"), - T_COFF_DefReloc(X64_Addr64, 40, "leaf_b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("top_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("top_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("mid_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("mid_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("leaf_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("leaf_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("top_a", "top_a", 0), - T_COFF_DefSymbol_ExternFunc("top_b", "top_b", 0), - T_COFF_DefSymbol_ExternFunc("mid_a", "mid_a", 0), - T_COFF_DefSymbol_ExternFunc("mid_b", "mid_b", 0), - T_COFF_DefSymbol_ExternFunc("leaf_a", "leaf_a", 0), - T_COFF_DefSymbol_ExternFunc("leaf_b", "leaf_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_multihop.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - for EachIndex(i, ArrayCount(vaddrs)) { - T_Ok(vaddrs[i] != 0); - } - T_Ok(vaddrs[0] != vaddrs[1]); - T_Ok(vaddrs[2] != vaddrs[3]); - T_Ok(vaddrs[4] != vaddrs[5]); -} - -TEST(icf_comdat_symlink_chain) -{ - U8 ret_small[] = { 0xc3 }; - U8 ret_large[] = { 0xc3, 0x90 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_chain_leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "leader", ".text$mn", str8_array_fixed(ret_large), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("leader", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("leader", "leader", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_chain_duplicate.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "dup", ".text$mn", str8_array_fixed(ret_small), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "local_dup"), - T_COFF_DefReloc(X64_Addr64, 8, "leader"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("dup", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_ExternFunc("dup", "dup", 0), - T_COFF_DefSymbol_Static("local_dup", "dup", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - T_COFF_DefSymbol_UndefFunc("leader"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_chain_selected.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "dup", ".text$mn", str8_array_fixed(ret_large), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("dup", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_ExternFunc("dup", "dup", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_chain_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_chain_leader.obj icf_chain_duplicate.obj icf_chain_selected.obj icf_chain_entry.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); -} - -TEST(icf_llvm_addrsig) -{ - char *main_c = "int foo() { return 123; }\n" - "int bar() { return 123; }\n" - "int main() {\n" - "int (*fn)() = &foo;\n" - "return fn != bar;\n" - "}\n"; - String8 main_path = t_make_file_path(arena, str8_lit("main.c")); - T_Ok(write_data_to_file_path(main_path, str8_cstring(main_c))); - - String8 main_obj_path = t_make_file_path(arena, str8_lit("main.obj"));; - t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", main_path, main_obj_path), max_U64); - T_Ok(g_last_exit_code == 0); - - String8 a_path = t_make_file_path(arena, str8_lit("a.exe")); - - t_invoke_linkerf("%S /opt:icf /out:a.exe libcmt.lib", main_obj_path); - T_Ok(g_last_exit_code == 0); - t_invoke(a_path, str8_zero(), max_U64); - T_Ok(g_last_exit_code == 1); - - t_invoke_linkerf("%S /opt:icf /out:a.exe libcmt.lib /llvm_addrsig:no", main_obj_path); - T_Ok(g_last_exit_code == 0); - t_invoke(a_path, str8_zero(), max_U64); - T_Ok(g_last_exit_code == 0); -} - -// .llvm_addrsig can name an undefined external whose definition is in another -// object; ICF must parse and mark the resolved symbol's object, not the referrer. -TEST(icf_llvm_addrsig_external_symbol) -{ - char *ref_c = "extern int ext_sig();\n" - "int (*ext_sig_addr)() = &ext_sig;\n" - "int entry() { return ext_sig_addr(); }\n"; - char *def_c = "int dummy0() { return 0; }\n" - "int dummy1() { return 1; }\n" - "int dummy2() { return 2; }\n" - "int dummy3() { return 3; }\n" - "int dummy4() { return 4; }\n" - "int ext_sig() { return 0; }\n"; - String8 ref_path = t_make_file_path(arena, str8_lit("ref.c")); - String8 def_path = t_make_file_path(arena, str8_lit("def.c")); - T_Ok(write_data_to_file_path(ref_path, str8_cstring(ref_c))); - T_Ok(write_data_to_file_path(def_path, str8_cstring(def_c))); - - String8 ref_obj_path = t_make_file_path(arena, str8_lit("ref.obj")); - String8 def_obj_path = t_make_file_path(arena, str8_lit("def.obj")); - t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", ref_path, ref_obj_path), max_U64); - T_Ok(g_last_exit_code == 0); - t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", def_path, def_obj_path), max_U64); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("%S %S /subsystem:console /entry:entry /opt:icf /out:addrsig_ext.exe libcmt.lib", ref_obj_path, def_obj_path); - T_Ok(g_last_exit_code == 0); -} - -TEST(icf_pdata_xdata_fold) -{ - U8 ret_text[] = { 0xc3 }; - U8 xdata[] = { 0x01, 0x00, 0x00, 0x00 }; - PE_IntelPdata pdata = {0}; - U8 entry_text[] = { 0xc3 }; - U8 addresses[4 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_pdata_xdata_fold.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "fn_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "fn_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_a", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_b", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "pdata_a", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "fn_a"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "fn_a"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$a"), - {0} - } - }, - { - "pdata_b", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "fn_b"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "fn_b"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$b"), - {0} - } - }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "$pdata$a"), - T_COFF_DefReloc(X64_Addr64, 8, "$pdata$b"), - T_COFF_DefReloc(X64_Addr64, 16, "$unwind$a"), - T_COFF_DefReloc(X64_Addr64, 24, "$unwind$b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("xdata_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("xdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("pdata_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("pdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), - T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("$unwind$a", "xdata_a", 0), - T_COFF_DefSymbol_Extern("$unwind$b", "xdata_b", 0), - T_COFF_DefSymbol_Extern("$pdata$a", "pdata_a", 0), - T_COFF_DefSymbol_Extern("$pdata$b", "pdata_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_xdata_fold.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[4] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); - T_Ok(vaddrs[2] != 0); - T_Ok(vaddrs[2] == vaddrs[3]); -} - -TEST(icf_pdata_differs_by_function_color) -{ - U8 ret_1[] = { 0xb8, 1, 0, 0, 0, 0xc3 }; - U8 ret_2[] = { 0xb8, 2, 0, 0, 0, 0xc3 }; - U8 xdata[] = { 0x01, 0x00, 0x00, 0x00 }; - PE_IntelPdata pdata = {0}; - U8 entry_text[] = { 0xc3 }; - U8 addresses[3 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_pdata_diff.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "fn_a", ".text$mn", str8_array_fixed(ret_1), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "fn_b", ".text$mn", str8_array_fixed(ret_2), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_a", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_b", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "pdata_a", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "fn_a"), T_COFF_DefReloc(X64_Addr32Nb, 4, "fn_a"), T_COFF_DefReloc(X64_Addr32Nb, 8, "$unwind$a"), {0} } }, - { "pdata_b", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "fn_b"), T_COFF_DefReloc(X64_Addr32Nb, 4, "fn_b"), T_COFF_DefReloc(X64_Addr32Nb, 8, "$unwind$b"), {0} } }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "$pdata$a"), T_COFF_DefReloc(X64_Addr64, 8, "$pdata$b"), T_COFF_DefReloc(X64_Addr64, 16, "$unwind$a"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("xdata_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("xdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("pdata_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("pdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("$unwind$a", "xdata_a", 0), T_COFF_DefSymbol_Extern("$unwind$b", "xdata_b", 0), - T_COFF_DefSymbol_Extern("$pdata$a", "pdata_a", 0), T_COFF_DefSymbol_Extern("$pdata$b", "pdata_b", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_diff.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[3] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[1] != 0); - T_Ok(vaddrs[0] != vaddrs[1]); -} - -TEST(icf_vftable_and_vbtable_policy) -{ - U8 table_bytes[sizeof(U64)] = {0}; - U8 target_a[] = { 1 }; - U8 target_b[] = { 2 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[8 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_tables.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "vf_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vf_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vb_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vb_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rtti_a", ".rdata$mn", str8_array_fixed(target_a), .flags = "r:data@1" }, - { "rtti_b", ".rdata$mn", str8_array_fixed(target_b), .flags = "r:data@1" }, - { "vf_ref_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_a"), {0} } }, - { "vf_ref_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_b"), {0} } }, - { "vb_ref_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_a"), {0} } }, - { "vb_ref_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_b"), {0} } }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7a@@6B@"), T_COFF_DefReloc(X64_Addr64, 8, "??_7b@@6B@"), - T_COFF_DefReloc(X64_Addr64, 16, "??_8a@@7B@"), T_COFF_DefReloc(X64_Addr64, 24, "??_8b@@7B@"), - T_COFF_DefReloc(X64_Addr64, 32, "??_7ra@@6B@"), T_COFF_DefReloc(X64_Addr64, 40, "??_7rb@@6B@"), - T_COFF_DefReloc(X64_Addr64, 48, "??_8ra@@7B@"), T_COFF_DefReloc(X64_Addr64, 56, "??_8rb@@7B@"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vf_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vf_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vb_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vb_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vf_ref_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vf_ref_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vb_ref_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vb_ref_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("??_7a@@6B@", "vf_a", 0), T_COFF_DefSymbol_Extern("??_7b@@6B@", "vf_b", 0), - T_COFF_DefSymbol_Extern("??_8a@@7B@", "vb_a", 0), T_COFF_DefSymbol_Extern("??_8b@@7B@", "vb_b", 0), - T_COFF_DefSymbol_Extern("??_7ra@@6B@", "vf_ref_a", 0), T_COFF_DefSymbol_Extern("??_7rb@@6B@", "vf_ref_b", 0), - T_COFF_DefSymbol_Extern("??_8ra@@7B@", "vb_ref_a", 0), T_COFF_DefSymbol_Extern("??_8rb@@7B@", "vb_ref_b", 0), - T_COFF_DefSymbol_Extern("rtti_a", "rtti_a", 0), T_COFF_DefSymbol_Extern("rtti_b", "rtti_b", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_tables.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[8] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] == vaddrs[1]); - T_Ok(vaddrs[2] != vaddrs[3]); - T_Ok(vaddrs[4] != vaddrs[5]); - T_Ok(vaddrs[6] != vaddrs[7]); -} - -TEST(icf_readonly_non_vftable_data_policy) -{ - U8 data_bytes[] = { 1, 2, 3, 4 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[4 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_rdata_policy.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "data_a", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data_b", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data_c", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data_d", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "data_a"), T_COFF_DefReloc(X64_Addr64, 8, "data_b"), T_COFF_DefReloc(X64_Addr64, 16, "data_c"), T_COFF_DefReloc(X64_Addr64, 24, "data_d"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("data_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("data_c", COFF_ComdatSelect_Any), T_COFF_DefSymbol_Secdef("data_d", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), T_COFF_DefSymbol_Extern("data_a", "data_a", 0), T_COFF_DefSymbol_Extern("data_b", "data_b", 0), T_COFF_DefSymbol_Extern("data_c", "data_c", 0), T_COFF_DefSymbol_Extern("data_d", "data_d", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_rdata_policy.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[4] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[1] != 0); - T_Ok(vaddrs[0] != vaddrs[1]); - T_Ok(vaddrs[2] != 0); - T_Ok(vaddrs[2] == vaddrs[3]); -} - -TEST(icf_cpp_identical_functions_fold) -{ - char source[] = - "extern \"C\" __declspec(noinline) int a(void) { return 42; }\n" - "extern \"C\" __declspec(noinline) int b(void) { return 42; }\n" - "extern \"C\" int (* volatile pa)(void) = a;\n" - "extern \"C\" int (* volatile pb)(void) = b;\n" - "extern \"C\" int entry(void) { return pa == pb ? 0 : 1; }\n"; - - T_Ok(t_write_file(str8_lit("icf_cpp_fold.cpp"), str8_cstring(source))); - T_Ok(t_invoke_cl("/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_fold.obj icf_cpp_fold.cpp")); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_fold.exe /opt:ref,icf /include:pa /include:pb icf_cpp_fold.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[2] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("icf_cpp_fold.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); -} - -TEST(icf_cpp_multihop_functions_do_not_fold) -{ - char source[] = - "__declspec(noinline) int leaf_a(void) { return 1; }\n" - "__declspec(noinline) int leaf_b(void) { return 2; }\n" - "__declspec(noinline) int mid_a(void) { return leaf_a(); }\n" - "__declspec(noinline) int mid_b(void) { return leaf_b(); }\n" - "__declspec(noinline) int top_a(void) { return mid_a(); }\n" - "__declspec(noinline) int top_b(void) { return mid_b(); }\n" - "int (* volatile p_top_a)(void) = top_a;\n" - "int (* volatile p_top_b)(void) = top_b;\n" - "int (* volatile p_mid_a)(void) = mid_a;\n" - "int (* volatile p_mid_b)(void) = mid_b;\n" - "int (* volatile p_leaf_a)(void) = leaf_a;\n" - "int (* volatile p_leaf_b)(void) = leaf_b;\n" - "int entry(void) {\n" - " if (p_top_a == p_top_b) { return 1; }\n" - " if (p_mid_a == p_mid_b) { return 2; }\n" - " if (p_leaf_a == p_leaf_b) { return 3; }\n" - " return 0;\n" - "}\n"; - - T_Ok(t_write_file(str8_lit("icf_cpp_multihop.c"), str8_cstring(source))); - T_Ok(t_invoke_cl("/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_multihop.obj icf_cpp_multihop.c")); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_multihop.exe /opt:ref,icf /include:p_top_a /include:p_top_b /include:p_mid_a /include:p_mid_b /include:p_leaf_a /include:p_leaf_b icf_cpp_multihop.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[6] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("icf_cpp_multihop.exe"), vaddrs, ArrayCount(vaddrs))); - for EachIndex(i, ArrayCount(vaddrs)) { - T_Ok(vaddrs[i] != 0); - } - T_Ok(vaddrs[0] != vaddrs[1]); - T_Ok(vaddrs[2] != vaddrs[3]); - T_Ok(vaddrs[4] != vaddrs[5]); -} - -#endif - -#if 0 -TEST(defer_imp_link) -{ - T_COFF_DefLib bar_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "bar.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "bar.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\xff\x25\x00\x00\x00\x00"), .flags = "rx:code" - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__imp_bar"), - T_COFF_DefSymbol_ExternFunc("qwe", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - T_COFF_DefLib foo_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "foo.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "foo.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\xff\x25\x00\x00\x00\x00"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 2, "bar"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("bar"), - T_COFF_DefSymbol_Undef("qwe"), - T_COFF_DefSymbol_ExternFunc("thunk", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - String8 bar_lib = t_coff_from_def_lib(arena, bar_lib_any); - String8 foo_lib = t_coff_from_def_lib(arena, foo_lib_any); - - T_Ok(t_write_file(str8_lit("bar.lib"), bar_lib)); - T_Ok(t_write_file(str8_lit("foo.lib"), foo_lib)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe bar.lib foo.lib entry.obj /include:thunk"); - T_Ok(g_last_exit_code == 0); -} -#endif - -#if 0 -TEST(reloc_apply_off_out_of_bounds) -{ - T_Ok(t_write_def_obj("bad.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, max_U32, "target"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsStatic("target", 0), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe bad.obj"); - T_Ok(g_last_exit_code != 0); -} -#endif - -#if 0 -TEST(lib_member_reloc_apply_off_out_of_bounds) -{ - T_Ok(t_write_def_lib("bad.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("bad_member.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, max_U32, "target"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsStatic("target", 0), - T_COFF_DefSymbol_Extern("bad", "data", 0), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /include:bad entry.obj bad.lib"); - T_Ok(g_last_exit_code != 0); + return data; } -#endif diff --git a/src/linker/tests/long_section_name.tst b/src/linker/tests/long_section_name.tst new file mode 100644 index 000000000..6ed80d5a0 --- /dev/null +++ b/src/linker/tests/long_section_name.tst @@ -0,0 +1,33 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + debug_info: { name: ".debug_info", permissions: (read, write), content: initialized_data, data: { text: "DEBUG_INFO" } } + debug_abbrev: { name: ".debug_abbrev", permissions: (read, write), content: initialized_data, data: { text: "DEBUG_ABBREV" } } + } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + // link test.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe test.obj", artifact: image } + } + steps: + { + // load linked exe + expect_pe: { artifact: image, expected: { pe: { sections: { ".debug_info": {}, ".debug_abbrev": {} } } } } + } +} diff --git a/src/linker/tests/machine_compat_check.tst b/src/linker/tests/machine_compat_check.tst new file mode 100644 index 000000000..3e572fa88 --- /dev/null +++ b/src/linker/tests/machine_compat_check.tst @@ -0,0 +1,102 @@ +test: +{ + + artifacts: + { + unknown_obj: + { + file_name: "unknown.obj" + coff: + { + object: + { + machine: unknown + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "unknown" } } + } + } + } + } + x64_obj: + { + file_name: "x64.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "x64" } } + } + } + } + } + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + } + symbols: + { + my_entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } + } + } + arm64_obj: + { + file_name: "arm64.obj" + coff: + { + object: + { + machine: arm64 + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "arm64" } } + } + } + } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj" + artifact: image + } + // test objs with conflicting machines + link: + { + args: "/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj arm64.obj" + expect_exit: nonzero + } + // check /MACHINE switch + link: + { + args: "/subsystem:console /entry:my_entry /out:a.exe /machine:amd64 arm64.obj entry.obj" + expect_exit: nonzero + } + } + + steps: + { + expect_pe: + { + artifact: image + expected: { pe: { arch: x64 } } + } + } +} diff --git a/src/linker/tests/map.tst b/src/linker/tests/map.tst new file mode 100644 index 000000000..c240ce390 --- /dev/null +++ b/src/linker/tests/map.tst @@ -0,0 +1,65 @@ +test: +{ + + artifacts: + { + map_input: + { + file_name: "map.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "map_entry", section: text, value: 0 } + local: { kind: static, name: "map_local", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/entry:map_entry /subsystem:console /out:map_test_explicit.exe /map:map_test_explicit.map map.obj" + } + + link: + { + args: "/entry:map_entry /subsystem:console /out:map_test_default.exe /map map.obj" + } + + link: + { + args: "/entry:map_entry /subsystem:console /out:map_test_collision.exe /map:map_test_collision.exe map.obj" + expect_exit: nonzero + } + } + + steps: + { + expect_file: { path: "map_test_explicit.map", contains: " Timestamp is " } + expect_file: { path: "map_test_explicit.map", contains: " Preferred load address is " } + expect_file: { path: "map_test_explicit.map", contains: " Start Length Name Class" } + expect_file: { path: "map_test_explicit.map", contains: " Publics by Value" } + expect_file: { path: "map_test_explicit.map", contains: "map_entry" } + expect_file: { path: "map_test_explicit.map", contains: " entry point at" } + expect_file: { path: "map_test_explicit.map", contains: " Static symbols" } + expect_file: { path: "map_test_explicit.map", contains: "map_local" } + expect_file: { path: "map_test_default.map", nonempty: true } + } +} diff --git a/src/linker/tests/merge.tst b/src/linker/tests/merge.tst new file mode 100644 index 000000000..f070fbc4d --- /dev/null +++ b/src/linker/tests/merge.tst @@ -0,0 +1,84 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: { machine: x64, sections: { test: { name: ".test", permissions: (read, write), content: initialized_data, data: { text: "hello, world" } } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + mixed_obj: + { + file_name: "mixed.obj" + coff: { object: { machine: x64, sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "d" } } + bss: { name: ".bss", permissions: (read, write), content: uninitialized_data, alignment: 1, data: { zero: 513 } } + } } } + } + order_obj: + { + file_name: "order.obj" + coff: { object: { machine: x64, sections: + { + a: { name: ".a$m", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "a" } } + z: { name: ".z$m", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "z" } } + } } } + } + created_image: { file_name: "merge_created.exe", pe: {} } + mixed_image: { file_name: "merge_mixed.exe", pe: {} } + order_image: { file_name: "merge_order.exe", pe: {} } + chained_image: { file_name: "merge_chained.exe", pe: {} } + } + + build: + { + // circular merge + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.test entry.obj test.obj", output: none, expect_exit: 28 } + + // circular merge with extra link + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.data /merge:.data=.test entry.obj test.obj", output: none, expect_exit: 28 } + + // merge with non-defined section + link: { args: "/subsystem:console /entry:entry /out:merge_created.exe /merge:.test=.qwe entry.obj test.obj", artifact: created_image } + // make sure linker created .qwe and merged .test into it + + // illegal merge with .reloc + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.reloc entry.obj test.obj", output: none, expect_exit: 26 } + + // illegal merge with .rsrc + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.rsrc entry.obj test.obj", output: none, expect_exit: 26 } + + // merge non-defined section with defined section + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.qwe=.test entry.obj test.obj", output: none } + + // BSS and initialized data have incompatible flags and remain separate. + link: { args: "/subsystem:console /entry:entry /out:merge_mixed.exe /merge:.bss=.data entry.obj mixed.obj", artifact: mixed_image } + + // merged contribution groups retain lexical order + link: { args: "/subsystem:console /entry:entry /out:merge_order.exe /merge:.a=.z entry.obj order.obj", artifact: order_image } + + // merge .test -> .qwe -> .data + link: { args: "/subsystem:console /entry:entry /out:merge_chained.exe /merge:.test=.qwe /merge:.qwe=.data entry.obj test.obj", artifact: chained_image } + } + + steps: + { + expect_pe: { artifact: created_image, expected: { pe: { sections: { ".qwe": { @bits_set(3221225536) raw_flags: 3221225536, data: 68656c6c6f2c20776f726c64 } } } } } + expect_pe: { artifact: mixed_image, expected: { pe: { sections: { ".data": { file_size: 512, virtual_size: 1 }, ".bss": { file_size: 0, virtual_size: 513 } }, optional: { sizeof_initialized_data: 512, sizeof_uninitialized_data: 1024 } } } } + expect_pe: { artifact: order_image, expected: { pe: { sections: { ".z": { data: 617a } } } } } + + // make sure linker merged .test into .data + expect_pe: { artifact: chained_image, expected: { pe: { sections: { ".data": { @bits_set(3221225536) raw_flags: 3221225536, data: 68656c6c6f2c20776f726c64 } } } } } + } +} diff --git a/src/linker/tests/merge_duplicate_types.tst b/src/linker/tests/merge_duplicate_types.tst new file mode 100644 index 000000000..bb23cd465 --- /dev/null +++ b/src/linker/tests/merge_duplicate_types.tst @@ -0,0 +1,285 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + pch_obj: + { + file_name: "pch.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + // signature + hex: "04000000" + // duplicate in a.obj + hex: "0a0002100300000000000000" + // unique procedure type + hex: "0e000810001000000200000000000000" + // PCH ender + hex: "06001400bebafeca" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000f1000000320000000e000111bebafeca7063682e6f626a00" + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + hex: "20003c1100000000d00000000000000000000e002400197f0000544f5254555245000000" + } + } + } + } + } + } + } + + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000160009150010000002000000bebafeca7063682e6f626a00" + hex: "0a0002100300000000000000" + hex: "0e000810001000000000000000000000" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000f1000000300000000c000111bebafeca612e6f626a00" + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + hex: "20003c1100000000d00000000000000000000e002400197f0000544f525455524500" + } + } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000160009150010000002000000bebafeca7063682e6f626a00" + hex: "0a0002100300000000000000" + hex: "0e000810001000000000000000000000" + } + } + } + } + } + } + } + + c_obj: + { + file_name: "c.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { zero: 0 } + } + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000" + hex: "0a0002101100000000000000" + hex: "0e000810001000000000000000000000" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000f10000005f000000" + // S_OBJNAME + hex: "0c000111bebafeca632e6f626a00" + // S_COMPILE3 + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + hex: "20003c1100000000d00000000000000000000e002400197f0000544f525455524500" + // S_LPROC32 + hex: "29000f110000000000000000000000000100000000000000000000000110000000000000010000666f6f00" + // S_PROC_ID_END + hex: "02000600" + // $$Symbols header + hex: "00" + } + } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe pch.obj a.obj b.obj c.obj entry.obj" + } + } + + steps: + { + // load msf + // find named streams + // find string table + // find TPI + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + tpi: + { + leaf_count: 5 + @count(5) leaves: + { + leaf_0: { kind: LF_POINTER, data_size: 8, type: 3, attributes: 0 } + leaf_1: { kind: LF_PROCEDURE, data_size: 12, return_type: 4096, call_kind: 2 } + leaf_2: { kind: LF_PROCEDURE, data_size: 12, return_type: 4096, call_kind: 0 } + leaf_3: { kind: LF_POINTER, data_size: 8, type: 17, attributes: 0 } + leaf_4: { kind: LF_PROCEDURE, data_size: 12, return_type: 4099, call_kind: 0 } + } + } + } + } + } + } +} diff --git a/src/linker/tests/ms_link_icf_section_flag_eligibility.tst b/src/linker/tests/ms_link_icf_section_flag_eligibility.tst new file mode 100644 index 000000000..953833887 --- /dev/null +++ b/src/linker/tests/ms_link_icf_section_flag_eligibility.tst @@ -0,0 +1,110 @@ +test: +{ + // ret + artifacts: + { + object: + { + file_name: "ms_icf_flags.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 96 } + relocations: + { + code_comdat_a_ref: { type: Addr64, offset: 0, symbol: code_comdat_a_symbol } + code_comdat_b_ref: { type: Addr64, offset: 8, symbol: code_comdat_b_symbol } + code_plain_a_ref: { type: Addr64, offset: 16, symbol: code_plain_a_symbol } + code_plain_b_ref: { type: Addr64, offset: 24, symbol: code_plain_b_symbol } + rdata_comdat_a_ref: { type: Addr64, offset: 32, symbol: rdata_comdat_a_symbol } + rdata_comdat_b_ref: { type: Addr64, offset: 40, symbol: rdata_comdat_b_symbol } + wdata_comdat_a_ref: { type: Addr64, offset: 48, symbol: wdata_comdat_a_symbol } + wdata_comdat_b_ref: { type: Addr64, offset: 56, symbol: wdata_comdat_b_symbol } + ro_code_comdat_a_ref: { type: Addr64, offset: 64, symbol: ro_code_comdat_a_symbol } + ro_code_comdat_b_ref: { type: Addr64, offset: 72, symbol: ro_code_comdat_b_symbol } + rw_code_comdat_a_ref: { type: Addr64, offset: 80, symbol: rw_code_comdat_a_symbol } + rw_code_comdat_b_ref: { type: Addr64, offset: 88, symbol: rw_code_comdat_b_symbol } + } + } + code_comdat_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + code_comdat_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + code_plain_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + code_plain_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + ro_code_comdat_a: { name: ".text$mn", permissions: (read), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + ro_code_comdat_b: { name: ".text$mn", permissions: (read), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + rw_code_comdat_a: { name: ".text$mn", permissions: (read, write), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + rw_code_comdat_b: { name: ".text$mn", permissions: (read, write), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + rdata_comdat_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + rdata_comdat_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + wdata_comdat_a: { name: ".data$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + wdata_comdat_b: { name: ".data$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + } + symbols: + { + code_comdat_a_definition: { kind: section_definition, section: code_comdat_a, selection: Any } + code_comdat_b_definition: { kind: section_definition, section: code_comdat_b, selection: Any } + rdata_comdat_a_definition: { kind: section_definition, section: rdata_comdat_a, selection: Any } + rdata_comdat_b_definition: { kind: section_definition, section: rdata_comdat_b, selection: Any } + wdata_comdat_a_definition: { kind: section_definition, section: wdata_comdat_a, selection: Any } + wdata_comdat_b_definition: { kind: section_definition, section: wdata_comdat_b, selection: Any } + ro_code_comdat_a_definition: { kind: section_definition, section: ro_code_comdat_a, selection: Any } + ro_code_comdat_b_definition: { kind: section_definition, section: ro_code_comdat_b, selection: Any } + rw_code_comdat_a_definition: { kind: section_definition, section: rw_code_comdat_a, selection: Any } + rw_code_comdat_b_definition: { kind: section_definition, section: rw_code_comdat_b, selection: Any } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + code_comdat_a_symbol: { kind: external_function, name: "code_comdat_a", section: code_comdat_a, value: 0 } + code_comdat_b_symbol: { kind: external_function, name: "code_comdat_b", section: code_comdat_b, value: 0 } + code_plain_a_symbol: { kind: external_function, name: "code_plain_a", section: code_plain_a, value: 0 } + code_plain_b_symbol: { kind: external_function, name: "code_plain_b", section: code_plain_b, value: 0 } + ro_code_comdat_a_symbol: { kind: external_function, name: "ro_code_comdat_a", section: ro_code_comdat_a, value: 0 } + ro_code_comdat_b_symbol: { kind: external_function, name: "ro_code_comdat_b", section: ro_code_comdat_b, value: 0 } + rw_code_comdat_a_symbol: { kind: external_function, name: "rw_code_comdat_a", section: rw_code_comdat_a, value: 0 } + rw_code_comdat_b_symbol: { kind: external_function, name: "rw_code_comdat_b", section: rw_code_comdat_b, value: 0 } + rdata_comdat_a_symbol: { kind: external, name: "rdata_comdat_a", section: rdata_comdat_a, value: 0 } + rdata_comdat_b_symbol: { kind: external, name: "rdata_comdat_b", section: rdata_comdat_b, value: 0 } + wdata_comdat_a_symbol: { kind: external, name: "wdata_comdat_a", section: wdata_comdat_a, value: 0 } + wdata_comdat_b_symbol: { kind: external, name: "wdata_comdat_b", section: wdata_comdat_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "ms_icf_flags.exe", pe: {} } + } + + build: + { + compile_link: + { + tool: msvc + artifact: image + args: "ms_icf_flags.obj /link /nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf_flags.exe /opt:ref,icf /include:addresses" + } + } + + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: equal } // executable code COMDATs fold + expect_pe_word: { artifact: image, section: ".data", offset: 64, type: u64, other_section: ".data", other_offset: 72, relation: equal } // read-only code COMDATs fold + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, other_section: ".data", other_offset: 24, relation: not_equal } // non-COMDAT code does not fold + expect_pe_word: { artifact: image, section: ".data", offset: 48, type: u64, other_section: ".data", other_offset: 56, relation: not_equal } // writable data COMDATs do not fold + expect_pe_word: { artifact: image, section: ".data", offset: 80, type: u64, other_section: ".data", other_offset: 88, relation: not_equal } // writable code COMDATs do not + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u64, other_section: ".data", other_offset: 40, relation: equal } // read-only data COMDATs fold + } +} diff --git a/src/linker/tests/ms_link_icfs_identical_comdats.tst b/src/linker/tests/ms_link_icfs_identical_comdats.tst new file mode 100644 index 000000000..231e6abda --- /dev/null +++ b/src/linker/tests/ms_link_icfs_identical_comdats.tst @@ -0,0 +1,85 @@ +test: +{ + // xor rax, rax + // ret + // ret + artifacts: + { + object: + { + file_name: "ms_icf.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + a: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: { hex: "4831c0c3" } + } + b: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: { hex: "4831c0c3" } + } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 16 } + relocations: + { + a_ref: { type: Addr64, offset: 0, symbol: a_symbol } + b_ref: { type: Addr64, offset: 8, symbol: b_symbol } + } + } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: Any } + b_definition: { kind: section_definition, section: b, selection: Any } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + a_symbol: { kind: external_function, name: "a", section: a, value: 0 } + b_symbol: { kind: external_function, name: "b", section: b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "ms_icf.exe", pe: {} } + } + + build: + { + compile_link: + { + tool: msvc + artifact: image + args: "ms_icf.obj /link /nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf.exe /opt:ref,icf /include:a /include:b /include:addresses" + } + } + + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: equal } // COMDAT are folded + } +} diff --git a/src/linker/tests/opt_ref_dangling_section.tst b/src/linker/tests/opt_ref_dangling_section.tst new file mode 100644 index 000000000..7f2de104b --- /dev/null +++ b/src/linker/tests/opt_ref_dangling_section.tst @@ -0,0 +1,54 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { f_ref: { type: Addr32Nb, offset: 0, symbol: f } } + } }, symbols: { + f: { kind: undefined, name: "f" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } } } + } + a_obj: + { + file_name: "a.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, flags: (link_comdat) + data: { concat: { text: "A0000", zero: 1 } } + relocations: { q_ref: { type: Addr32Nb, offset: 0, symbol: q } } + } }, symbols: { + q: { kind: undefined, name: "q" } + data_def: { kind: section_definition, section: data, selection: Largest } + f: { kind: external, name: "f", section: data, value: 0 } + } } } + } + b_lib: + { + file_name: "b.lib" + coff: { library: { second_linker_member: true, members: { b_member: { + path: "b.obj" + object: { machine: x64, sections: { + q: { name: ".q", permissions: (read, write), content: initialized_data, data: { hex: "01020304" } } + data: { + name: ".data", permissions: (read, write), content: initialized_data, flags: (link_comdat) + data: { concat: { text: "BBBBBBBBBBBBBBB", zero: 1 } } + } + }, symbols: { + q: { kind: external, name: "q", section: q, value: 0 } + data_def: { kind: section_definition, section: data, selection: Largest } + f: { kind: external, name: "f", section: data, value: 0 } + } } + } } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.lib", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".q": {} } } } } } +} diff --git a/src/linker/tests/opt_ref_weak_alias_comdat.tst b/src/linker/tests/opt_ref_weak_alias_comdat.tst new file mode 100644 index 000000000..81cedb705 --- /dev/null +++ b/src/linker/tests/opt_ref_weak_alias_comdat.tst @@ -0,0 +1,32 @@ +test: +{ + artifacts: + { + weak_obj: + { + file_name: "weak.obj" + coff: { object: { machine: x64, + sections: { target: { name: ".target", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target" } } } + symbols: { + target_def: { kind: section_definition, section: target, selection: Any } + target: { kind: external, name: "target", section: target, value: 0 } + weak_target: { kind: weak, name: "weak_target", fallback: target, search: alias } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "48c7c000000000c3" } + relocations: { weak_ref: { type: Addr32Nb, offset: 3, symbol: weak_target } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + weak_target: { kind: undefined, name: "weak_target" } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /opt:ref /out:a.exe entry.obj weak.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".target": {} } } } } } +} diff --git a/src/linker/tests/out_of_bounds_section_number.tst b/src/linker/tests/out_of_bounds_section_number.tst new file mode 100644 index 000000000..64a25a847 --- /dev/null +++ b/src/linker/tests/out_of_bounds_section_number.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + // bad.obj + bad_obj: + { + file_name: "bad.obj" + bytes: { data: { concat: + { + hex: "64860100000000003f0000000100000000000000" + hex: "2e666f6f000000000000000000000000030000003c000000000000000000000000000000400000c0" + text: "foo" + hex: "666f6f0000000000000000007b0000000200" + hex: "04000000" + } } } + } + + // entry.obj + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "48c7c000000000c3" } + relocations: { foo: { type: Addr32Nb, offset: 0, symbol: foo } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } } + } + } + + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj", output: none, expect_exit: nonzero } } + steps: {} +} diff --git a/src/linker/tests/patch_cv_symbol_tree.tst b/src/linker/tests/patch_cv_symbol_tree.tst new file mode 100644 index 000000000..d8a15d714 --- /dev/null +++ b/src/linker/tests/patch_cv_symbol_tree.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { patch_cv_symbol_tree: {} } +} diff --git a/src/linker/tests/pch_sig_fallback.tst b/src/linker/tests/pch_sig_fallback.tst new file mode 100644 index 000000000..36d9d3ec6 --- /dev/null +++ b/src/linker/tests/pch_sig_fallback.tst @@ -0,0 +1,252 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + // signature + u32le: 4 + + // duplicate in a.obj + u16le: 10 + u16le: 0x1002 + u32le: 3 + u32le: 0 + + // unique procedure type + u16le: 14 + u16le: 0x1008 + u32le: 0x1000 + hex: "02 00" + u16le: 0 + u32le: 0 + + // PCH ender + u16le: 6 + u16le: 0x0014 + u32le: 0xcafebabe + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + align4: + { + concat: + { + u32le: 0x000000f1 + size32le: + { + concat: + { + size16le: + { + concat: + { + u16le: 0x1101 + u32le: 0xcafebabe + work_path: "a.obj" + zero: 1 + } + } + + size16le: + { + concat: + { + u16le: 0x113c + u32le: 0 + u16le: 0x00d0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 14 + u16le: 36 + u16le: 32537 + u16le: 0 + text: "TORTURE" + zero: 1 + } + } + } + } + } + } + } + } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + + u16le: 40 + u16le: 0x1509 + u32le: 0x1000 + u32le: 2 + u32le: 0xcafebabe + text: "corrupt-pch-file-path.obj" + zero: 1 + + u16le: 10 + u16le: 0x1002 + u32le: 3 + u32le: 0 + + u16le: 14 + u16le: 0x1008 + u32le: 0x1000 + hex: "00 00" + u16le: 0 + u32le: 0 + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + align4: + { + concat: + { + u32le: 0x000000f1 + size32le: + { + concat: + { + size16le: + { + concat: + { + u16le: 0x1101 + u32le: 0xcafebabe + work_path: "a.obj" + zero: 1 + } + } + + size16le: + { + concat: + { + u16le: 0x113c + u32le: 0 + u16le: 0x00d0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 14 + u16le: 36 + u16le: 32537 + u16le: 0 + text: "TORTURE" + zero: 1 + } + } + } + } + } + } + } + } + } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj" + } + } + + steps: {} +} diff --git a/src/linker/tests/pdbstripped.tst b/src/linker/tests/pdbstripped.tst new file mode 100644 index 000000000..4ba16bbbd --- /dev/null +++ b/src/linker/tests/pdbstripped.tst @@ -0,0 +1,161 @@ +test: +{ + + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + u32le: 0x000000f1 + u32le: 172 + + u16le: 16 + u16le: 0x1101 + u32le: 0x123 + text: "debug.obj" + zero: 1 + + u16le: 49 + u16le: 0x1147 + zero: 35 + text: "global_proc" + zero: 1 + + u16le: 2 + u16le: 0x114f + + u16le: 21 + u16le: 0x1108 + u32le: 0 + text: "global_typedef" + zero: 1 + + u16le: 48 + u16le: 0x1146 + zero: 35 + text: "local_proc" + zero: 1 + + u16le: 20 + u16le: 0x1108 + u32le: 0 + text: "local_typedef" + zero: 1 + + u16le: 2 + u16le: 0x114f + } + } + } + } + } + } + } + + pub_obj: + { + file_name: "pub.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { text: "FOOBAR" } } + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "QWE" } } + } + symbols: + { + global_func: { kind: external_function, name: "global_func", section: text, value: 1 } + global_var: { kind: external, name: "global_var", section: data, value: 1 } + static_var: { kind: static, name: "static_var", section: data, value: 1 } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe /pdbstripped:a.stripped.pdb entry.obj pub.obj debug.obj" + } + } + + steps: + { + expect_pdb: + { + path: "a.stripped.pdb" + expected: + { + pdb: + { + dbi: + { + @range(1, 0xffffffffffffffff) module_count + + // modules must contain only stubs for static procs + @all("*.c11_size", 0) + @all("*.c13_size", 0) + @all("*.module_symbols.symbols.*.kind", S_LPROC32, S_END) + modules: + { + module_0: { object_file_name: "entry.obj", symbol_size: 0 } + module_1: { object_file_name: "pub.obj", symbol_size: 0 } + module_2: { object_file_name: "debug.obj", @range(1, 0xffffffffffffffff) symbol_size } + } + } + + // global symbol stream must have public and references to the static stubs + @all("symbols.*.kind", S_PUB32, S_LPROCREF) + global_symbols: {} + + // types must be stripped + tpi: { header_only: true } + ipi: { header_only: true } + } + } + } + } +} diff --git a/src/linker/tests/psi_addr_map_radix_sort.tst b/src/linker/tests/psi_addr_map_radix_sort.tst new file mode 100644 index 000000000..81e26853e --- /dev/null +++ b/src/linker/tests/psi_addr_map_radix_sort.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { psi_addr_map_radix_sort: {} } +} diff --git a/src/linker/tests/reloc_against_removed_comdat.tst b/src/linker/tests/reloc_against_removed_comdat.tst new file mode 100644 index 000000000..0616997bd --- /dev/null +++ b/src/linker/tests/reloc_against_removed_comdat.tst @@ -0,0 +1,71 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { rdata: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "1Hello, World!", zero: 1 } } } } + symbols: + { + rdata_def: { kind: section_definition, section: rdata, selection: Largest } + test: { kind: external, name: "TEST", section: rdata, value: 1 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + comdat: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "H", zero: 1 } } } + regular: + { + name: ".rdata" + permissions: (read) + content: initialized_data + data: { zero: 4 } + relocations: { static_ref: { type: Addr32Nb, offset: 0, symbol: static_symbol } } + } + } + symbols: + { + comdat_def: { kind: section_definition, section: comdat, selection: Largest } + test: { kind: external, name: "TEST", section: comdat, value: 1 } + static_symbol: { kind: static, name: "STATIC", section: comdat, value: 2 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj" } } + steps: {} +} diff --git a/src/linker/tests/reloc_apply_off_out_of_bounds.tst b/src/linker/tests/reloc_apply_off_out_of_bounds.tst new file mode 100644 index 000000000..e4ffe3536 --- /dev/null +++ b/src/linker/tests/reloc_apply_off_out_of_bounds.tst @@ -0,0 +1,30 @@ +test: +{ + artifacts: + { + bad_obj: + { + file_name: "bad.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "00000000" } + relocations: { target: { type: Addr32, offset: 4294967295, symbol: target } } + } } + symbols: + { + target: { kind: absolute, name: "target", value: 0, storage: static } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:relocation_offset_out_of_bounds.exe bad.obj", expect_exit: nonzero } } + steps: {} +} diff --git a/src/linker/tests/relocate_undefined_section_symbol.tst b/src/linker/tests/relocate_undefined_section_symbol.tst new file mode 100644 index 000000000..8459abb6c --- /dev/null +++ b/src/linker/tests/relocate_undefined_section_symbol.tst @@ -0,0 +1,150 @@ +test: +{ + + // TODO: relocations against undefined section symbols + artifacts: + { + entry_obj: + { + file_name: "relocate_undefined_section_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "48c7c000000000c3" } + relocations: { caller_ref: { type: Addr32Nb, offset: 3, symbol: caller } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + caller: { kind: undefined, name: "caller" } + } + } } + } + caller_obj: + { + file_name: "relocate_undefined_section_caller.obj" + coff: { object: + { + machine: x64 + sections: + { + caller: + { + name: ".caller" + permissions: (read, write) + content: initialized_data + alignment: 1 + flags: (link_comdat) + data: { zero: 100 } + relocations: + { + target0_ref: { type: Addr32Nb, offset: 0, symbol: target0_section } + target1_ref: { type: Addr32Nb, offset: 4, symbol: target1_section } + target2_ref: { type: Addr32Nb, offset: 8, symbol: target2_section } + target3_ref: { type: Addr32Nb, offset: 12, symbol: target3_section } + target4_ref: { type: Addr32Nb, offset: 16, symbol: target4_section } + target5_ref: { type: Addr32Nb, offset: 20, symbol: target5_section } + target6_ref: { type: Addr32Nb, offset: 24, symbol: target6_section } + target7_ref: { type: Addr32Nb, offset: 28, symbol: target7_section } + target8_ref: { type: Addr32Nb, offset: 32, symbol: target8_section } + target9_ref: { type: Addr32Nb, offset: 36, symbol: target9_section } + target10_ref: { type: Addr32Nb, offset: 40, symbol: target10_section } + target11_ref: { type: Addr32Nb, offset: 44, symbol: target11_section } + target12_ref: { type: Addr32Nb, offset: 48, symbol: target12_section } + target13_ref: { type: Addr32Nb, offset: 52, symbol: target13_section } + target14_ref: { type: Addr32Nb, offset: 56, symbol: target14_section } + target15_ref: { type: Addr32Nb, offset: 60, symbol: target15_section } + target16_ref: { type: Addr32Nb, offset: 64, symbol: target16_section } + target17_ref: { type: Addr32Nb, offset: 68, symbol: target17_section } + target18_ref: { type: Addr32Nb, offset: 72, symbol: target18_section } + target19_ref: { type: Addr32Nb, offset: 76, symbol: target19_section } + target20_ref: { type: Addr32Nb, offset: 80, symbol: target20_section } + target21_ref: { type: Addr32Nb, offset: 84, symbol: target21_section } + target22_ref: { type: Addr32Nb, offset: 88, symbol: target22_section } + target23_ref: { type: Addr32Nb, offset: 92, symbol: target23_section } + target24_ref: { type: Addr32Nb, offset: 96, symbol: target24_section } + } + } + } + symbols: + { + caller_def: { kind: section_definition, section: caller, selection: Any } + caller: { kind: external, name: "caller", section: caller, value: 0 } + target0_section: { kind: undefined_section, name: ".target0", value: 1073741888 } + target1_section: { kind: undefined_section, name: ".target1", value: 1073741888 } + target2_section: { kind: undefined_section, name: ".target2", value: 1073741888 } + target3_section: { kind: undefined_section, name: ".target3", value: 1073741888 } + target4_section: { kind: undefined_section, name: ".target4", value: 1073741888 } + target5_section: { kind: undefined_section, name: ".target5", value: 1073741888 } + target6_section: { kind: undefined_section, name: ".target6", value: 1073741888 } + target7_section: { kind: undefined_section, name: ".target7", value: 1073741888 } + target8_section: { kind: undefined_section, name: ".target8", value: 1073741888 } + target9_section: { kind: undefined_section, name: ".target9", value: 1073741888 } + target10_section: { kind: undefined_section, name: ".target10", value: 1073741888 } + target11_section: { kind: undefined_section, name: ".target11", value: 1073741888 } + target12_section: { kind: undefined_section, name: ".target12", value: 1073741888 } + target13_section: { kind: undefined_section, name: ".target13", value: 1073741888 } + target14_section: { kind: undefined_section, name: ".target14", value: 1073741888 } + target15_section: { kind: undefined_section, name: ".target15", value: 1073741888 } + target16_section: { kind: undefined_section, name: ".target16", value: 1073741888 } + target17_section: { kind: undefined_section, name: ".target17", value: 1073741888 } + target18_section: { kind: undefined_section, name: ".target18", value: 1073741888 } + target19_section: { kind: undefined_section, name: ".target19", value: 1073741888 } + target20_section: { kind: undefined_section, name: ".target20", value: 1073741888 } + target21_section: { kind: undefined_section, name: ".target21", value: 1073741888 } + target22_section: { kind: undefined_section, name: ".target22", value: 1073741888 } + target23_section: { kind: undefined_section, name: ".target23", value: 1073741888 } + target24_section: { kind: undefined_section, name: ".target24", value: 1073741888 } + } + } } + } + + target0: { file_name: "relocate_undefined_section_target0.obj", coff: { object: { machine: x64, sections: { target: { name: ".target0", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target0" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target0", section: target, value: 0 } } } } } + target1: { file_name: "relocate_undefined_section_target1.obj", coff: { object: { machine: x64, sections: { target: { name: ".target1", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target1" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target1", section: target, value: 0 } } } } } + target2: { file_name: "relocate_undefined_section_target2.obj", coff: { object: { machine: x64, sections: { target: { name: ".target2", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target2" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target2", section: target, value: 0 } } } } } + target3: { file_name: "relocate_undefined_section_target3.obj", coff: { object: { machine: x64, sections: { target: { name: ".target3", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target3" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target3", section: target, value: 0 } } } } } + target4: { file_name: "relocate_undefined_section_target4.obj", coff: { object: { machine: x64, sections: { target: { name: ".target4", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target4" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target4", section: target, value: 0 } } } } } + target5: { file_name: "relocate_undefined_section_target5.obj", coff: { object: { machine: x64, sections: { target: { name: ".target5", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target5" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target5", section: target, value: 0 } } } } } + target6: { file_name: "relocate_undefined_section_target6.obj", coff: { object: { machine: x64, sections: { target: { name: ".target6", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target6" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target6", section: target, value: 0 } } } } } + target7: { file_name: "relocate_undefined_section_target7.obj", coff: { object: { machine: x64, sections: { target: { name: ".target7", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target7" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target7", section: target, value: 0 } } } } } + target8: { file_name: "relocate_undefined_section_target8.obj", coff: { object: { machine: x64, sections: { target: { name: ".target8", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target8" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target8", section: target, value: 0 } } } } } + target9: { file_name: "relocate_undefined_section_target9.obj", coff: { object: { machine: x64, sections: { target: { name: ".target9", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target9" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target9", section: target, value: 0 } } } } } + target10: { file_name: "relocate_undefined_section_target10.obj", coff: { object: { machine: x64, sections: { target: { name: ".target10", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target10" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target10", section: target, value: 0 } } } } } + target11: { file_name: "relocate_undefined_section_target11.obj", coff: { object: { machine: x64, sections: { target: { name: ".target11", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target11" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target11", section: target, value: 0 } } } } } + target12: { file_name: "relocate_undefined_section_target12.obj", coff: { object: { machine: x64, sections: { target: { name: ".target12", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target12" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target12", section: target, value: 0 } } } } } + target13: { file_name: "relocate_undefined_section_target13.obj", coff: { object: { machine: x64, sections: { target: { name: ".target13", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target13" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target13", section: target, value: 0 } } } } } + target14: { file_name: "relocate_undefined_section_target14.obj", coff: { object: { machine: x64, sections: { target: { name: ".target14", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target14" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target14", section: target, value: 0 } } } } } + target15: { file_name: "relocate_undefined_section_target15.obj", coff: { object: { machine: x64, sections: { target: { name: ".target15", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target15" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target15", section: target, value: 0 } } } } } + target16: { file_name: "relocate_undefined_section_target16.obj", coff: { object: { machine: x64, sections: { target: { name: ".target16", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target16" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target16", section: target, value: 0 } } } } } + target17: { file_name: "relocate_undefined_section_target17.obj", coff: { object: { machine: x64, sections: { target: { name: ".target17", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target17" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target17", section: target, value: 0 } } } } } + target18: { file_name: "relocate_undefined_section_target18.obj", coff: { object: { machine: x64, sections: { target: { name: ".target18", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target18" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target18", section: target, value: 0 } } } } } + target19: { file_name: "relocate_undefined_section_target19.obj", coff: { object: { machine: x64, sections: { target: { name: ".target19", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target19" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target19", section: target, value: 0 } } } } } + target20: { file_name: "relocate_undefined_section_target20.obj", coff: { object: { machine: x64, sections: { target: { name: ".target20", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target20" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target20", section: target, value: 0 } } } } } + target21: { file_name: "relocate_undefined_section_target21.obj", coff: { object: { machine: x64, sections: { target: { name: ".target21", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target21" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target21", section: target, value: 0 } } } } } + target22: { file_name: "relocate_undefined_section_target22.obj", coff: { object: { machine: x64, sections: { target: { name: ".target22", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target22" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target22", section: target, value: 0 } } } } } + target23: { file_name: "relocate_undefined_section_target23.obj", coff: { object: { machine: x64, sections: { target: { name: ".target23", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target23" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target23", section: target, value: 0 } } } } } + target24: { file_name: "relocate_undefined_section_target24.obj", coff: { object: { machine: x64, sections: { target: { name: ".target24", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target24" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target24", section: target, value: 0 } } } } } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /opt:ref /out:relocate_undefined_section_symbol.exe relocate_undefined_section_entry.obj relocate_undefined_section_caller.obj relocate_undefined_section_target0.obj relocate_undefined_section_target1.obj relocate_undefined_section_target2.obj relocate_undefined_section_target3.obj relocate_undefined_section_target4.obj relocate_undefined_section_target5.obj relocate_undefined_section_target6.obj relocate_undefined_section_target7.obj relocate_undefined_section_target8.obj relocate_undefined_section_target9.obj relocate_undefined_section_target10.obj relocate_undefined_section_target11.obj relocate_undefined_section_target12.obj relocate_undefined_section_target13.obj relocate_undefined_section_target14.obj relocate_undefined_section_target15.obj relocate_undefined_section_target16.obj relocate_undefined_section_target17.obj relocate_undefined_section_target18.obj relocate_undefined_section_target19.obj relocate_undefined_section_target20.obj relocate_undefined_section_target21.obj relocate_undefined_section_target22.obj relocate_undefined_section_target23.obj relocate_undefined_section_target24.obj" + } + } + + steps: + { + // T_Ok(match_count > 0); + } +} diff --git a/src/linker/tests/removed_section.tst b/src/linker/tests/removed_section.tst new file mode 100644 index 000000000..02cea364e --- /dev/null +++ b/src/linker/tests/removed_section.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + // TEST is defined in a section explicitly removed by the object flags. + sections: { test: { name: ".test", permissions: (read, execute), content: code, flags: (link_remove), data: { hex: "c3" } } } + symbols: { test: { kind: external, name: "TEST", section: test, value: 0 } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "48c7c000000000c3" } + relocations: { test: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + test: { kind: undefined, name: "TEST" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + build: + { + // A relocation cannot resolve to a definition in a removed section. + link: { args: "/subsystem:console /entry:entry /out:removed_section.exe test.obj entry.obj", expect_exit: nonzero } + } + steps: {} +} diff --git a/src/linker/tests/second_member_header.tst b/src/linker/tests/second_member_header.tst new file mode 100644 index 000000000..0579139d0 --- /dev/null +++ b/src/linker/tests/second_member_header.tst @@ -0,0 +1,34 @@ +test: +{ + artifacts: + { + test_lib: + { + file_name: "test.lib" + coff: { library: { second_linker_member: true, members: { symbols: { + path: "obj.obj" + object: { machine: x64, symbols: { + eight: { kind: absolute, name: "8", value: 8, storage: external } + one: { kind: absolute, name: "1", value: 1, storage: external } + nine: { kind: absolute, name: "9", value: 9, storage: external } + seven: { kind: absolute, name: "7", value: 7, storage: external } + four: { kind: absolute, name: "4", value: 4, storage: external } + five: { kind: absolute, name: "5", value: 5, storage: external } + two: { kind: absolute, name: "2", value: 2, storage: external } + three: { kind: absolute, name: "3", value: 3, storage: external } + six: { kind: absolute, name: "6", value: 6, storage: external } + } } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9" } } + steps: {} +} diff --git a/src/linker/tests/sect_align.tst b/src/linker/tests/sect_align.tst new file mode 100644 index 000000000..fd8d82653 --- /dev/null +++ b/src/linker/tests/sect_align.tst @@ -0,0 +1,56 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + shift: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "q" } } + none: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "abc" } } + a1: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "wr" } } + a2: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 2, data: { text: "e" } } + a4: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 4, data: { text: "ttttt" } } + a8: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 8, data: { text: "g" } } + a16: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 16, data: { text: "o" } } + a32: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 32, data: { text: "p" } } + a64: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 64, data: { text: "f" } } + a128: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 128, data: { text: "x" } } + a256: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 256, data: { text: "c" } } + a512: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 512, data: { text: "v" } } + a1024: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1024, data: { text: "b" } } + a2048: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 2048, data: { text: "n" } } + a4096: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 4096, data: { text: "m" } } + a8192: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 8192, data: { text: "z" } } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe /align:8192 test.obj", artifact: image } } + steps: + { + expect_pe_bytes: { artifact: image, section: ".a", offset: 0, hex: "71" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 16, hex: "616263" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 19, hex: "7772" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 22, hex: "65" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 24, hex: "7474747474" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 32, hex: "67" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 48, hex: "6f" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 64, hex: "70" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 128, hex: "66" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 256, hex: "78" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 512, hex: "63" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 1024, hex: "76" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 2048, hex: "62" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 4096, hex: "6e" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 8192, hex: "6d" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 16384, hex: "7a" } + } +} diff --git a/src/linker/tests/sect_symbol.tst b/src/linker/tests/sect_symbol.tst new file mode 100644 index 000000000..f81fcecc6 --- /dev/null +++ b/src/linker/tests/sect_symbol.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + sect_obj: + { + file_name: "sect.obj" + coff: { object: + { + machine: x64 + sections: { mysect1: { name: ".mysect$1", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "hello, world" } } } + directives: { directive: "/merge:.mysect=.data" } + } } + } + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 8 } + relocations: { mysect_address: { type: Addr64, offset: 0, symbol: mysect_group } } + } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + mysect_group: { kind: undefined_section, name: ".mysect$2222", value: 3221225536 } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe main.obj sect.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".data", target_offset: 8 } + expect_pe_bytes: { artifact: image, section: ".data", offset: 8, hex: "68656c6c6f2c20776f726c64" } + } +} diff --git a/src/linker/tests/section_directive_align_grouped_section.tst b/src/linker/tests/section_directive_align_grouped_section.tst new file mode 100644 index 000000000..42b6d8b9c --- /dev/null +++ b/src/linker/tests/section_directive_align_grouped_section.tst @@ -0,0 +1,36 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + prot_obj: + { + file_name: "prot.obj" + coff: { object: + { + machine: x64 + sections: { prot_mem: { name: "prot$mem", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "mem" } } } + directives: { directive: "/SECTION:prot,R,ALIGN=8192" } + } } + } + image: { file_name: "grouped_section_align.exe", pe: {} } + } + build: + { + // Apply read-only section flags and 8192-byte alignment to the grouped prot output section. + link: { args: "/subsystem:console /entry:entry /out:grouped_section_align.exe entry.obj prot.obj", artifact: image } + } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { "prot": { alignment: 8192, @bits_set(1088421952) raw_flags: 1088421952 } } } } } + } +} diff --git a/src/linker/tests/section_directive_read_only_grouped_section.tst b/src/linker/tests/section_directive_read_only_grouped_section.tst new file mode 100644 index 000000000..dad4167f2 --- /dev/null +++ b/src/linker/tests/section_directive_read_only_grouped_section.tst @@ -0,0 +1,35 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + prot_obj: + { + file_name: "prot.obj" + coff: { object: + { + machine: x64 + sections: + { + prot_a: { name: "prot$a", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "A" } } + prot_mem: { name: "prot$mem", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "mem" } } + prot_z: { name: "prot$z", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "Z" } } + } + directives: { directive: "/SECTION:prot,R" } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj prot.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { "prot": { @bits_set(1073741888) @bits_clear(2684354560) raw_flags: 1073741888 } } } } } } +} diff --git a/src/linker/tests/section_sort.tst b/src/linker/tests/section_sort.tst new file mode 100644 index 000000000..12f0f63cc --- /dev/null +++ b/src/linker/tests/section_sort.tst @@ -0,0 +1,43 @@ +test: +{ + artifacts: + { + data_obj: + { + file_name: "data.obj" + coff: { object: + { + machine: x64 + sections: + { + idata_2: { name: ".idata$2", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "last" } } + idata_5: { name: ".idata$5", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "first" } } + rdata: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, data: { text: "middle" } } + data_z: { name: ".data$z", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "five" } } + data_a: { name: ".data$a", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "three" } } + data_bbbbb: { name: ".data$bbbbb", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "four" } } + data_empty: { name: ".data$", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "two" } } + data: { name: ".data", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "one" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe data.obj entry.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: + { + ".data": { data: 66697273746f6e6574776f7468726565666f7572666976656c617374 } + ".rdata": { data: 6d6964646c65 } + } } } } } +} diff --git a/src/linker/tests/simple_lib_test.tst b/src/linker/tests/simple_lib_test.tst new file mode 100644 index 000000000..4bc5f255c --- /dev/null +++ b/src/linker/tests/simple_lib_test.tst @@ -0,0 +1,43 @@ +test: +{ + artifacts: + { + test_lib: + { + file_name: "test.lib" + coff: { library: { second_linker_member: true, members: { test_member: { + path: "test.obj" + object: { machine: unknown, + sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data + data: { concat: { text: "The quick brown fox jumps over the lazy dog", zero: 1 } } + } } + symbols: { test: { kind: external, name: "test", section: data, value: 0 } } + } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1 + data: { hex: "48c7c000000000c3" } + relocations: { test_ref: { type: Addr32Nb, offset: 3, symbol: test } } + } }, symbols: { + test: { kind: undefined, name: "test" } + entry: { kind: external, name: "my_entry", section: text, value: 7 } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe entry.obj test.lib", artifact: image } } + steps: + { + // was test payload linked? + expect_pe: { artifact: image, expected: { pe: { sections: { ".data": { data: 54686520717569636b2062726f776e20666f78206a756d7073206f76657220746865206c617a7920646f6700 } } } } } + // do we have enough bytes to read text? + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { virtual_size: 8 } } } } } + // linker must pull-in test.obj and patch relocation for "test" symbol + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: u32, target_section: ".data", target_offset: 0, target_address: rva } + } +} diff --git a/src/linker/tests/simple_link_test.tst b/src/linker/tests/simple_link_test.tst new file mode 100644 index 000000000..f1513af69 --- /dev/null +++ b/src/linker/tests/simple_link_test.tst @@ -0,0 +1,94 @@ +test: +{ + artifacts: + { + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "qwe" } } + zero: { name: ".zero", permissions: (read, write), content: uninitialized_data, data: { zero: 5 } } + } + symbols: { my_entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/entry:my_entry /subsystem:console /fixed /filealign:512 /align:4096 /out:a.exe main.obj" + artifact: image + } + } + + steps: + { + expect_coff: + { + artifact: main_obj + expected: { coff: { object: + { + machine: Amd64 + section_count: 3 + sections: + { + section_1: { name: ".text", data: c3 } + section_2: { name: ".data", data: 717765 } + section_3: { name: ".zero", data: "" } + } + symbol_count: 1 + symbols: { symbol_0: { name: "my_entry", value: 0, section_number: 1 } } + } } } + } + expect_pe: + { + artifact: image + expected: { pe: + { + is_pe32: false + arch: x64 + subsystem: console + section_count: 3 + section_alignment: 4096 + file_alignment: 512 + symbol_count: 0 + data_directory_count: 16 + entry_point: 4096 + // check section alignment + @count(3) sections: + { + ".text": { virtual_size: 1, virtual_offset: 4096, @aligned(512) file_size, file_offset: 512, data: c3 } + ".data": { virtual_size: 3, @aligned(4096) virtual_offset, @aligned(512) file_size, data: 717765 } + ".zero": { virtual_size: 5, @aligned(4096) virtual_offset, @aligned(512) file_size } + } + optional: + { + sizeof_code: 512 + sizeof_initialized_data: 512 + sizeof_uninitialized_data: 512 + code_base: 4096 + image_base: 5368709120 + major_os_version: 6 + minor_os_version: 0 + major_image_version: 0 + minor_image_version: 0 + major_subsystem_version: 6 + minor_subsystem_version: 0 + win32_version: 0 + sizeof_image: 16384 + sizeof_headers: 512 + dll_characteristics: 33056 + loader_flags: 0 + } + } } + } + } +} diff --git a/src/linker/tests/u64_array_radix_sort_parallel.tst b/src/linker/tests/u64_array_radix_sort_parallel.tst new file mode 100644 index 000000000..0e3a86cf4 --- /dev/null +++ b/src/linker/tests/u64_array_radix_sort_parallel.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { u64_array_radix_sort_parallel: {} } +} diff --git a/src/linker/tests/undef_reloc_section.tst b/src/linker/tests/undef_reloc_section.tst new file mode 100644 index 000000000..ed89b9ca0 --- /dev/null +++ b/src/linker/tests/undef_reloc_section.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + main_obj: + { + file_name: "main.obj" + coff: { object: { machine: x64, sections: { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "0000000000000000" } + relocations: { reloc_ref: { type: Addr64, offset: 0, symbol: reloc } } + } + }, symbols: { + entry: { kind: external, name: "my_entry", section: text, value: 0 } + reloc: { kind: undefined_section, name: ".reloc", value: 1107296320 } + } } } + } + sec_defn_obj: + { + file_name: "sec_defn.obj" + coff: { object: { machine: x64, sections: { mysect: { + name: ".mysect", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "010203" } + } }, symbols: { mysect_definition: { kind: section_definition, section: mysect, selection: Null } } } } + } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe main.obj sec_defn.obj", expect_exit: 25 } } + steps: {} +} diff --git a/src/linker/tests/undef_section.tst b/src/linker/tests/undef_section.tst new file mode 100644 index 000000000..e76c4e95f --- /dev/null +++ b/src/linker/tests/undef_section.tst @@ -0,0 +1,52 @@ +test: +{ + artifacts: + { + sec_defn_obj: + { + file_name: "sec_defn.obj" + coff: { object: + { + machine: x64 + sections: { mysect: { name: ".mysect", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "010203" } } } + symbols: { mysect_definition: { kind: section_definition, section: mysect, selection: Null } } + } } + } + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 4 } + relocations: { mysect_voff: { type: Addr32Nb, offset: 0, symbol: mysect } } + } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + mysect: { kind: undefined_section, name: ".mysect", value: 1073741888 } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "undef_section.exe", pe: {} } + } + build: + { + // The section symbol's expected flags select .mysect and Addr32Nb stores its image-relative offset. + link: { args: "/subsystem:console /entry:my_entry /out:undef_section.exe main.obj sec_defn.obj", artifact: image } + } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u32, target_section: ".mysect", target_offset: 0, target_address: rva } + expect_pe_bytes: { artifact: image, section: ".mysect", offset: 0, hex: "010203" } + } +} diff --git a/src/linker/tests/undef_weak_lib.tst b/src/linker/tests/undef_weak_lib.tst new file mode 100644 index 000000000..5310f4456 --- /dev/null +++ b/src/linker/tests/undef_weak_lib.tst @@ -0,0 +1,27 @@ +test: +{ + artifacts: + { + weak_obj: { file_name: "weak.obj", coff: { object: { machine: x64, symbols: { + b: { kind: absolute, name: "b", value: 3271557120, storage: external } + a: { kind: weak, name: "a", fallback: b, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "00000000" } + relocations: { a_ref: { type: Addr32Nb, offset: 0, symbol: a } } + } }, symbols: { + a: { kind: undefined, name: "a" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } } } + } + } + build: + { + // undefined symbol must always replace weak symbol with search library + link: { args: "/subsystem:console /out:a.exe /entry:entry entry.obj weak.obj", expect_exit: 47 } + } + steps: {} +} diff --git a/src/linker/tests/undef_weak_search_alias.tst b/src/linker/tests/undef_weak_search_alias.tst new file mode 100644 index 000000000..4dbfd8e39 --- /dev/null +++ b/src/linker/tests/undef_weak_search_alias.tst @@ -0,0 +1,37 @@ +test: +{ + artifacts: + { + weak_obj: + { + file_name: "weak.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "deadbeef" } + } }, symbols: { + ptr: { kind: undefined, name: "ptr" } + foo: { kind: weak, name: "foo", fallback: ptr, search: alias } + } } } + } + ptr_obj: { file_name: "ptr.obj", coff: { object: { machine: x64, symbols: { + entry: { kind: undefined, name: "entry" } + ptr: { kind: weak, name: "ptr", fallback: entry, search: alias } + } } } } + undef_obj: + { + file_name: "undef.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "00000000" } + relocations: { foo_ref: { type: Addr32Nb, offset: 0, symbol: foo } } + } }, symbols: { foo: { kind: undefined, name: "foo" } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } + } }, symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe weak.obj entry.obj ptr.obj undef.obj" } } + steps: {} +} diff --git a/src/linker/tests/utf16_rsp.tst b/src/linker/tests/utf16_rsp.tst new file mode 100644 index 000000000..8e64d9178 --- /dev/null +++ b/src/linker/tests/utf16_rsp.tst @@ -0,0 +1,24 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + rsp_file: + { + file_name: "args.rsp" + bytes: { data: { hex: "fffe2f00730075006200730079007300740065006d003a0063006f006e0073006f006c00650020002f0065006e007400720079003a0065006e0074007200790020002f006f00750074003a0061002e00650078006500200065006e007400720079002e006f0062006a000a00" } } + } + } + + build: { link: { args: "@args.rsp", output: none } } + steps: {} +} diff --git a/src/linker/tests/validate_gsi.tst b/src/linker/tests/validate_gsi.tst new file mode 100644 index 000000000..4d8760635 --- /dev/null +++ b/src/linker/tests/validate_gsi.tst @@ -0,0 +1,157 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + debug_obj: + { + file_name: "debug.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + concat: + { + // C13 signature and symbols subsection header. + u32le: 4 + u32le: 0xf1 + u32le: 102 + + // S_CONSTANT: type 0, immediate value 263, "CV_SymKind_BLOCK16". + u16le: 27 + u16le: 0x1107 + u32le: 0 + u16le: 263 + text: "CV_SymKind_BLOCK16" + zero: 1 + + // S_GDATA32: type 0, section 2 + 0x25440, "__newclmap". + u16le: 23 + u16le: 0x110d + u32le: 0 + u32le: 0x25440 + u16le: 2 + text: "__newclmap" + zero: 1 + + // S_GDATA32: type 0, section 1 + 123, "coffeebabe". + u16le: 23 + u16le: 0x110d + u32le: 0 + u32le: 123 + u16le: 1 + text: "coffeebabe" + zero: 1 + + // S_GDATA32: type 0, section 1 + 123, "deadbeef". + u16le: 21 + u16le: 0x110d + u32le: 0 + u32le: 123 + u16le: 1 + text: "deadbeef" + zero: 1 + + // Align the complete C13 subsection to four bytes. + zero: 2 + } + } + } + } + } + } + } + + // Raw helper output for byte-for-byte parity with the typed CodeView records above. + raw_debug_obj: + { + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + hex: "04000000 f1000000 66000000 1b000711 00000000 0701 43565f53796d4b696e645f424c4f434b3136 00 17000d11 00000000 40540200 0200 5f5f6e6577636c6d6170 00 17000d11 00000000 7b000000 0100 636f6666656562616265 00 15000d11 00000000 7b000000 0100 6465616462656566 00 0000" + } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj" + } + } + + steps: + { + compare: { left: debug_obj, right: raw_debug_obj } + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + gsi: + { + @count(4) symbols: + { + "CV_SymKind_BLOCK16": { kind: S_CONSTANT } + "__newclmap": { kind: S_GDATA32 } + coffeebabe: { kind: S_GDATA32 } + deadbeef: { kind: S_GDATA32 } + } + indexed_symbol_count: 4 + kind_counts: { S_CONSTANT: 1, S_GDATA32: 3 } + } + } + } + } + } +} diff --git a/src/linker/tests/validate_gsi_procs_and_typedefs.tst b/src/linker/tests/validate_gsi_procs_and_typedefs.tst new file mode 100644 index 000000000..518e6745d --- /dev/null +++ b/src/linker/tests/validate_gsi_procs_and_typedefs.tst @@ -0,0 +1,176 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + debug_obj: + { + file_name: "debug.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + concat: + { + // C13 signature and symbols subsection header. + u32le: 4 + u32le: 0xf1 + u32le: 190 + + // S_OBJNAME: signature 0x123, "debug.obj". + u16le: 16 + u16le: 0x1101 + u32le: 0x123 + text: "debug.obj" + zero: 1 + + // S_GPROC32_ID: zeroed procedure fields, "global_proc". + u16le: 49 + u16le: 0x1147 + zero: 32 + u16le: 0 + hex: "00" + text: "global_proc" + zero: 1 + + // S_PROC_ID_END. + u16le: 2 + u16le: 0x114f + + // S_UDT: type 0, "global_typedef". + u16le: 21 + u16le: 0x1108 + u32le: 0 + text: "global_typedef" + zero: 1 + + // S_OBJNAME: signature 0x123, "debug.obj". + u16le: 16 + u16le: 0x1101 + u32le: 0x123 + text: "debug.obj" + zero: 1 + + // S_LPROC32_ID: zeroed procedure fields, "local_proc". + u16le: 48 + u16le: 0x1146 + zero: 32 + u16le: 0 + hex: "00" + text: "local_proc" + zero: 1 + + // S_UDT: type 0, "local_typedef". + u16le: 20 + u16le: 0x1108 + u32le: 0 + text: "local_typedef" + zero: 1 + + // S_PROC_ID_END. + u16le: 2 + u16le: 0x114f + + // Align the complete C13 subsection to four bytes. + zero: 2 + } + } + } + } + } + } + } + + // Raw helper output for byte-for-byte parity with the typed CodeView records above. + raw_debug_obj: + { + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + hex: "04000000 f1000000 be000000 10000111 23010000 64656275672e6f626a 00 31004711 0000000000000000000000000000000000000000000000000000000000000000 0000 00 676c6f62616c5f70726f63 00 02004f11 15000811 00000000 676c6f62616c5f74797065646566 00 10000111 23010000 64656275672e6f626a 00 30004611 0000000000000000000000000000000000000000000000000000000000000000 0000 00 6c6f63616c5f70726f63 00 14000811 00000000 6c6f63616c5f74797065646566 00 02004f11 0000" + } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj" + } + } + + steps: + { + compare: { left: debug_obj, right: raw_debug_obj } + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + gsi: + { + @count(3) symbols: + { + global_proc: { kind: S_PROCREF, suc_name: 0, sym_off: 24, imod: 2 } + local_proc: { kind: S_LPROCREF, suc_name: 0, sym_off: 100, imod: 2 } + global_typedef: { kind: S_UDT } + @absent local_typedef + } + indexed_symbol_count: 3 + kind_counts: { S_PROCREF: 1, S_LPROCREF: 1, S_UDT: 1 } + } + } + } + } + } +} diff --git a/src/linker/tests/validate_info_stream.tst b/src/linker/tests/validate_info_stream.tst new file mode 100644 index 000000000..c5d408789 --- /dev/null +++ b/src/linker/tests/validate_info_stream.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { validate_info_stream: {} } +} diff --git a/src/linker/tests/validate_psi.tst b/src/linker/tests/validate_psi.tst new file mode 100644 index 000000000..6ca523d5d --- /dev/null +++ b/src/linker/tests/validate_psi.tst @@ -0,0 +1,85 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + test_obj: + { + file_name: "test.obj" + coff: + { + object: + { + machine: x64 + sections: + { + // Executable bytes containing the externally visible function. + text: { name: ".text", permissions: (read, execute), content: code, data: { text: "FOOBAR" } } + // Writable bytes containing the global and static variables. + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "QWE" } } + } + symbols: + { + global_func: { kind: external_function, name: "global_func", section: text, value: 1 } + global_var: { kind: external, name: "global_var", section: data, value: 1 } + static_var: { kind: static, name: "static_var", section: data, value: 1 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full entry.obj test.obj" + } + } + + steps: + { + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + psi: + { + symbols: + { + global_func: { kind: S_PUB32, flags: 2, @range(1, 65535) section } + global_var: { kind: S_PUB32, flags: 0, @range(1, 65535) section } + @absent static_var + } + // Every indexed PSI record must be PUB32, and at least one must exist. + @count(1) kind_counts: { @range(1, 4294967295) S_PUB32 } + @range(1, 4294967295) indexed_symbol_count + } + } + } + } + } +} diff --git a/src/linker/tests/weak_alias_comdat_duplicate_fallback.tst b/src/linker/tests/weak_alias_comdat_duplicate_fallback.tst new file mode 100644 index 000000000..b9cb50288 --- /dev/null +++ b/src/linker/tests/weak_alias_comdat_duplicate_fallback.tst @@ -0,0 +1,493 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + dummy_0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_3: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_4: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_5: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_6: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_7: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_8: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_9: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_10: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_11: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_12: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_13: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_14: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_15: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_16: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_17: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_18: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_19: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_20: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_21: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_22: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_23: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_24: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_25: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_26: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_27: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_28: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_29: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_30: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_31: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_32: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_33: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_34: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_35: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_36: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_37: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_38: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_39: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_40: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_41: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_42: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_43: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_44: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_45: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_46: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_47: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_48: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_49: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_50: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_51: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_52: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_53: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_54: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_55: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_56: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_57: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_58: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_59: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_60: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_61: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_62: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_63: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + text0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + text1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + xdata0: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + text2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0c3" } } + } + symbols: + { + dummy_0_def: { kind: section_definition, section: dummy_0, selection: Any } + dummy_0_symbol: { kind: external_function, name: "?dummy_a_0@@YAXXZ", section: dummy_0, value: 0 } + dummy_1_def: { kind: section_definition, section: dummy_1, selection: Any } + dummy_1_symbol: { kind: external_function, name: "?dummy_a_1@@YAXXZ", section: dummy_1, value: 0 } + dummy_2_def: { kind: section_definition, section: dummy_2, selection: Any } + dummy_2_symbol: { kind: external_function, name: "?dummy_a_2@@YAXXZ", section: dummy_2, value: 0 } + dummy_3_def: { kind: section_definition, section: dummy_3, selection: Any } + dummy_3_symbol: { kind: external_function, name: "?dummy_a_3@@YAXXZ", section: dummy_3, value: 0 } + dummy_4_def: { kind: section_definition, section: dummy_4, selection: Any } + dummy_4_symbol: { kind: external_function, name: "?dummy_a_4@@YAXXZ", section: dummy_4, value: 0 } + dummy_5_def: { kind: section_definition, section: dummy_5, selection: Any } + dummy_5_symbol: { kind: external_function, name: "?dummy_a_5@@YAXXZ", section: dummy_5, value: 0 } + dummy_6_def: { kind: section_definition, section: dummy_6, selection: Any } + dummy_6_symbol: { kind: external_function, name: "?dummy_a_6@@YAXXZ", section: dummy_6, value: 0 } + dummy_7_def: { kind: section_definition, section: dummy_7, selection: Any } + dummy_7_symbol: { kind: external_function, name: "?dummy_a_7@@YAXXZ", section: dummy_7, value: 0 } + dummy_8_def: { kind: section_definition, section: dummy_8, selection: Any } + dummy_8_symbol: { kind: external_function, name: "?dummy_a_8@@YAXXZ", section: dummy_8, value: 0 } + dummy_9_def: { kind: section_definition, section: dummy_9, selection: Any } + dummy_9_symbol: { kind: external_function, name: "?dummy_a_9@@YAXXZ", section: dummy_9, value: 0 } + dummy_10_def: { kind: section_definition, section: dummy_10, selection: Any } + dummy_10_symbol: { kind: external_function, name: "?dummy_a_10@@YAXXZ", section: dummy_10, value: 0 } + dummy_11_def: { kind: section_definition, section: dummy_11, selection: Any } + dummy_11_symbol: { kind: external_function, name: "?dummy_a_11@@YAXXZ", section: dummy_11, value: 0 } + dummy_12_def: { kind: section_definition, section: dummy_12, selection: Any } + dummy_12_symbol: { kind: external_function, name: "?dummy_a_12@@YAXXZ", section: dummy_12, value: 0 } + dummy_13_def: { kind: section_definition, section: dummy_13, selection: Any } + dummy_13_symbol: { kind: external_function, name: "?dummy_a_13@@YAXXZ", section: dummy_13, value: 0 } + dummy_14_def: { kind: section_definition, section: dummy_14, selection: Any } + dummy_14_symbol: { kind: external_function, name: "?dummy_a_14@@YAXXZ", section: dummy_14, value: 0 } + dummy_15_def: { kind: section_definition, section: dummy_15, selection: Any } + dummy_15_symbol: { kind: external_function, name: "?dummy_a_15@@YAXXZ", section: dummy_15, value: 0 } + dummy_16_def: { kind: section_definition, section: dummy_16, selection: Any } + dummy_16_symbol: { kind: external_function, name: "?dummy_a_16@@YAXXZ", section: dummy_16, value: 0 } + dummy_17_def: { kind: section_definition, section: dummy_17, selection: Any } + dummy_17_symbol: { kind: external_function, name: "?dummy_a_17@@YAXXZ", section: dummy_17, value: 0 } + dummy_18_def: { kind: section_definition, section: dummy_18, selection: Any } + dummy_18_symbol: { kind: external_function, name: "?dummy_a_18@@YAXXZ", section: dummy_18, value: 0 } + dummy_19_def: { kind: section_definition, section: dummy_19, selection: Any } + dummy_19_symbol: { kind: external_function, name: "?dummy_a_19@@YAXXZ", section: dummy_19, value: 0 } + dummy_20_def: { kind: section_definition, section: dummy_20, selection: Any } + dummy_20_symbol: { kind: external_function, name: "?dummy_a_20@@YAXXZ", section: dummy_20, value: 0 } + dummy_21_def: { kind: section_definition, section: dummy_21, selection: Any } + dummy_21_symbol: { kind: external_function, name: "?dummy_a_21@@YAXXZ", section: dummy_21, value: 0 } + dummy_22_def: { kind: section_definition, section: dummy_22, selection: Any } + dummy_22_symbol: { kind: external_function, name: "?dummy_a_22@@YAXXZ", section: dummy_22, value: 0 } + dummy_23_def: { kind: section_definition, section: dummy_23, selection: Any } + dummy_23_symbol: { kind: external_function, name: "?dummy_a_23@@YAXXZ", section: dummy_23, value: 0 } + dummy_24_def: { kind: section_definition, section: dummy_24, selection: Any } + dummy_24_symbol: { kind: external_function, name: "?dummy_a_24@@YAXXZ", section: dummy_24, value: 0 } + dummy_25_def: { kind: section_definition, section: dummy_25, selection: Any } + dummy_25_symbol: { kind: external_function, name: "?dummy_a_25@@YAXXZ", section: dummy_25, value: 0 } + dummy_26_def: { kind: section_definition, section: dummy_26, selection: Any } + dummy_26_symbol: { kind: external_function, name: "?dummy_a_26@@YAXXZ", section: dummy_26, value: 0 } + dummy_27_def: { kind: section_definition, section: dummy_27, selection: Any } + dummy_27_symbol: { kind: external_function, name: "?dummy_a_27@@YAXXZ", section: dummy_27, value: 0 } + dummy_28_def: { kind: section_definition, section: dummy_28, selection: Any } + dummy_28_symbol: { kind: external_function, name: "?dummy_a_28@@YAXXZ", section: dummy_28, value: 0 } + dummy_29_def: { kind: section_definition, section: dummy_29, selection: Any } + dummy_29_symbol: { kind: external_function, name: "?dummy_a_29@@YAXXZ", section: dummy_29, value: 0 } + dummy_30_def: { kind: section_definition, section: dummy_30, selection: Any } + dummy_30_symbol: { kind: external_function, name: "?dummy_a_30@@YAXXZ", section: dummy_30, value: 0 } + dummy_31_def: { kind: section_definition, section: dummy_31, selection: Any } + dummy_31_symbol: { kind: external_function, name: "?dummy_a_31@@YAXXZ", section: dummy_31, value: 0 } + dummy_32_def: { kind: section_definition, section: dummy_32, selection: Any } + dummy_32_symbol: { kind: external_function, name: "?dummy_a_32@@YAXXZ", section: dummy_32, value: 0 } + dummy_33_def: { kind: section_definition, section: dummy_33, selection: Any } + dummy_33_symbol: { kind: external_function, name: "?dummy_a_33@@YAXXZ", section: dummy_33, value: 0 } + dummy_34_def: { kind: section_definition, section: dummy_34, selection: Any } + dummy_34_symbol: { kind: external_function, name: "?dummy_a_34@@YAXXZ", section: dummy_34, value: 0 } + dummy_35_def: { kind: section_definition, section: dummy_35, selection: Any } + dummy_35_symbol: { kind: external_function, name: "?dummy_a_35@@YAXXZ", section: dummy_35, value: 0 } + dummy_36_def: { kind: section_definition, section: dummy_36, selection: Any } + dummy_36_symbol: { kind: external_function, name: "?dummy_a_36@@YAXXZ", section: dummy_36, value: 0 } + dummy_37_def: { kind: section_definition, section: dummy_37, selection: Any } + dummy_37_symbol: { kind: external_function, name: "?dummy_a_37@@YAXXZ", section: dummy_37, value: 0 } + dummy_38_def: { kind: section_definition, section: dummy_38, selection: Any } + dummy_38_symbol: { kind: external_function, name: "?dummy_a_38@@YAXXZ", section: dummy_38, value: 0 } + dummy_39_def: { kind: section_definition, section: dummy_39, selection: Any } + dummy_39_symbol: { kind: external_function, name: "?dummy_a_39@@YAXXZ", section: dummy_39, value: 0 } + dummy_40_def: { kind: section_definition, section: dummy_40, selection: Any } + dummy_40_symbol: { kind: external_function, name: "?dummy_a_40@@YAXXZ", section: dummy_40, value: 0 } + dummy_41_def: { kind: section_definition, section: dummy_41, selection: Any } + dummy_41_symbol: { kind: external_function, name: "?dummy_a_41@@YAXXZ", section: dummy_41, value: 0 } + dummy_42_def: { kind: section_definition, section: dummy_42, selection: Any } + dummy_42_symbol: { kind: external_function, name: "?dummy_a_42@@YAXXZ", section: dummy_42, value: 0 } + dummy_43_def: { kind: section_definition, section: dummy_43, selection: Any } + dummy_43_symbol: { kind: external_function, name: "?dummy_a_43@@YAXXZ", section: dummy_43, value: 0 } + dummy_44_def: { kind: section_definition, section: dummy_44, selection: Any } + dummy_44_symbol: { kind: external_function, name: "?dummy_a_44@@YAXXZ", section: dummy_44, value: 0 } + dummy_45_def: { kind: section_definition, section: dummy_45, selection: Any } + dummy_45_symbol: { kind: external_function, name: "?dummy_a_45@@YAXXZ", section: dummy_45, value: 0 } + dummy_46_def: { kind: section_definition, section: dummy_46, selection: Any } + dummy_46_symbol: { kind: external_function, name: "?dummy_a_46@@YAXXZ", section: dummy_46, value: 0 } + dummy_47_def: { kind: section_definition, section: dummy_47, selection: Any } + dummy_47_symbol: { kind: external_function, name: "?dummy_a_47@@YAXXZ", section: dummy_47, value: 0 } + dummy_48_def: { kind: section_definition, section: dummy_48, selection: Any } + dummy_48_symbol: { kind: external_function, name: "?dummy_a_48@@YAXXZ", section: dummy_48, value: 0 } + dummy_49_def: { kind: section_definition, section: dummy_49, selection: Any } + dummy_49_symbol: { kind: external_function, name: "?dummy_a_49@@YAXXZ", section: dummy_49, value: 0 } + dummy_50_def: { kind: section_definition, section: dummy_50, selection: Any } + dummy_50_symbol: { kind: external_function, name: "?dummy_a_50@@YAXXZ", section: dummy_50, value: 0 } + dummy_51_def: { kind: section_definition, section: dummy_51, selection: Any } + dummy_51_symbol: { kind: external_function, name: "?dummy_a_51@@YAXXZ", section: dummy_51, value: 0 } + dummy_52_def: { kind: section_definition, section: dummy_52, selection: Any } + dummy_52_symbol: { kind: external_function, name: "?dummy_a_52@@YAXXZ", section: dummy_52, value: 0 } + dummy_53_def: { kind: section_definition, section: dummy_53, selection: Any } + dummy_53_symbol: { kind: external_function, name: "?dummy_a_53@@YAXXZ", section: dummy_53, value: 0 } + dummy_54_def: { kind: section_definition, section: dummy_54, selection: Any } + dummy_54_symbol: { kind: external_function, name: "?dummy_a_54@@YAXXZ", section: dummy_54, value: 0 } + dummy_55_def: { kind: section_definition, section: dummy_55, selection: Any } + dummy_55_symbol: { kind: external_function, name: "?dummy_a_55@@YAXXZ", section: dummy_55, value: 0 } + dummy_56_def: { kind: section_definition, section: dummy_56, selection: Any } + dummy_56_symbol: { kind: external_function, name: "?dummy_a_56@@YAXXZ", section: dummy_56, value: 0 } + dummy_57_def: { kind: section_definition, section: dummy_57, selection: Any } + dummy_57_symbol: { kind: external_function, name: "?dummy_a_57@@YAXXZ", section: dummy_57, value: 0 } + dummy_58_def: { kind: section_definition, section: dummy_58, selection: Any } + dummy_58_symbol: { kind: external_function, name: "?dummy_a_58@@YAXXZ", section: dummy_58, value: 0 } + dummy_59_def: { kind: section_definition, section: dummy_59, selection: Any } + dummy_59_symbol: { kind: external_function, name: "?dummy_a_59@@YAXXZ", section: dummy_59, value: 0 } + dummy_60_def: { kind: section_definition, section: dummy_60, selection: Any } + dummy_60_symbol: { kind: external_function, name: "?dummy_a_60@@YAXXZ", section: dummy_60, value: 0 } + dummy_61_def: { kind: section_definition, section: dummy_61, selection: Any } + dummy_61_symbol: { kind: external_function, name: "?dummy_a_61@@YAXXZ", section: dummy_61, value: 0 } + dummy_62_def: { kind: section_definition, section: dummy_62, selection: Any } + dummy_62_symbol: { kind: external_function, name: "?dummy_a_62@@YAXXZ", section: dummy_62, value: 0 } + dummy_63_def: { kind: section_definition, section: dummy_63, selection: Any } + dummy_63_symbol: { kind: external_function, name: "?dummy_a_63@@YAXXZ", section: dummy_63, value: 0 } + text0_def: { kind: section_definition, section: text0, selection: Any } + shared_count_delete: { kind: external_function, name: "??_G__shared_count@__Cr@std@@MEAAPEAXI@Z", section: text0, value: 0 } + text1_def: { kind: section_definition, section: text1, selection: Any } + shared_weak_count_delete: { kind: external_function, name: "??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", section: text1, value: 0 } + xdata0_def: { kind: section_definition, section: xdata0, selection: Associative, associate: text0 } + text2_def: { kind: section_definition, section: text2, selection: Any } + ptr_delete: { kind: external_function, name: "??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", section: text2, value: 0 } + ptr_destroy: { kind: weak, name: "??_E?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", fallback: ptr_delete, search: alias } + shared_count_destroy: { kind: weak, name: "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_count_delete, search: alias } + shared_weak_count_destroy: { kind: weak, name: "??_E__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_weak_count_delete, search: alias } + } + } } + } + + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + dummy_0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_3: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_4: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_5: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_6: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_7: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_8: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_9: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_10: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_11: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_12: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_13: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_14: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_15: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_16: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_17: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_18: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_19: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_20: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_21: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_22: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_23: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_24: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_25: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_26: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_27: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_28: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_29: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_30: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_31: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_32: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_33: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_34: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_35: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_36: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_37: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_38: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_39: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_40: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_41: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_42: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_43: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_44: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_45: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_46: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_47: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_48: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_49: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_50: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_51: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_52: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_53: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_54: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_55: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_56: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_57: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_58: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_59: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_60: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_61: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_62: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_63: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + text0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + text1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + xdata0: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + text2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0c3" } } + } + symbols: + { + dummy_0_def: { kind: section_definition, section: dummy_0, selection: Any } + dummy_0_symbol: { kind: external_function, name: "?dummy_b_0@@YAXXZ", section: dummy_0, value: 0 } + dummy_1_def: { kind: section_definition, section: dummy_1, selection: Any } + dummy_1_symbol: { kind: external_function, name: "?dummy_b_1@@YAXXZ", section: dummy_1, value: 0 } + dummy_2_def: { kind: section_definition, section: dummy_2, selection: Any } + dummy_2_symbol: { kind: external_function, name: "?dummy_b_2@@YAXXZ", section: dummy_2, value: 0 } + dummy_3_def: { kind: section_definition, section: dummy_3, selection: Any } + dummy_3_symbol: { kind: external_function, name: "?dummy_b_3@@YAXXZ", section: dummy_3, value: 0 } + dummy_4_def: { kind: section_definition, section: dummy_4, selection: Any } + dummy_4_symbol: { kind: external_function, name: "?dummy_b_4@@YAXXZ", section: dummy_4, value: 0 } + dummy_5_def: { kind: section_definition, section: dummy_5, selection: Any } + dummy_5_symbol: { kind: external_function, name: "?dummy_b_5@@YAXXZ", section: dummy_5, value: 0 } + dummy_6_def: { kind: section_definition, section: dummy_6, selection: Any } + dummy_6_symbol: { kind: external_function, name: "?dummy_b_6@@YAXXZ", section: dummy_6, value: 0 } + dummy_7_def: { kind: section_definition, section: dummy_7, selection: Any } + dummy_7_symbol: { kind: external_function, name: "?dummy_b_7@@YAXXZ", section: dummy_7, value: 0 } + dummy_8_def: { kind: section_definition, section: dummy_8, selection: Any } + dummy_8_symbol: { kind: external_function, name: "?dummy_b_8@@YAXXZ", section: dummy_8, value: 0 } + dummy_9_def: { kind: section_definition, section: dummy_9, selection: Any } + dummy_9_symbol: { kind: external_function, name: "?dummy_b_9@@YAXXZ", section: dummy_9, value: 0 } + dummy_10_def: { kind: section_definition, section: dummy_10, selection: Any } + dummy_10_symbol: { kind: external_function, name: "?dummy_b_10@@YAXXZ", section: dummy_10, value: 0 } + dummy_11_def: { kind: section_definition, section: dummy_11, selection: Any } + dummy_11_symbol: { kind: external_function, name: "?dummy_b_11@@YAXXZ", section: dummy_11, value: 0 } + dummy_12_def: { kind: section_definition, section: dummy_12, selection: Any } + dummy_12_symbol: { kind: external_function, name: "?dummy_b_12@@YAXXZ", section: dummy_12, value: 0 } + dummy_13_def: { kind: section_definition, section: dummy_13, selection: Any } + dummy_13_symbol: { kind: external_function, name: "?dummy_b_13@@YAXXZ", section: dummy_13, value: 0 } + dummy_14_def: { kind: section_definition, section: dummy_14, selection: Any } + dummy_14_symbol: { kind: external_function, name: "?dummy_b_14@@YAXXZ", section: dummy_14, value: 0 } + dummy_15_def: { kind: section_definition, section: dummy_15, selection: Any } + dummy_15_symbol: { kind: external_function, name: "?dummy_b_15@@YAXXZ", section: dummy_15, value: 0 } + dummy_16_def: { kind: section_definition, section: dummy_16, selection: Any } + dummy_16_symbol: { kind: external_function, name: "?dummy_b_16@@YAXXZ", section: dummy_16, value: 0 } + dummy_17_def: { kind: section_definition, section: dummy_17, selection: Any } + dummy_17_symbol: { kind: external_function, name: "?dummy_b_17@@YAXXZ", section: dummy_17, value: 0 } + dummy_18_def: { kind: section_definition, section: dummy_18, selection: Any } + dummy_18_symbol: { kind: external_function, name: "?dummy_b_18@@YAXXZ", section: dummy_18, value: 0 } + dummy_19_def: { kind: section_definition, section: dummy_19, selection: Any } + dummy_19_symbol: { kind: external_function, name: "?dummy_b_19@@YAXXZ", section: dummy_19, value: 0 } + dummy_20_def: { kind: section_definition, section: dummy_20, selection: Any } + dummy_20_symbol: { kind: external_function, name: "?dummy_b_20@@YAXXZ", section: dummy_20, value: 0 } + dummy_21_def: { kind: section_definition, section: dummy_21, selection: Any } + dummy_21_symbol: { kind: external_function, name: "?dummy_b_21@@YAXXZ", section: dummy_21, value: 0 } + dummy_22_def: { kind: section_definition, section: dummy_22, selection: Any } + dummy_22_symbol: { kind: external_function, name: "?dummy_b_22@@YAXXZ", section: dummy_22, value: 0 } + dummy_23_def: { kind: section_definition, section: dummy_23, selection: Any } + dummy_23_symbol: { kind: external_function, name: "?dummy_b_23@@YAXXZ", section: dummy_23, value: 0 } + dummy_24_def: { kind: section_definition, section: dummy_24, selection: Any } + dummy_24_symbol: { kind: external_function, name: "?dummy_b_24@@YAXXZ", section: dummy_24, value: 0 } + dummy_25_def: { kind: section_definition, section: dummy_25, selection: Any } + dummy_25_symbol: { kind: external_function, name: "?dummy_b_25@@YAXXZ", section: dummy_25, value: 0 } + dummy_26_def: { kind: section_definition, section: dummy_26, selection: Any } + dummy_26_symbol: { kind: external_function, name: "?dummy_b_26@@YAXXZ", section: dummy_26, value: 0 } + dummy_27_def: { kind: section_definition, section: dummy_27, selection: Any } + dummy_27_symbol: { kind: external_function, name: "?dummy_b_27@@YAXXZ", section: dummy_27, value: 0 } + dummy_28_def: { kind: section_definition, section: dummy_28, selection: Any } + dummy_28_symbol: { kind: external_function, name: "?dummy_b_28@@YAXXZ", section: dummy_28, value: 0 } + dummy_29_def: { kind: section_definition, section: dummy_29, selection: Any } + dummy_29_symbol: { kind: external_function, name: "?dummy_b_29@@YAXXZ", section: dummy_29, value: 0 } + dummy_30_def: { kind: section_definition, section: dummy_30, selection: Any } + dummy_30_symbol: { kind: external_function, name: "?dummy_b_30@@YAXXZ", section: dummy_30, value: 0 } + dummy_31_def: { kind: section_definition, section: dummy_31, selection: Any } + dummy_31_symbol: { kind: external_function, name: "?dummy_b_31@@YAXXZ", section: dummy_31, value: 0 } + dummy_32_def: { kind: section_definition, section: dummy_32, selection: Any } + dummy_32_symbol: { kind: external_function, name: "?dummy_b_32@@YAXXZ", section: dummy_32, value: 0 } + dummy_33_def: { kind: section_definition, section: dummy_33, selection: Any } + dummy_33_symbol: { kind: external_function, name: "?dummy_b_33@@YAXXZ", section: dummy_33, value: 0 } + dummy_34_def: { kind: section_definition, section: dummy_34, selection: Any } + dummy_34_symbol: { kind: external_function, name: "?dummy_b_34@@YAXXZ", section: dummy_34, value: 0 } + dummy_35_def: { kind: section_definition, section: dummy_35, selection: Any } + dummy_35_symbol: { kind: external_function, name: "?dummy_b_35@@YAXXZ", section: dummy_35, value: 0 } + dummy_36_def: { kind: section_definition, section: dummy_36, selection: Any } + dummy_36_symbol: { kind: external_function, name: "?dummy_b_36@@YAXXZ", section: dummy_36, value: 0 } + dummy_37_def: { kind: section_definition, section: dummy_37, selection: Any } + dummy_37_symbol: { kind: external_function, name: "?dummy_b_37@@YAXXZ", section: dummy_37, value: 0 } + dummy_38_def: { kind: section_definition, section: dummy_38, selection: Any } + dummy_38_symbol: { kind: external_function, name: "?dummy_b_38@@YAXXZ", section: dummy_38, value: 0 } + dummy_39_def: { kind: section_definition, section: dummy_39, selection: Any } + dummy_39_symbol: { kind: external_function, name: "?dummy_b_39@@YAXXZ", section: dummy_39, value: 0 } + dummy_40_def: { kind: section_definition, section: dummy_40, selection: Any } + dummy_40_symbol: { kind: external_function, name: "?dummy_b_40@@YAXXZ", section: dummy_40, value: 0 } + dummy_41_def: { kind: section_definition, section: dummy_41, selection: Any } + dummy_41_symbol: { kind: external_function, name: "?dummy_b_41@@YAXXZ", section: dummy_41, value: 0 } + dummy_42_def: { kind: section_definition, section: dummy_42, selection: Any } + dummy_42_symbol: { kind: external_function, name: "?dummy_b_42@@YAXXZ", section: dummy_42, value: 0 } + dummy_43_def: { kind: section_definition, section: dummy_43, selection: Any } + dummy_43_symbol: { kind: external_function, name: "?dummy_b_43@@YAXXZ", section: dummy_43, value: 0 } + dummy_44_def: { kind: section_definition, section: dummy_44, selection: Any } + dummy_44_symbol: { kind: external_function, name: "?dummy_b_44@@YAXXZ", section: dummy_44, value: 0 } + dummy_45_def: { kind: section_definition, section: dummy_45, selection: Any } + dummy_45_symbol: { kind: external_function, name: "?dummy_b_45@@YAXXZ", section: dummy_45, value: 0 } + dummy_46_def: { kind: section_definition, section: dummy_46, selection: Any } + dummy_46_symbol: { kind: external_function, name: "?dummy_b_46@@YAXXZ", section: dummy_46, value: 0 } + dummy_47_def: { kind: section_definition, section: dummy_47, selection: Any } + dummy_47_symbol: { kind: external_function, name: "?dummy_b_47@@YAXXZ", section: dummy_47, value: 0 } + dummy_48_def: { kind: section_definition, section: dummy_48, selection: Any } + dummy_48_symbol: { kind: external_function, name: "?dummy_b_48@@YAXXZ", section: dummy_48, value: 0 } + dummy_49_def: { kind: section_definition, section: dummy_49, selection: Any } + dummy_49_symbol: { kind: external_function, name: "?dummy_b_49@@YAXXZ", section: dummy_49, value: 0 } + dummy_50_def: { kind: section_definition, section: dummy_50, selection: Any } + dummy_50_symbol: { kind: external_function, name: "?dummy_b_50@@YAXXZ", section: dummy_50, value: 0 } + dummy_51_def: { kind: section_definition, section: dummy_51, selection: Any } + dummy_51_symbol: { kind: external_function, name: "?dummy_b_51@@YAXXZ", section: dummy_51, value: 0 } + dummy_52_def: { kind: section_definition, section: dummy_52, selection: Any } + dummy_52_symbol: { kind: external_function, name: "?dummy_b_52@@YAXXZ", section: dummy_52, value: 0 } + dummy_53_def: { kind: section_definition, section: dummy_53, selection: Any } + dummy_53_symbol: { kind: external_function, name: "?dummy_b_53@@YAXXZ", section: dummy_53, value: 0 } + dummy_54_def: { kind: section_definition, section: dummy_54, selection: Any } + dummy_54_symbol: { kind: external_function, name: "?dummy_b_54@@YAXXZ", section: dummy_54, value: 0 } + dummy_55_def: { kind: section_definition, section: dummy_55, selection: Any } + dummy_55_symbol: { kind: external_function, name: "?dummy_b_55@@YAXXZ", section: dummy_55, value: 0 } + dummy_56_def: { kind: section_definition, section: dummy_56, selection: Any } + dummy_56_symbol: { kind: external_function, name: "?dummy_b_56@@YAXXZ", section: dummy_56, value: 0 } + dummy_57_def: { kind: section_definition, section: dummy_57, selection: Any } + dummy_57_symbol: { kind: external_function, name: "?dummy_b_57@@YAXXZ", section: dummy_57, value: 0 } + dummy_58_def: { kind: section_definition, section: dummy_58, selection: Any } + dummy_58_symbol: { kind: external_function, name: "?dummy_b_58@@YAXXZ", section: dummy_58, value: 0 } + dummy_59_def: { kind: section_definition, section: dummy_59, selection: Any } + dummy_59_symbol: { kind: external_function, name: "?dummy_b_59@@YAXXZ", section: dummy_59, value: 0 } + dummy_60_def: { kind: section_definition, section: dummy_60, selection: Any } + dummy_60_symbol: { kind: external_function, name: "?dummy_b_60@@YAXXZ", section: dummy_60, value: 0 } + dummy_61_def: { kind: section_definition, section: dummy_61, selection: Any } + dummy_61_symbol: { kind: external_function, name: "?dummy_b_61@@YAXXZ", section: dummy_61, value: 0 } + dummy_62_def: { kind: section_definition, section: dummy_62, selection: Any } + dummy_62_symbol: { kind: external_function, name: "?dummy_b_62@@YAXXZ", section: dummy_62, value: 0 } + dummy_63_def: { kind: section_definition, section: dummy_63, selection: Any } + dummy_63_symbol: { kind: external_function, name: "?dummy_b_63@@YAXXZ", section: dummy_63, value: 0 } + text0_def: { kind: section_definition, section: text0, selection: Any } + shared_count_delete: { kind: external_function, name: "??_G__shared_count@__Cr@std@@MEAAPEAXI@Z", section: text0, value: 0 } + text1_def: { kind: section_definition, section: text1, selection: Any } + shared_weak_count_delete: { kind: external_function, name: "??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", section: text1, value: 0 } + xdata0_def: { kind: section_definition, section: xdata0, selection: Associative, associate: text0 } + text2_def: { kind: section_definition, section: text2, selection: Any } + ptr_delete: { kind: external_function, name: "??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", section: text2, value: 0 } + ptr_destroy: { kind: weak, name: "??_E?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", fallback: ptr_delete, search: alias } + shared_count_destroy: { kind: weak, name: "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_count_delete, search: alias } + shared_weak_count_destroy: { kind: weak, name: "??_E__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_weak_count_delete, search: alias } + } + } } + } + + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } + } + relocations: { shared_count_ref: { type: Addr32Nb, offset: 3, symbol: shared_count_destroy } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + shared_count_destroy: { kind: undefined, name: "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z" } + } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj" + } + link: + { + args: "/subsystem:console /entry:entry /out:a.exe b.obj a.obj entry.obj" + } + } + steps: {} +} diff --git a/src/linker/tests/weak_alias_vs_weak_alias.tst b/src/linker/tests/weak_alias_vs_weak_alias.tst new file mode 100644 index 000000000..4efffde90 --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_alias.tst @@ -0,0 +1,30 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + qwe: { kind: absolute, name: "qwe", value: 273, storage: external } + sym: { kind: weak, name: "sym", fallback: qwe, search: alias } + } } } } + b_obj: { file_name: "b.obj", coff: { object: { machine: x64, symbols: { + ewq: { kind: absolute, name: "ewq", value: 546, storage: external } + sym: { kind: weak, name: "sym", fallback: ewq, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { sym_ref: { type: Addr32, offset: 3, symbol: sym } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + sym: { kind: undefined, name: "sym" } + } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj", expect_exit: 24 } } + steps: {} +} diff --git a/src/linker/tests/weak_alias_vs_weak_antidep.tst b/src/linker/tests/weak_alias_vs_weak_antidep.tst new file mode 100644 index 000000000..99bfa6bed --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_antidep.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: alias } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_alias_vs_weak_lib.tst b/src/linker/tests/weak_alias_vs_weak_lib.tst new file mode 100644 index 000000000..99bfa6bed --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_lib.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: alias } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_alias_vs_weak_nolib.tst b/src/linker/tests/weak_alias_vs_weak_nolib.tst new file mode 100644 index 000000000..256f89205 --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_nolib.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: alias } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_alias.tst b/src/linker/tests/weak_antidep_vs_weak_alias.tst new file mode 100644 index 000000000..e8eb9d413 --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_alias.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c011010000c3 } } } } } } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_antidep.tst b/src/linker/tests/weak_antidep_vs_weak_antidep.tst new file mode 100644 index 000000000..0170df800 --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_antidep.tst @@ -0,0 +1,39 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + a_first_image: { file_name: "a_first.exe", pe: {} } + entry_first_image: { file_name: "entry_first.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from a.obj + link: { args: "/subsystem:console /entry:entry /out:a_first.exe a.obj entry.obj", artifact: a_first_image } + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:entry_first.exe entry.obj a.obj", artifact: entry_first_image } + } + steps: + { + expect_pe: { artifact: a_first_image, expected: { pe: { sections: { ".text": { data: 48c7c011010000c3 } } } } } + expect_pe: { artifact: entry_first_image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } + } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_lib.tst b/src/linker/tests/weak_antidep_vs_weak_lib.tst new file mode 100644 index 000000000..e3aaac390 --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_lib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_nolib.tst b/src/linker/tests/weak_antidep_vs_weak_nolib.tst new file mode 100644 index 000000000..0ea8dc8ff --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_nolib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_cycle.tst b/src/linker/tests/weak_cycle.tst new file mode 100644 index 000000000..b9085c3bb --- /dev/null +++ b/src/linker/tests/weak_cycle.tst @@ -0,0 +1,44 @@ +test: +{ + + artifacts: + { + ab: + { + file_name: "ab.obj" + coff: { object: { machine: x64, symbols: { + B: { kind: undefined, name: "B" } + A: { kind: weak, name: "A", fallback: B, search: alias } + } } } + } + ba: + { + file_name: "ba.obj" + coff: { object: { machine: x64, symbols: { + A: { kind: undefined, name: "A" } + B: { kind: weak, name: "B", fallback: A, search: alias } + } } } + } + entry: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } + } }, symbols: { + my_entry: { kind: external, name: "my_entry", section: text, value: 0 } + } } } + } + } + + build: + { + // give a generous 3 seconds + link: + { + args: "/subsystem:console /entry:my_entry entry.obj ab.obj ba.obj" + timeout_ms: 3000 + expect_exit: any + } + } + steps: {} +} diff --git a/src/linker/tests/weak_lib_vs_weak_alias.tst b/src/linker/tests/weak_lib_vs_weak_alias.tst new file mode 100644 index 000000000..d52e1ad9c --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_alias.tst @@ -0,0 +1,31 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", expect_exit: 24 } + } + steps: {} +} diff --git a/src/linker/tests/weak_lib_vs_weak_antidep.tst b/src/linker/tests/weak_lib_vs_weak_antidep.tst new file mode 100644 index 000000000..d15e5ac3d --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_antidep.tst @@ -0,0 +1,32 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from a.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_lib_vs_weak_lib.tst b/src/linker/tests/weak_lib_vs_weak_lib.tst new file mode 100644 index 000000000..c6bd136e4 --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_lib.tst @@ -0,0 +1,39 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + a_first_image: { file_name: "a_first.exe", pe: {} } + entry_first_image: { file_name: "entry_first.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from a.obj + link: { args: "/subsystem:console /entry:entry /out:a_first.exe a.obj entry.obj", artifact: a_first_image } + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:entry_first.exe entry.obj a.obj", artifact: entry_first_image } + } + steps: + { + expect_pe: { artifact: a_first_image, expected: { pe: { sections: { ".text": { data: 48c7c011010000c3 } } } } } + expect_pe: { artifact: entry_first_image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } + } +} diff --git a/src/linker/tests/weak_lib_vs_weak_nolib.tst b/src/linker/tests/weak_lib_vs_weak_nolib.tst new file mode 100644 index 000000000..046ad1ff1 --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_nolib.tst @@ -0,0 +1,32 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_nolib_vs_weak_alias.tst b/src/linker/tests/weak_nolib_vs_weak_alias.tst new file mode 100644 index 000000000..a8518f0f9 --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_alias.tst @@ -0,0 +1,27 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", expect_exit: 24 } } + steps: {} +} diff --git a/src/linker/tests/weak_nolib_vs_weak_antidep.tst b/src/linker/tests/weak_nolib_vs_weak_antidep.tst new file mode 100644 index 000000000..2b8ae25e9 --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_antidep.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_nolib_vs_weak_lib.tst b/src/linker/tests/weak_nolib_vs_weak_lib.tst new file mode 100644 index 000000000..e5446306a --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_lib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_nolib_vs_weak_nolib.tst b/src/linker/tests/weak_nolib_vs_weak_nolib.tst new file mode 100644 index 000000000..79a1e5579 --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_nolib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_tag.tst b/src/linker/tests/weak_tag.tst new file mode 100644 index 000000000..84ed22dcb --- /dev/null +++ b/src/linker/tests/weak_tag.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + weak_tag_obj: + { + file_name: "weak_tag.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "00000000" } + relocations: { strong_second_ref: { type: Addr32, offset: 0, symbol: strong_second } } + } }, symbols: { + abs: { kind: absolute, name: "abs", value: 305419896, storage: static } + strong_first: { kind: weak, name: "strong_first", fallback: abs, search: alias } + strong_second: { kind: weak, name: "strong_second", fallback: strong_first, search: alias } + } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } + } }, symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe weak_tag.obj entry.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".data": { virtual_size: 4, data: 78563412 } } } } } } +} diff --git a/src/linker/tests/weak_vs_common.tst b/src/linker/tests/weak_vs_common.tst new file mode 100644 index 000000000..697f1b037 --- /dev/null +++ b/src/linker/tests/weak_vs_common.tst @@ -0,0 +1,39 @@ +test: +{ + artifacts: + { + weak_obj: + { + file_name: "weak.obj" + coff: { object: { machine: x64, + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "a" } } } + symbols: { + a: { kind: static, name: "_a", section: a, value: 0 } + w: { kind: weak, name: "w", fallback: a, search: search_library } + } + } } + } + common_obj: { file_name: "common.obj", coff: { object: { machine: x64, symbols: { w: { kind: common, name: "w", size: 2 } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { w_ref: { type: Addr32Nb, offset: 0, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + w: { kind: undefined, name: "w" } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe common.obj weak.obj entry.obj" } + link: { args: "/subsystem:console /entry:entry /out:a.exe weak.obj common.obj entry.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".bss": { virtual_size: 2, file_size: 0 } } } } } } +} diff --git a/src/linker/tests/whole_archive.tst b/src/linker/tests/whole_archive.tst new file mode 100644 index 000000000..9b64fb54d --- /dev/null +++ b/src/linker/tests/whole_archive.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + a_lib: + { + file_name: "a.lib" + coff: { library: { second_linker_member: true, members: { a_member: { + path: "a.obj" + object: { machine: x64, sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "a" } } } } + } } } } + } + b_lib: + { + file_name: "b.lib" + coff: { library: { second_linker_member: true, members: { b_member: { + path: "b.obj" + object: { machine: x64, sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, data: { text: "b" } } } } + } } } } + } + all_image: { file_name: "all_libs.exe", pe: {} } + a_image: { file_name: "only_a.exe", pe: {} } + b_image: { file_name: "only_b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:all_libs.exe entry.obj /wholearchive a.lib b.lib", artifact: all_image } + link: { args: "/subsystem:console /entry:entry /out:only_a.exe entry.obj /wholearchive:a.lib a.lib b.lib", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:only_b.exe /wholearchive:b.lib a.lib b.lib entry.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: all_image, expected: { pe: { sections: { ".a": {}, ".b": {} } } } } + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".a": {}, @absent ".b" } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { @absent ".a", ".b": {} } } } } + } +} diff --git a/src/linker/tests/zero_length_comdat_referenced_by_reloc.tst b/src/linker/tests/zero_length_comdat_referenced_by_reloc.tst new file mode 100644 index 000000000..ba5c2a3b3 --- /dev/null +++ b/src/linker/tests/zero_length_comdat_referenced_by_reloc.tst @@ -0,0 +1,52 @@ +// A referenced zero-sized COMDAT symbol is meaningful enough for +// relocations, even though the COMDAT contributes no bytes to the image. +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + pad: { name: ".rdata$a", permissions: (read), content: initialized_data, alignment: 1, data: { text: "xy" } } + empty: { name: ".rdata$b", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { zero: 0 } } + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 8 } + relocations: { empty_ref: { type: Addr64, offset: 0, symbol: empty } } + } + } + symbols: + { + empty_def: { kind: section_definition, section: empty, selection: Any } + empty: { kind: external, name: "EMPTY", section: empty, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref entry.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 2 } + expect_pe_bytes: { artifact: image, section: ".rdata", offset: 0, hex: "7879" } + } +} diff --git a/src/linker/tests/zero_length_static_comdat_referenced_by_reloc.tst b/src/linker/tests/zero_length_static_comdat_referenced_by_reloc.tst new file mode 100644 index 000000000..7c329260a --- /dev/null +++ b/src/linker/tests/zero_length_static_comdat_referenced_by_reloc.tst @@ -0,0 +1,44 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + empty: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { zero: 0 } } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: + { + concat: + { + hex: "488d0500000000" // lea rax, [rip + EMPTY] + hex: "c3" + } + } + relocations: { empty_ref: { type: Rel32, offset: 3, symbol: empty } } + } + } + symbols: + { + empty_def: { kind: section_definition, section: empty, selection: NoDuplicates } + empty: { kind: static, name: "EMPTY", section: empty, value: 0 } + text_def: { kind: section_definition, section: text, selection: NoDuplicates } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,noicf test.obj" } } + steps: {} +} diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 3dc2c1467..f355a2066 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -1,6 +1,30 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) +// +// DUAL-PATH thread pool. +// +// NON-SHARED mode (no /RAD_SHARED_THREAD_POOL): UPSTREAM's barrier +// implementation, VERBATIM. Workers park on a kernel barrier between passes +// (zero-syscall steady state, no per-pass semaphore traffic). tp_run_tasks +// brackets the work loop in barrier_wait at entry+exit; tp_worker_main loops +// calling it; tp_for_parallel just inits state and joins as worker 0. No +// governor thread, no budget/wake/governor semaphores, no main_semaphore. +// +// SHARED mode (/RAD_SHARED_THREAD_POOL): OUR cross-process governor. Workers +// are PARKED on wake_semaphore and woken one-per-grant; a per-process +// governor thread borrows global budget slots and wakes parked workers; +// completion is signalled via main_semaphore; path-B barrier passes run a +// fair-share cohort (tp_barrier_begin/end + tp_for_parallel_reserve). +// +// Dispatch and the worker entry point branch on pool->is_shared (== name.size>0). +// All governor/budget state is allocated only when is_shared, so non-shared mode +// has zero extra threads and zero extra synchronization objects vs upstream. +// + +//////////////////////////////////////////////////////////////////////////////// +//~ NON-SHARED (upstream barrier) path -- VERBATIM from origin/dev. + internal void tp_run_tasks(TP_Context *pool, TP_Worker *worker) { @@ -36,60 +60,342 @@ tp_worker_main(void *raw_worker) } } +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED-mode governor stats (summary line). Zero cost when the pool is off: +// every call site below is on a shared-mode-only path. + +#define TP_STALL_ABORT_US 30000000ull + +global TP_SharedStats g_tp_shared_stats; + +internal void +tp_stats_level_add(S64 delta) +{ + // transitions are rare (one per grant/release, thousands per link), so a tiny + // spinlock around the integrator is cheaper than any clever lock-free scheme + for (; ins_atomic_u64_eval_cond_assign(&g_tp_shared_stats.lock, 1, 0) != 0; ) { } + U64 now_us = now_time_us(); + g_tp_shared_stats.area_us += (U64)(g_tp_shared_stats.level * (S64)(now_us - g_tp_shared_stats.last_us)); + g_tp_shared_stats.last_us = now_us; + g_tp_shared_stats.level += delta; + ins_atomic_u64_eval_assign(&g_tp_shared_stats.lock, 0); +} + +internal void +tp_stats_park_add(U64 worker_us) +{ + ins_atomic_u64_add_eval(&g_tp_shared_stats.park_us, worker_us); +} + +internal void +tp_stats_snapshot(F64 *grant_avg_out, F64 *park_seconds_out) +{ + *grant_avg_out = 0; + *park_seconds_out = 0; + if (g_tp_shared_stats.begin_us != 0) { + tp_stats_level_add(0); // finalize the integral up to now + U64 wall_us = g_tp_shared_stats.last_us - g_tp_shared_stats.begin_us; + if (wall_us > 0) { + *grant_avg_out = (F64)g_tp_shared_stats.area_us / (F64)wall_us; + } + *park_seconds_out = (F64)g_tp_shared_stats.park_us / 1000000.0; + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED-mode cross-process attach counter (summary line procs=). Counter +// SEMAPHORE, not a named section -- see the ".nproc.v3" comment in +// thread_pool.h for why (UBA virtualizes named sections per-process). + +global Semaphore g_tp_procs_sem; // zero handle = not attached +global U32 g_tp_procs_maxseen; // process-local max of observed n + +internal void +tp_procs_attach(Arena *scratch_arena, String8 name) +{ + String8 sem_name = push_str8f(scratch_arena, "%S.nproc." TP_NPROC_V, name); + Semaphore sem = semaphore_alloc(0, TP_NPROC_MAX, sem_name); // create-or-open, count starts at 0 + if (sem.u64[0] == 0) { + return; // best-effort: no semaphore, procs= prints 0/0 + } + U32 prev = 0; + if (!semaphore_drop_prev(sem, &prev)) { // attach: hold one permit + semaphore_release(sem); + return; + } + g_tp_procs_sem = sem; + g_tp_procs_maxseen = prev + 1; +} + +internal void +tp_procs_snapshot(U32 *attached_out, U32 *maxseen_out) +{ + *attached_out = 0; + *maxseen_out = 0; + if (g_tp_procs_sem.u64[0] != 0) { + U32 prev = 0; + if (semaphore_drop_prev(g_tp_procs_sem, &prev)) { // read: +1 ... + semaphore_take(g_tp_procs_sem, 0); // ... then undo (0-timeout, count > 0 by construction) + // prev = count BEFORE the transient release = #attached, which already + // includes THIS process's attach permit -- no +1 (unlike attach) + U32 n = prev; + g_tp_procs_maxseen = Max(g_tp_procs_maxseen, n); + *attached_out = n; + } + *maxseen_out = g_tp_procs_maxseen; + } +} + +internal void +tp_procs_detach(void) +{ + if (g_tp_procs_sem.u64[0] != 0) { + semaphore_take(g_tp_procs_sem, 0); // give the attach permit back (0-timeout: never block an exit path) + semaphore_release(g_tp_procs_sem); + MemoryZeroStruct(&g_tp_procs_sem); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED (cross-process governor) path -- OURS. + +internal void +tp_for_parallel_init_state(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) +{ + pool->task_arena = task_arena; + pool->task_func = task_func; + pool->task_data = task_data; + pool->task_count = task_count; + pool->task_done = 0; + ins_atomic_u64_eval_assign(&pool->task_left, task_count); +} + +// +// SHARED work loop. Semaphore-completion model (no barrier bracket): pure +// work-stealing on the atomic task_left decrement; the last finisher pings +// main_semaphore so the dispatching main thread can return. +// +internal void +tp_run_tasks_shared(TP_Context *pool, TP_Worker *worker) +{ + for (;;) { + S64 task_left = ins_atomic_u64_dec_eval(&pool->task_left); + + // are there any tasks left to run? + if (task_left < 0) { + break; + } + + // run task + Arena *arena = pool->task_arena ? pool->task_arena->v[worker->id] : 0; + U64 task_id = pool->task_count - (task_left+1); + pool->task_func(arena, worker->id, task_id, pool->task_data, pool); + + // cache task count so we dont touch pool memory after atomic inc + U64 task_count = pool->task_count; + + // on last task ping main thread (main_semaphore is null when worker_count==1, + // in which case main runs everything inline and never waits) + U64 task_done = ins_atomic_u64_inc_eval(&pool->task_done); + if (task_done == task_count && pool->worker_count > 1) { + semaphore_drop(pool->main_semaphore); + } + } +} + +// +// SHARED worker. Parked on wake_semaphore. Woken one-per-grant. Two wake kinds: +// - path A (barrier-free, governor-driven): the worker was woken because the +// governor acquired a global budget slot for it. When the worker drains +// (tp_run_tasks_shared returns), it RETURNS that slot: release(budget) + +// granted--, so the slot can flow to another process mid-pass. +// - path B (barrier pass): the dispatching thread reserved cohort slots up +// front and woke exactly the cohort's workers. The worker just runs the pass +// and re-parks; the dispatcher releases the slots in bulk afterwards. The +// worker must NOT touch the budget here (cohort must stay live for the pass). +// internal void tp_worker_main_shared(void *raw_worker) { TP_Worker *worker = raw_worker; TP_Context *pool = worker->pool; for (; pool->is_live; ) { - if (semaphore_take(pool->exec_semaphore, max_U64)) { - tp_run_tasks(pool, worker); + if (!semaphore_take(pool->wake_semaphore, max_U64)) { + continue; + } + if (!pool->is_live) { + break; + } + // capture pass kind at wake time (only one pass kind is active at once) + B32 barrier_pass = pool->barrier_pass; + + tp_run_tasks_shared(pool, worker); + + if (!barrier_pass) { + // path A: hand my budget slot back so another process can use it + tp_stats_level_add(-1); + ins_atomic_u64_dec_eval(&pool->granted); + semaphore_drop(pool->budget_semaphore); } } } -internal TP_Context * +// +// Per-process governor. Sleeps until main signals a path-A pass is active, then +// acquires global budget slots (only while THIS process has pending demand) and +// wakes one local parked worker per slot. Slots are returned by the workers +// themselves when they drain, so the governor only ever ACQUIRES. +// +internal void +tp_governor_main(void *raw_pool) +{ + TP_Context *pool = raw_pool; + for (; pool->is_live; ) { + // wait for a pass to begin (or for shutdown) + if (!semaphore_take(pool->governor_semaphore, max_U64)) { + continue; + } + if (!pool->is_live) { + break; + } + + // Grant slots while the pass is live and there is demand. Cap total live + // grants at worker_count-1 (main/worker 0 is the worker_count-th runner and + // never consumes a slot). `granted` is decremented by workers as they drain. + for (; ins_atomic_u32_eval(&pool->pass_active); ) { + S64 task_left = ins_atomic_u64_eval((U64 *)&pool->task_left); + S64 demand = task_left > 0 ? task_left : 0; + S64 cap = (S64)pool->worker_count - 1; + S64 live = ins_atomic_u64_eval((U64 *)&pool->granted); + S64 want = Min(cap, demand) - live; + + if (want > 0) { + // Bounded wait so we re-check pass_active/demand and never block forever + // on budget that may never free if the pass ends first. + U64 wait_begin_us = now_time_us(); + B32 got_slot = semaphore_take(pool->budget_semaphore, wait_begin_us + 1000); + // stats: while we waited here, `want` runnable workers sat parked on budget + tp_stats_park_add((now_time_us() - wait_begin_us) * (U64)want); + if (got_slot) { + // Publish the grant (granted++) BEFORE checking pass_active, and ABORT + // with granted-- if the pass already ended. This makes main's path-A + // drain-spin (waits granted==0) observe any in-flight grant and block + // until the governor resolves it -- so main cannot exit tp_for_parallel + // and start a path-B barrier pass (which sets barrier_pass=1) while a + // grant is pending. Hence a woken worker ALWAYS captures barrier_pass==0 + // for a path-A grant and does its paired granted--. + // + // The earlier "check pass_active, then granted++" ordering was NOT + // atomic: the governor could pass the check, get preempted while main + // ended the pass + drained granted to 0 + started a path-B pass, then + // wake a worker that captured barrier_pass==1, skipped granted--, and + // wedged granted>0 forever (observed: main spinning in tp_for_parallel, + // all workers parked). granted++ first closes that window. + ins_atomic_u64_inc_eval(&pool->granted); + if (ins_atomic_u32_eval(&pool->pass_active)) { + tp_stats_level_add(+1); + semaphore_drop(pool->wake_semaphore); + } else { + ins_atomic_u64_dec_eval(&pool->granted); // abort: pass ended + semaphore_drop(pool->budget_semaphore); // give the slot back + } + } + } else { + // This pass needs no more grants. task_left only decreases, and a worker + // returns its grant only after draining the shared queue: demand cannot + // become uncovered again within this pass. Park on the outer semaphore; + // the next path-A dispatch (or shutdown) supplies a persistent wake. + // Sleep(0) here just yielded repeatedly, burning a core while long-running + // tasks finished, including the entire explicit input-unmap pass. + break; + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ Alloc / release (dual). + +internal TP_Context * tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) { ProfBeginDynamic("Alloc Thread Pool [Worker Count: %u]", worker_count); AssertAlways(worker_count > 0); - B32 is_shared = (name.size > 0); + B32 is_shared = (name.size > 0); + Temp scratch = scratch_begin(&arena, 1); + + // init pool + TP_Context *pool = push_array(arena, TP_Context, 1); + pool->run_barrier = barrier_alloc(worker_count); + pool->barrier = barrier_alloc(worker_count); + pool->is_live = 1; + pool->is_shared = is_shared; + pool->worker_count = worker_count; + pool->worker_arr = push_array(arena, TP_Worker, worker_count); // alloc semaphores - Semaphore exec_semaphore = {0}; - if (worker_count > 1) { - if (is_shared) { + if (is_shared) { + // SHARED: governor + budget + wake + completion. Only allocated here, so + // non-shared mode pays for none of it. + if (worker_count > 1) { AssertAlways(worker_count <= max_worker_count); - exec_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + + pool->main_semaphore = semaphore_alloc(0, 1, str8_zero()); + + // ONE NAMED cross-process semaphore. CreateSemaphoreW on an existing name + // returns the existing object (first process inits with this count; later + // processes attach and the supplied count is ignored by the OS), so all + // processes share one BUDGET. + // BUDGET: init=max=max_worker_count (the machine core budget). + // FAIR-SHARE: there is no longer a barrier-lock. A barrier pass (path B) + // does NOT amass the full cohort; it runs at whatever budget is free right + // now (best-effort), so multiple processes can run barrier passes + // concurrently and none can deadlock waiting to amass the machine. + // ".v2" LAYOUT-VERSION suffix: see TP_SharedBlock in thread_pool.h -- old + // exes ("%S.budget", no procs section) and new exes must never share + // kernel objects for the same pool name + String8 budget_name = push_str8f(scratch.arena, "%S.budget." TP_SHARED_V, name); + pool->budget_semaphore = semaphore_alloc(max_worker_count, max_worker_count, budget_name); + pool->max_worker_count = max_worker_count; + + // local wake/governor signalling. governor_semaphore is a 0/1 "at least one + // pending pass" flag: main pings it with semaphore_drop_if_room (a redundant + // ping while one is already pending is a harmless no-op, since the pending + // signal will make the governor re-evaluate the current pass_active anyway). + pool->wake_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + pool->governor_semaphore = semaphore_alloc(0, 1, str8_zero()); } } // pick entry point for the workers void *worker_entry = is_shared ? tp_worker_main_shared : tp_worker_main; - // init pool - TP_Context *pool = push_array(arena, TP_Context, 1); - pool->exec_semaphore = exec_semaphore; - pool->run_barrier = barrier_alloc(worker_count); - pool->barrier = barrier_alloc(worker_count); - pool->is_live = 1; - pool->worker_count = worker_count; - pool->worker_arr = push_array(arena, TP_Worker, worker_count); - // init worker data for (U64 i = 0; i < worker_count; i += 1) { TP_Worker *worker = &pool->worker_arr[i]; worker->id = i; worker->pool = pool; } - + // launch worker threads for (U64 i = 1; i < worker_count; i += 1) { TP_Worker *worker = &pool->worker_arr[i]; worker->handle = thread_launch(worker_entry, worker); } - + + // launch the per-process governor (shared mode only) + if (is_shared && worker_count > 1) { + pool->governor_handle = thread_launch(tp_governor_main, pool); + } + + // stats: start the grant_avg integration window (shared mode only) + if (is_shared) { + g_tp_shared_stats.begin_us = g_tp_shared_stats.last_us = now_time_us(); + tp_procs_attach(scratch.arena, name); // summary line procs= (attach counter + peak watermark) + } + + scratch_end(scratch); ProfEnd(); return pool; } @@ -99,24 +405,45 @@ tp_release(TP_Context *pool) { pool->is_live = 0; - B32 is_shared = pool->exec_semaphore.u64[0] != 0; - if (is_shared) { - for EachIndex(i, pool->worker_count) { - semaphore_drop(pool->exec_semaphore); + if (pool->is_shared) { + if (pool->worker_count > 1) { + // wake governor so it observes !is_live and exits (a pending ping is fine) + semaphore_drop_if_room(pool->governor_semaphore); + // wake every parked worker so each observes !is_live and exits. Wakes here + // are NOT path-A grants (no budget was taken), so mark barrier_pass to keep + // workers from touching the budget on their way out. + pool->barrier_pass = 1; + for (U64 i = 1; i < pool->worker_count; i += 1) { + semaphore_drop(pool->wake_semaphore); + } + } + for (U64 i = 1; i < pool->worker_count; i += 1) { + thread_detach(pool->worker_arr[i].handle); + } + if (pool->worker_count > 1) { + thread_detach(pool->governor_handle); + semaphore_release(pool->budget_semaphore); + semaphore_release(pool->wake_semaphore); + semaphore_release(pool->governor_semaphore); + semaphore_release(pool->main_semaphore); + } + } else { + // NON-SHARED: upstream verbatim. Workers are parked on the barrier; flipping + // is_live and waking the barrier lets each observe !is_live and exit. + for (U64 i = 1; i < pool->worker_count; i += 1) { + thread_detach(pool->worker_arr[i].handle); } } - for (U64 i = 1; i < pool->worker_count; i += 1) { - thread_detach(pool->worker_arr[i].handle); - } - if (is_shared) { - semaphore_release(pool->exec_semaphore); - } + barrier_release(pool->run_barrier); barrier_release(pool->barrier); MemoryZeroStruct(pool); } +//////////////////////////////////////////////////////////////////////////////// +//~ Arenas / temps -- shared by both modes (unchanged). + internal TP_Arena * tp_arena_alloc(TP_Context *pool) { @@ -124,7 +451,12 @@ tp_arena_alloc(TP_Context *pool) Temp scratch = scratch_begin(0,0); Arena **arr = push_array(scratch.arena, Arena *, pool->worker_count); for (U64 i = 0; i < pool->worker_count; ++i) { - arr[i] = arena_alloc("THREAD_POOL"); + // 2MB commit quantum: these per-worker arenas take the bulk of the link's + // ~50GB of MEM_COMMIT growth; the default 64KB quantum turns that into + // ~800K NtAllocateVirtualMemory calls from 64 threads serialized on the + // process address-space lock. Slack is bounded by workers x live arenas x + // quantum (single-digit MBs per worker), far below the syscall cost. + arr[i] = arena_alloc(.commit_size = MB(2), .name = "THREAD_POOL"); } Arena **dst = push_array(arr[0], Arena *, pool->worker_count); MemoryCopy(dst, arr, sizeof(Arena*) * pool->worker_count); @@ -179,11 +511,23 @@ tp_temp_end(TP_Temp temp) ProfEnd(); } +//////////////////////////////////////////////////////////////////////////////// +//~ Dispatch (dual). + internal void tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) { - if (task_count) { - // init run + if (task_count == 0) { + return; + } + + if (!pool->is_shared) { + // + // NON-SHARED: UPSTREAM verbatim. Init state, then join the barrier as worker + // 0; the already-parked workers (looping in tp_run_tasks) rendezvous at the + // entry barrier, steal tasks, and rendezvous again at the exit barrier. No + // semaphores, no governor. + // pool->task_arena = task_arena; pool->task_func = task_func; pool->task_data = task_data; @@ -191,26 +535,265 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF pool->task_done = 0; pool->task_left = task_count; - // if we are in shared mode -> ping - if (*pool->exec_semaphore.u64) { - U64 drop_count64 = pool->worker_count - 1; - U32 drop_count = safe_cast_u32(drop_count64); - semaphore_drop_count(pool->exec_semaphore, drop_count); + // run tasks on main worker + tp_run_tasks(pool, &pool->worker_arr[0]); + return; + } + + // + // SHARED: OUR governor dispatch. + // + tp_for_parallel_init_state(pool, task_arena, task_count, task_func, task_data); + + if (pool->worker_count == 1) { + // no workers: main runs everything inline + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + return; + } + + // PATH A: barrier-free dispatch (the common case). Pure work-stealing via the + // atomic task_left decrement in tp_run_tasks_shared. Main (worker 0) ALWAYS + // runs and never consumes a global budget slot -> per-process forward-progress + // guarantee. The governor opportunistically borrows budget slots and wakes + // local parked workers; each woken worker returns its slot when it drains. + + // announce a path-A pass and let the governor recruit workers as budget frees + pool->barrier_pass = 0; + ins_atomic_u32_eval_assign(&pool->pass_active, 1); + semaphore_drop_if_room(pool->governor_semaphore); + + // main always runs (no slot consumed) + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + + // all tasks done (last finisher pinged main_semaphore) + semaphore_take(pool->main_semaphore, max_U64); + + // End the pass so the governor stops issuing new grants. + ins_atomic_u32_eval_assign(&pool->pass_active, 0); + + // CRITICAL: before returning we must guarantee that no woken worker is still + // inside tp_run_tasks_shared. Otherwise the next pass's init_state (which resets + // task_left/task_done) would race a straggler still looping on this pass and + // corrupt the counters / lose the completion ping -> deadlock. + // + // Every governor grant is paired with exactly one wake permit and one worker + // that, on draining, does `granted--; drop(budget)`. Even grants issued in the + // tiny window before pass_active was cleared have a pending wake permit that a + // worker will consume, drain immediately (task_left<0), and account for. So + // `granted` monotonically drains to 0 once the governor has stopped; spin + // until it does. This is brief (workers see task_left<0 and exit at once). + U64 stall_begin_us = now_time_us(); + for (; ins_atomic_u64_eval((U64 *)&pool->granted) != 0; ) { + sleep_ms(0); + U64 elapsed_us = now_time_us() - stall_begin_us; + if (elapsed_us >= TP_STALL_ABORT_US) { + // Never perform stderr I/O here. Under UBA that enters the WriteFile detour and can block, + // turning the diagnostic path itself into a permanent hang before the fail-fast executes. + AssertAlways(0); } + } +} - // run tasks on main worker - tp_run_tasks(pool, pool->worker_arr); - Assert(pool->task_done == task_count); +//////////////////////////////////////////////////////////////////////////////// +//~ FAIR-SHARE barrier-pass cohort bracket (SHARED path B; no-op in non-shared). +// +// A barrier pass (path B) runs at the cohort this process currently holds, NOT +// the full machine. tp_barrier_begin grabs whatever budget slots are FREE RIGHT +// NOW (best-effort, never blocking to amass), up to worker_count-1, and pins the +// pool to cohort C = 1 (main) + grabbed slots for the pass duration: +// - pool->worker_count := C (so every tp->worker_count read -- divide_work, +// lane_count, per-worker array sizing in the +// caller's setup -- sees the cohort) +// - pool->barrier := a fresh C-sized barrier (so barrier_wait/broadcast/sum +// rendezvous exactly the C participants) +// - the grabbed slots are HELD until tp_barrier_end (cohort stays live; the +// governor only touches budget during a path-A pass, which cannot overlap a +// barrier pass within this process). +// +// Deadlock-freedom: tp_barrier_begin NEVER blocks on budget. If the machine is +// busy and zero slots are free, C == 1 and the pass runs serially on main. A +// process therefore ALWAYS makes progress (>= main) and never waits to amass -> +// no starvation, no deadlock, no barrier-lock. Output is width-independent +// (proven: w1 == w64), so a cohort-C pass is byte-identical to a full-width pass. +// +// In NON-SHARED mode tp_barrier_begin/end are no-ops (return pool->worker_count / +// early-out) and tp_for_parallel_reserve degrades to the plain upstream +// full-width barrier pass via tp_for_parallel. +// +internal U32 +tp_barrier_begin(TP_Context *pool) +{ + if (!pool->is_shared || pool->worker_count == 1) { + return pool->worker_count; // no-op: non-shared / single-worker + } + if (pool->barrier_depth > 0) { + pool->barrier_depth += 1; // nested: cohort already pinned + return pool->worker_count; + } + + // best-effort grab: take as many free budget slots as we can WITHOUT blocking + // (endt_us==0 -> WaitForSingleObject(.,0) non-blocking poll). Stop at the first + // empty take or when we hold worker_count-1. + U32 want = pool->worker_count - 1; + U32 extra = 0; + for (; extra < want; ) { + if (semaphore_take(pool->budget_semaphore, 0)) { + extra += 1; + } else { + break; // no more free slots right now + } } + + // FAIR-SHARE FLOOR: the cohort is pinned for the whole bracket, so a bracket + // opened at a bad instant (siblings momentarily holding the machine) would run + // a long phase at width 1-2 even after the machine empties. If the free-slot + // sweep landed below this process's fair share (machine budget / attached + // processes), keep taking with bounded waits until we reach it or the deadline + // expires. Slots flow back continuously as sibling path-A workers drain, so + // this normally fills within a few ms; if every sibling is pinned in its own + // long bracket the deadline bounds the wait and we proceed with what we hold -- + // never a deadlock, cohort >= 1 always. + if (extra < want) { + U32 procs = 0, procs_maxseen = 0; + tp_procs_snapshot(&procs, &procs_maxseen); + if (procs > 1) { + U32 fair = pool->max_worker_count / procs; + fair = Clamp(1, fair, want + 1); + if (1 + extra < fair) { + U64 deadline_us = now_time_us() + TP_BARRIER_FLOOR_WAIT_US; + for (; 1 + extra < fair; ) { + U64 now_us = now_time_us(); + if (now_us >= deadline_us) { break; } + U64 slice_us = Min(deadline_us - now_us, 5000); + if (semaphore_take(pool->budget_semaphore, now_us + slice_us)) { + extra += 1; + } + } + } + } + } + + U32 cohort = 1 + extra; // main + grabbed workers + + pool->barrier_saved_workers = pool->worker_count; + pool->barrier_cohort_extra = extra; + pool->barrier_saved = pool->barrier; + pool->barrier = barrier_alloc(cohort); + pool->worker_count = cohort; + pool->barrier_pass = 1; + pool->barrier_depth = 1; + + // stats: cohort slots are held for the whole bracket; the shortfall is the + // budget we wanted but could not grab (parked lanes while this pass runs) + pool->barrier_begin_us = now_time_us(); + pool->barrier_shortfall = want - extra; + if (extra > 0) { tp_stats_level_add((S64)extra); } + + return cohort; +} + +internal void +tp_barrier_end(TP_Context *pool) +{ + if (!pool->is_shared || pool->barrier_saved_workers == 0) { + return; // no-op: non-shared / single-worker / not in a bracket + } + pool->barrier_depth -= 1; + if (pool->barrier_depth > 0) { + return; // nested: outer bracket still owns the cohort + } + + U32 extra = pool->barrier_cohort_extra; + + // stats: release the held slots from the integral; account parked lanes + if (extra > 0) { tp_stats_level_add(-(S64)extra); } + if (pool->barrier_shortfall > 0) { + tp_stats_park_add((now_time_us() - pool->barrier_begin_us) * (U64)pool->barrier_shortfall); + } + pool->barrier_begin_us = 0; + pool->barrier_shortfall = 0; + + // restore the full-width pool + the original barrier + barrier_release(pool->barrier); + pool->barrier = pool->barrier_saved; + pool->worker_count = pool->barrier_saved_workers; + pool->barrier_pass = 0; + + pool->barrier_saved_workers = 0; + pool->barrier_cohort_extra = 0; + MemoryZeroStruct(&pool->barrier_saved); + + // hand the grabbed budget slots back to the machine + semaphore_drop_n(pool->budget_semaphore, extra); } +// +// PATH B: barrier-pass dispatch (fair-share). Runs the task once per lane on the +// CURRENT cohort (main + woken workers). If the caller has not already opened a +// tp_barrier_begin/end bracket, this opens one (cohort = whatever is free now), +// runs, and closes it. The passed task_count is ignored for sizing -- the pass +// always runs exactly pool->worker_count tasks (== cohort). +// +// In NON-SHARED mode this degrades to the plain upstream full-width barrier pass +// (tp_for_parallel), so non-shared barrier passes behave exactly as upstream. +// +internal void +tp_for_parallel_reserve(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) +{ + if (task_count == 0) { + return; + } + + if (!pool->is_shared || pool->worker_count == 1) { + // non-shared (or single worker): identical to the plain dispatch (upstream + // full-width barrier pass in non-shared mode) + tp_for_parallel(pool, task_arena, task_count, task_func, task_data); + return; + } + + // open a cohort bracket unless the caller already pinned one + B32 opened = 0; + if (pool->barrier_depth == 0) { + tp_barrier_begin(pool); + opened = 1; + } + + U32 cohort = pool->worker_count; // pinned for the whole pass + + if (cohort == 1) { + // machine fully busy: run serially on main (byte-identical -- width independent) + tp_for_parallel_init_state(pool, task_arena, cohort, task_func, task_data); + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + } else { + tp_for_parallel_init_state(pool, task_arena, cohort, task_func, task_data); + + // wake exactly the cohort's workers (ids 1..cohort-1). These are barrier-pass + // wakes: workers must NOT return budget on drain -- the slots are held by the + // bracket and released in tp_barrier_end so the cohort stays live for the pass. + semaphore_drop_n(pool->wake_semaphore, cohort - 1); + + // main is the cohort-th participant (lane 0) + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + + // wait for the cohort to finish + semaphore_take(pool->main_semaphore, max_U64); + } + + if (opened) { + tp_barrier_end(pool); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ Helpers -- shared by both modes (unchanged). + internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count) { U64 per_count = CeilIntegerDiv(item_count, worker_count); Rng1U64 *range_arr = push_array_no_zero(arena, Rng1U64, worker_count + 1); for (U64 i = 0; i < worker_count; i += 1) { - range_arr[i] = rng_1u64(Min(item_count, i * per_count), + range_arr[i] = rng_1u64(Min(item_count, i * per_count), Min(item_count, i * per_count + per_count)); } @@ -250,4 +833,3 @@ tp_sum_u64(TP_Context *tp, U64 task_id, U64 v) barrier_wait(tp->barrier); return result; } - diff --git a/src/linker/thread_pool/thread_pool.h b/src/linker/thread_pool/thread_pool.h index f89f17852..d7eb23fe1 100644 --- a/src/linker/thread_pool/thread_pool.h +++ b/src/linker/thread_pool/thread_pool.h @@ -38,6 +38,28 @@ typedef struct TP_Context U64 broadcast_size; U64 sum; + // shared (cross-process) governor mode; all zero in non-shared mode + B32 is_shared; + Semaphore budget_semaphore; // NAMED: global core budget; init=max=max_worker_count + Semaphore wake_semaphore; // local: governor/dispatcher wakes one parked worker per drop + Semaphore governor_semaphore; // local: main pings governor that a path-A pass is active + Thread governor_handle; + volatile U32 pass_active; // 1 while a path-A (barrier-free) pass is in flight + volatile U32 barrier_pass; // 1 while the current wake cohort is a path-B barrier pass + volatile S64 granted; // budget slots currently held by woken path-A workers + U32 max_worker_count; // machine core budget (budget_semaphore init/max) + + // FAIR-SHARE barrier-pass cohort state (path B). A barrier pass runs at the + // cohort the governor currently allows this process to hold: main + however + // many budget slots are free RIGHT NOW (best-effort, never amassed). Pinned + // for the pass duration. See tp_barrier_begin/tp_barrier_end. + U32 barrier_depth; // >0 while inside a tp_barrier_begin/end bracket (re-entrant guard) + U32 barrier_saved_workers; // worker_count to restore at tp_barrier_end + U32 barrier_cohort_extra; // budget slots held for this barrier pass (cohort = 1 + this) + Barrier barrier_saved; // pool->barrier to restore at tp_barrier_end + U64 barrier_begin_us; // stats: bracket open time (for park accounting) + U32 barrier_shortfall; // stats: budget slots we wanted but could not grab for this pass + U32 worker_count; TP_Worker *worker_arr; @@ -57,5 +79,78 @@ internal TP_Temp tp_temp_begin(TP_Arena *arena); internal void tp_temp_end(TP_Temp temp); #define tp_for_parallel_prof(pool, arena, task_count, task_func, task_data, zone_name) ProfBegin(zone_name); tp_for_parallel(pool, arena, task_count, task_func, task_data); ProfEnd(); internal void tp_for_parallel(TP_Context *pool, TP_Arena *arena, U64 task_count, TP_TaskFunc *task_func, void *task_data); +// FAIR-SHARE barrier-pass cohort bracket. Between tp_barrier_begin and +// tp_barrier_end, pool->worker_count is PINNED to the cohort this process +// currently holds (1 + budget slots free right now, capped at the full count), +// and pool->barrier is sized to that cohort. Returns the cohort count C. A caller +// that pre-distributes work by tp->worker_count (sizes per-worker arrays, builds +// divide_work ranges, sets a task-data .worker_count) MUST do that setup inside +// the bracket so it sees C, then call tp_for_parallel_reserve with task_count==C. +// Re-entrant: nested begins just return the pinned C. In non-shared mode (or +// worker_count==1) it is a no-op that returns worker_count. +internal U32 tp_barrier_begin(TP_Context *pool); +internal void tp_barrier_end(TP_Context *pool); +// Barrier-pass dispatch: task_func uses barrier_wait/tp_broadcast/tp_sum_u64. The +// cohort is whatever this process currently holds (fair-share): if not already +// inside a tp_barrier_begin/end bracket this opens one itself, runs the pass at +// the pinned cohort, and closes it. Output is width-independent so any cohort +// (down to 1 = main only) produces byte-identical results. The passed task_count +// is IGNORED for sizing; the pass always runs exactly pool->worker_count (==cohort) +// tasks, one per lane. In non-shared mode it is identical to tp_for_parallel. +internal void tp_for_parallel_reserve(TP_Context *pool, TP_Arena *arena, U64 task_count, TP_TaskFunc *task_func, void *task_data); +#define tp_for_parallel_reserve_prof(pool, arena, task_count, task_func, task_data, zone_name) ProfBegin(zone_name); tp_for_parallel_reserve(pool, arena, task_count, task_func, task_data); ProfEnd(); internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count); #define tp_broadcast(p) tp_broadcast_(tp, task_id, p, sizeof(*p)) + +// SHARED-mode governor stats (for the end-of-link summary line). All counters +// are only ever touched from shared-mode-only code paths, so the non-shared +// pool pays zero cost. `level` is the count of global budget slots this +// process currently HOLDS (path-A grants + path-B cohort extras); it is +// integrated over time (area_us = sum level x dt, QPC-stamped on every +// grant/release transition) so grant_avg = area/wall. `park_us` accumulates +// worker-microseconds spent waiting on budget while this process had pending +// demand (path A: governor budget waits x wanted-worker count; path B: pass +// duration x cohort shortfall). +typedef struct TP_SharedStats +{ + volatile U64 lock; // spinlock for the {last_us, level, area_us} integrator + U64 begin_us; // pool alloc time (grant_avg denominator start) + U64 last_us; // last transition stamp + S64 level; // budget slots currently held + U64 area_us; // integral of level over time + volatile U64 park_us; // worker-us parked on budget while work was available +} TP_SharedStats; + +internal void tp_stats_level_add(S64 delta); +internal void tp_stats_park_add(U64 worker_us); +internal void tp_stats_snapshot(F64 *grant_avg_out, F64 *park_seconds_out); + +// SHARED-mode cross-process attach counter (summary line: procs=/). +// Lives in a named counter SEMAPHORE ".nproc.v3"; the budget +// semaphore is ".budget.v2". Names carry a version suffix so an old +// radlink pointed at the SAME /RAD_SHARED_THREAD_POOL name never shares kernel +// objects with a new one: mixed old/new farms run independent pools instead of +// corrupting one. +// +// WHY A SEMAPHORE: UBA detours CreateFileMapping and VIRTUALIZES named +// sections per-process, so the previous ".procs.v2" shared-memory +// block reported procs=1/1 under UBA. Named SEMAPHORES pass through the detour +// (the budget semaphore is demonstrably shared in prod: fair grants across +// processes), so the counter is the semaphore's own count: +// attach = release(+1, &prev) -> n = prev+1 (each attached process holds one permit) +// detach = 0-timeout wait (-1) +// read = release(+1, &prev) -> n = prev (prev already counts our own +// permit), then 0-timeout wait to undo +// The transient read +1 can inflate a concurrent reader's n by 1 -- advisory +// counter, acceptable. Peak cannot be tracked exactly cross-process without +// shm, so `maxseen` is this process's local max of n observed at attach and at +// the summary read. Best-effort: a process that dies without detaching leaves +// the count high until the semaphore object itself dies with its last handle. +#define TP_SHARED_V "v2" +#define TP_NPROC_V "v3" +#define TP_NPROC_MAX (1u << 20) // far above any plausible concurrent-link count +#define TP_BARRIER_FLOOR_WAIT_US 200000 // tp_barrier_begin: max wait to reach the fair-share cohort floor + +internal void tp_procs_snapshot(U32 *attached_out, U32 *maxseen_out); // 0/0 when no shared pool +internal void tp_procs_detach(void); // give the permit back + close (idempotent) + diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index 3daa94f79..842ef5491 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -711,6 +711,54 @@ semaphore_drop_count(Semaphore semaphore, U64 count) } } +internal void +semaphore_drop_if_room(Semaphore semaphore) +{ + // POSIX sem_t is unbounded, so a post can never be rejected for being "full". + semaphore_drop(semaphore); +} + +internal B32 +semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out) +{ + // best-effort: sem_getvalue+sem_post is not atomic (unlike win32 + // ReleaseSemaphore's lpPreviousCount); callers use this for advisory + // counters only + *prev_count_out = 0; + if(semaphore.u64[0] == 0) { return 0; } + int value = 0; + if(sem_getvalue((sem_t*)*semaphore.u64, &value) == 0 && value > 0) { *prev_count_out = (U32)value; } + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)*semaphore.u64)); + return err == 0; +} + +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if(semaphore.u64[0] != 0) + { + for(U32 i = 0; i < count; i += 1) + { + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)semaphore.u64[0])); + Assert(err == 0); + } + } +} + +internal B32 +semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us) +{ + for(U32 i = 0; i < count; i += 1) + { + if(!semaphore_take(semaphore, endt_us)) + { + semaphore_drop_n(semaphore, i); + return 0; + } + } + return 1; +} + //- rjf: barriers internal Barrier diff --git a/src/mdesk/mdesk.h b/src/mdesk/mdesk.h index d6c6e5fec..4c0cc2617 100644 --- a/src/mdesk/mdesk.h +++ b/src/mdesk/mdesk.h @@ -276,6 +276,7 @@ internal MD_NodeFlags md_node_flags_from_token_flags(MD_TokenFlags flags); //- rjf: nil internal B32 md_node_is_nil(MD_Node *node); +internal B32 t_md_node_is_valid(MD_Node *node); //- rjf: iteration #define MD_EachNode(it, first) (MD_Node *it = first; !md_node_is_nil(it); it = it->next) diff --git a/src/pe/pe_make_import_table.c b/src/pe/pe_make_import_table.c index eb20b0e45..0baf5390b 100644 --- a/src/pe/pe_make_import_table.c +++ b/src/pe/pe_make_import_table.c @@ -286,11 +286,14 @@ pe_make_import_dll_obj_static(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mach str8_list_push(obj_writer->arena, &ilt_sect->data, ordinal_data); str8_list_push(obj_writer->arena, &iat_sect->data, ordinal_data); } break; + case COFF_ImportBy_NameNoPrefix: + case COFF_ImportBy_Undecorate: case COFF_ImportBy_Name: { COFF_ObjSection *int_sect = coff_obj_writer_push_section(obj_writer, str8_lit(".idata$6"), PE_IDATA_SECTION_FLAGS|COFF_SectionFlag_Align2Bytes|comdat_flags, str8_zero()); coff_obj_writer_push_symbol_associative(obj_writer, int_sect, iat_sect); COFF_ObjSymbol *int_symbol = coff_obj_writer_push_symbol_static(obj_writer, int_sect->name, 0, int_sect); - String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, import_header.func_name); + String8 lookup_name = coff_import_lookup_name_from_import_by(import_header.func_name, import_header.import_by); + String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, lookup_name); str8_list_push(obj_writer->arena, &int_sect->data, int_data); str8_list_push_aligner(obj_writer->arena, &int_sect->data, 0, 2); @@ -302,8 +305,6 @@ pe_make_import_dll_obj_static(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mach str8_list_push(obj_writer->arena, &ilt_sect->data, str8_array(import_entry, import_size)); str8_list_push(obj_writer->arena, &iat_sect->data, str8_array(import_entry, import_size)); } break; - case COFF_ImportBy_Undecorate: { NotImplemented; } break; - case COFF_ImportBy_NameNoPrefix: { NotImplemented; } break; default: { InvalidPath; } break; } @@ -461,9 +462,12 @@ pe_make_import_dll_obj_delayed(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mac coff_obj_writer_section_push_reloc_addr(obj_writer, uiat_sect, uiat_offset, load_thunk_symbol); } } break; + case COFF_ImportBy_NameNoPrefix: + case COFF_ImportBy_Undecorate: case COFF_ImportBy_Name: { // put together name look up entry - String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, import_header.func_name); + String8 lookup_name = coff_import_lookup_name_from_import_by(import_header.func_name, import_header.import_by); + String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, lookup_name); U64 int_data_offset = int_sect->data.total_size; str8_list_push(obj_writer->arena, &int_sect->data, int_data); @@ -503,8 +507,6 @@ pe_make_import_dll_obj_delayed(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mac coff_obj_writer_section_push_reloc_addr(obj_writer, uiat_sect, uiat_data_offset, load_thunk_symbol); } } break; - case COFF_ImportBy_Undecorate: { NotImplemented; } break; - case COFF_ImportBy_NameNoPrefix: { NotImplemented; } break; } } diff --git a/src/third_party/blake3/LICENSE_CC0 b/src/third_party/blake3/LICENSE_CC0 new file mode 100644 index 000000000..0e259d42c --- /dev/null +++ b/src/third_party/blake3/LICENSE_CC0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/src/third_party/blake3/asm/LICENSE b/src/third_party/blake3/asm/LICENSE deleted file mode 100644 index f5892efc3..000000000 --- a/src/third_party/blake3/asm/LICENSE +++ /dev/null @@ -1,330 +0,0 @@ -This work is released into the public domain with CC0 1.0. Alternatively, it is -licensed under the Apache License 2.0. - -------------------------------------------------------------------------------- - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2019 Jack O'Connor and Samuel Neves - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/third_party/blake3/asm/blake3.h b/src/third_party/blake3/asm/blake3.h deleted file mode 100644 index f694dcf27..000000000 --- a/src/third_party/blake3/asm/blake3.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef BLAKE3_H -#define BLAKE3_H - -#include -#include - -#if !defined(BLAKE3_API) -# if defined(_WIN32) || defined(__CYGWIN__) -# if defined(BLAKE3_DLL) -# if defined(BLAKE3_DLL_EXPORTS) -# define BLAKE3_API __declspec(dllexport) -# else -# define BLAKE3_API __declspec(dllimport) -# endif -# define BLAKE3_PRIVATE -# else -# define BLAKE3_API -# define BLAKE3_PRIVATE -# endif -# elif __GNUC__ >= 4 -# define BLAKE3_API __attribute__((visibility("default"))) -# define BLAKE3_PRIVATE __attribute__((visibility("hidden"))) -# else -# define BLAKE3_API -# define BLAKE3_PRIVATE -# endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define BLAKE3_VERSION_STRING "1.5.0" -#define BLAKE3_KEY_LEN 32 -#define BLAKE3_OUT_LEN 32 -#define BLAKE3_BLOCK_LEN 64 -#define BLAKE3_CHUNK_LEN 1024 -#define BLAKE3_MAX_DEPTH 54 - -// This struct is a private implementation detail. It has to be here because -// it's part of blake3_hasher below. -typedef struct { - uint32_t cv[8]; - uint64_t chunk_counter; - uint8_t buf[BLAKE3_BLOCK_LEN]; - uint8_t buf_len; - uint8_t blocks_compressed; - uint8_t flags; -} blake3_chunk_state; - -typedef struct { - uint32_t key[8]; - blake3_chunk_state chunk; - uint8_t cv_stack_len; - // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, - // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk - // requires a 4th entry, rather than merging everything down to 1, because we - // don't know whether more input is coming. This is different from how the - // reference implementation does things. - uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; -} blake3_hasher; - -BLAKE3_API const char *blake3_version(void); -BLAKE3_API void blake3_hasher_init(blake3_hasher *self); -BLAKE3_API void blake3_hasher_init_keyed(blake3_hasher *self, - const uint8_t key[BLAKE3_KEY_LEN]); -BLAKE3_API void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); -BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, - size_t context_len); -BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len); -BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, - size_t out_len); -BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, - uint8_t *out, size_t out_len); -BLAKE3_API void blake3_hasher_reset(blake3_hasher *self); - -#ifdef __cplusplus -} -#endif - -#endif /* BLAKE3_H */ diff --git a/src/third_party/blake3/asm/blake3_dispatch.c b/src/third_party/blake3/asm/blake3_dispatch.c deleted file mode 100644 index 6f814569b..000000000 --- a/src/third_party/blake3/asm/blake3_dispatch.c +++ /dev/null @@ -1,278 +0,0 @@ -#include -#include -#include - -#include "blake3_impl.h" - -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#elif defined(__GNUC__) -#include -#else -#undef IS_X86 /* Unimplemented! */ -#endif -#endif - -#define MAYBE_UNUSED(x) (void)((x)) - -#if defined(IS_X86) -static uint64_t xgetbv(void) { -#if defined(_MSC_VER) - return _xgetbv(0); -#else - uint32_t eax = 0, edx = 0; - __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); - return ((uint64_t)edx << 32) | eax; -#endif -} - -static void cpuid(uint32_t out[4], uint32_t id) { -#if defined(_MSC_VER) - __cpuid((int *)out, id); -#elif defined(__i386__) || defined(_M_IX86) - __asm__ __volatile__("movl %%ebx, %1\n" - "cpuid\n" - "xchgl %1, %%ebx\n" - : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id)); -#else - __asm__ __volatile__("cpuid\n" - : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id)); -#endif -} - -static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { -#if defined(_MSC_VER) - __cpuidex((int *)out, id, sid); -#elif defined(__i386__) || defined(_M_IX86) - __asm__ __volatile__("movl %%ebx, %1\n" - "cpuid\n" - "xchgl %1, %%ebx\n" - : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id), "c"(sid)); -#else - __asm__ __volatile__("cpuid\n" - : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id), "c"(sid)); -#endif -} - -#endif - -enum cpu_feature { - SSE2 = 1 << 0, - SSSE3 = 1 << 1, - SSE41 = 1 << 2, - AVX = 1 << 3, - AVX2 = 1 << 4, - AVX512F = 1 << 5, - AVX512VL = 1 << 6, - /* ... */ - UNDEFINED = 1 << 30 -}; - -#if !defined(BLAKE3_TESTING) -static /* Allow the variable to be controlled manually for testing */ -#endif - volatile int g_cpu_features = UNDEFINED; - -#if !defined(BLAKE3_TESTING) -static -#endif - enum cpu_feature - get_cpu_features(void) { - - /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ - long features = g_cpu_features; - if (features != UNDEFINED) { - return (enum cpu_feature)features; - } else { -#if defined(IS_X86) - uint32_t regs[4] = {0}; - uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; - (void)edx; - features = 0; - cpuid(regs, 0); - const int max_id = *eax; - cpuid(regs, 1); -#if defined(__amd64__) || defined(_M_X64) - features |= SSE2; -#else - if (*edx & (1UL << 26)) - features |= SSE2; -#endif - if (*ecx & (1UL << 9)) - features |= SSSE3; - if (*ecx & (1UL << 19)) - features |= SSE41; - - if (*ecx & (1UL << 27)) { // OSXSAVE - const uint64_t mask = xgetbv(); - if ((mask & 6) == 6) { // SSE and AVX states - if (*ecx & (1UL << 28)) - features |= AVX; - if (max_id >= 7) { - cpuidex(regs, 7, 0); - if (*ebx & (1UL << 5)) - features |= AVX2; - if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm - if (*ebx & (1UL << 31)) - features |= AVX512VL; - if (*ebx & (1UL << 16)) - features |= AVX512F; - } - } - } - } - g_cpu_features = features; - return (enum cpu_feature)features; -#else - /* How to detect NEON? */ - return 0; -#endif - } -} - -void blake3_compress_in_place(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if (features & AVX512VL) { - blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_compress_in_place_sse2(cv, block, block_len, counter, flags); - return; - } -#endif -#endif - blake3_compress_in_place_portable(cv, block, block_len, counter, flags); -} - -void blake3_compress_xof(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags, - uint8_t out[64]) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if (features & AVX512VL) { - blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out); - return; - } -#endif -#endif - blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); -} - -void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { - blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_AVX2) - if (features & AVX2) { - blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#endif - -#if BLAKE3_USE_NEON == 1 - blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, out); - return; -#endif - - blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); -} - -// The dynamically detected SIMD degree of the current platform. -size_t blake3_simd_degree(void) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { - return 16; - } -#endif -#if !defined(BLAKE3_NO_AVX2) - if (features & AVX2) { - return 8; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - return 4; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - return 4; - } -#endif -#endif -#if BLAKE3_USE_NEON == 1 - return 4; -#endif - return 1; -} diff --git a/src/third_party/blake3/asm/blake3.c b/src/third_party/blake3/blake3.c similarity index 87% rename from src/third_party/blake3/asm/blake3.c rename to src/third_party/blake3/blake3.c index 692f4b021..00f91f444 100644 --- a/src/third_party/blake3/asm/blake3.c +++ b/src/third_party/blake3/blake3.c @@ -1,6 +1,7 @@ #include #include #include +#include #include "blake3.h" #include "blake3_impl.h" @@ -88,24 +89,30 @@ INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, size_t out_len) { + if (out_len == 0) { + return; + } uint64_t output_block_counter = seek / 64; size_t offset_within_block = seek % 64; uint8_t wide_buf[64]; - while (out_len > 0) { - blake3_compress_xof(self->input_cv, self->block, self->block_len, - output_block_counter, self->flags | ROOT, wide_buf); - size_t available_bytes = 64 - offset_within_block; - size_t memcpy_len; - if (out_len > available_bytes) { - memcpy_len = available_bytes; - } else { - memcpy_len = out_len; - } - memcpy(out, wide_buf + offset_within_block, memcpy_len); - out += memcpy_len; - out_len -= memcpy_len; + if(offset_within_block) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + const size_t available_bytes = 64 - offset_within_block; + const size_t bytes = out_len > available_bytes ? available_bytes : out_len; + memcpy(out, wide_buf + offset_within_block, bytes); + out += bytes; + out_len -= bytes; output_block_counter += 1; - offset_within_block = 0; + } + if(out_len / 64) { + blake3_xof_many(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, out, out_len / 64); + } + output_block_counter += out_len / 64; + out += out_len & -64; + out_len -= out_len & -64; + if(out_len) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + memcpy(out, wide_buf, out_len); } } @@ -134,9 +141,7 @@ INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, input_len -= BLAKE3_BLOCK_LEN; } - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; + chunk_state_fill_buf(self, input, input_len); } INLINE output_t chunk_state_output(const blake3_chunk_state *self) { @@ -154,10 +159,10 @@ INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], // Given some input larger than one chunk, return the number of bytes that // should go in the left subtree. This is the largest power-of-2 number of // chunks that leaves at least 1 byte for the right subtree. -INLINE size_t left_len(size_t content_len) { - // Subtract 1 to reserve at least one byte for the right side. content_len +INLINE size_t left_subtree_len(size_t input_len) { + // Subtract 1 to reserve at least one byte for the right side. input_len // should always be greater than BLAKE3_CHUNK_LEN. - size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; + size_t full_chunks = (input_len - 1) / BLAKE3_CHUNK_LEN; return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; } @@ -261,11 +266,10 @@ INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, // Why not just have the caller split the input on the first update(), instead // of implementing this special rule? Because we don't want to limit SIMD or // multi-threading parallelism for that update(). -static size_t blake3_compress_subtree_wide(const uint8_t *input, - size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, - uint8_t flags, uint8_t *out) { +size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb) { // Note that the single chunk case does *not* bump the SIMD degree up to 2 // when it is 1. If this implementation adds multi-threading in the future, // this gives us the option of multi-threading even the 2-chunk case, which @@ -279,7 +283,7 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, // the input into left and right subtrees. (Note that this is only optimal // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree // of 3 or something, we'll need a more complicated strategy.) - size_t left_input_len = left_len(input_len); + size_t left_input_len = left_subtree_len(input_len); size_t right_input_len = input_len - left_input_len; const uint8_t *right_input = &input[left_input_len]; uint64_t right_chunk_counter = @@ -299,12 +303,24 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, } uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; - // Recurse! If this implementation adds multi-threading support in the - // future, this is where it will go. - size_t left_n = blake3_compress_subtree_wide(input, left_input_len, key, - chunk_counter, flags, cv_array); - size_t right_n = blake3_compress_subtree_wide( - right_input, right_input_len, key, right_chunk_counter, flags, right_cvs); + // Recurse! + size_t left_n = SIZE_MAX; + size_t right_n = SIZE_MAX; + +#if defined(BLAKE3_USE_TBB) + blake3_compress_subtree_wide_join_tbb( + key, flags, use_tbb, + // left-hand side + input, left_input_len, chunk_counter, cv_array, &left_n, + // right-hand side + right_input, right_input_len, right_chunk_counter, right_cvs, &right_n); +#else + left_n = blake3_compress_subtree_wide( + input, left_input_len, key, chunk_counter, flags, cv_array, use_tbb); + right_n = blake3_compress_subtree_wide(right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs, + use_tbb); +#endif // BLAKE3_USE_TBB // The special case again. If simd_degree=1, then we'll have left_n=1 and // right_n=1. Rather than compressing them into a single output, return @@ -330,32 +346,37 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, // // As with compress_subtree_wide(), this function is not used on inputs of 1 // chunk or less. That's a different codepath. -INLINE void compress_subtree_to_parent_node( - const uint8_t *input, size_t input_len, const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { +INLINE void +compress_subtree_to_parent_node(const uint8_t *input, size_t input_len, + const uint32_t key[8], uint64_t chunk_counter, + uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN], + bool use_tbb) { #if defined(BLAKE3_TESTING) assert(input_len > BLAKE3_CHUNK_LEN); #endif uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, - chunk_counter, flags, cv_array); + chunk_counter, flags, cv_array, use_tbb); assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); - - // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, + // The following loop never executes when MAX_SIMD_DEGREE_OR_2 is 2, because + // as we just asserted, num_cvs will always be <=2 in that case. But GCC + // (particularly GCC 8.5) can't tell that it never executes, and if NDEBUG is + // set then it emits incorrect warnings here. We tried a few different + // hacks to silence these, but in the end our hacks just produced different + // warnings (see https://github.com/BLAKE3-team/BLAKE3/pull/380). Out of + // desperation, we ifdef out this entire loop when we know it's not needed. +#if MAX_SIMD_DEGREE_OR_2 > 2 + // If MAX_SIMD_DEGREE_OR_2 is greater than 2 and there's enough input, // compress_subtree_wide() returns more than 2 chaining values. Condense // them into 2 by forming parent nodes repeatedly. uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - // The second half of this loop condition is always true, and we just - // asserted it above. But GCC can't tell that it's always true, and if NDEBUG - // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious - // warnings here. GCC 8.5 is particularly sensitive, so if you're changing - // this code, test it against that version. - while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { + while (num_cvs > 2) { num_cvs = compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); } +#endif memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); } @@ -427,7 +448,7 @@ INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { // of the whole tree, and it would need to be ROOT finalized. We can't // compress it until we know. // 2) This 64 KiB input might complete a larger tree, whose root node is -// similarly going to be the the root of the whole tree. For example, maybe +// similarly going to be the root of the whole tree. For example, maybe // we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the // node at the root of the 256 KiB subtree until we know how to finalize it. // @@ -452,8 +473,8 @@ INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], self->cv_stack_len += 1; } -void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len) { +INLINE void blake3_hasher_update_base(blake3_hasher *self, const void *input, + size_t input_len, bool use_tbb) { // Explicitly checking for zero avoids causing UB by passing a null pointer // to memcpy. This comes up in practice with things like: // std::vector v; @@ -539,7 +560,7 @@ void blake3_hasher_update(blake3_hasher *self, const void *input, uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, self->chunk.chunk_counter, - self->chunk.flags, cv_pair); + self->chunk.flags, cv_pair, use_tbb); hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], self->chunk.chunk_counter + (subtree_chunks / 2)); @@ -561,6 +582,20 @@ void blake3_hasher_update(blake3_hasher *self, const void *input, } } +void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = false; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} + +#if defined(BLAKE3_USE_TBB) +void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = true; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} +#endif // BLAKE3_USE_TBB + void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len) { blake3_hasher_finalize_seek(self, 0, out, out_len); diff --git a/src/third_party/blake3/c/blake3.h b/src/third_party/blake3/blake3.h similarity index 90% rename from src/third_party/blake3/c/blake3.h rename to src/third_party/blake3/blake3.h index f694dcf27..c5b799870 100644 --- a/src/third_party/blake3/c/blake3.h +++ b/src/third_party/blake3/blake3.h @@ -30,7 +30,7 @@ extern "C" { #endif -#define BLAKE3_VERSION_STRING "1.5.0" +#define BLAKE3_VERSION_STRING "1.8.7" #define BLAKE3_KEY_LEN 32 #define BLAKE3_OUT_LEN 32 #define BLAKE3_BLOCK_LEN 64 @@ -38,7 +38,7 @@ extern "C" { #define BLAKE3_MAX_DEPTH 54 // This struct is a private implementation detail. It has to be here because -// it's part of blake3_hasher below. +// it's part of the blake3_hasher structure defined below. typedef struct { uint32_t cv[8]; uint64_t chunk_counter; @@ -69,6 +69,10 @@ BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const voi size_t context_len); BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, size_t input_len); +#if defined(BLAKE3_USE_TBB) +BLAKE3_API void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len); +#endif // BLAKE3_USE_TBB BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len); BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, diff --git a/src/third_party/blake3/asm/blake3_avx2_x86-64_unix.S b/src/third_party/blake3/blake3_avx2_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_avx2_x86-64_unix.S rename to src/third_party/blake3/blake3_avx2_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_avx2_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_avx2_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_avx2_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_avx2_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S b/src/third_party/blake3/blake3_avx512_x86-64_unix.S similarity index 55% rename from src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S rename to src/third_party/blake3/blake3_avx512_x86-64_unix.S index a06aede0f..9642e413a 100644 --- a/src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S +++ b/src/third_party/blake3/blake3_avx512_x86-64_unix.S @@ -19,6 +19,8 @@ .global _blake3_compress_in_place_avx512 .global blake3_compress_xof_avx512 .global _blake3_compress_xof_avx512 +.global blake3_xof_many_avx512 +.global _blake3_xof_many_avx512 #ifdef __APPLE__ .text @@ -2553,6 +2555,2243 @@ blake3_compress_xof_avx512: vmovdqu xmmword ptr [r9+0x30], xmm3 ret +.p2align 6 +blake3_xof_many_avx512: +_blake3_xof_many_avx512: + _CET_ENDBR + mov r10,QWORD PTR [rsp+0x8] + cmp r10,0x1 + ja 2f + vmovdqu xmm0,XMMWORD PTR [rdi] + vmovdqu xmm1,XMMWORD PTR [rdi+0x10] + movzx eax,r8b + movzx edx,dl + shl rax,0x20 + add rdx,rax + vmovq xmm3,rcx + vmovq xmm4,rdx + vpunpcklqdq xmm3,xmm3,xmm4 + vmovaps xmm2,XMMWORD PTR [BLAKE3_IV+rip] + vmovups xmm8,XMMWORD PTR [rsi] + vmovups xmm9,XMMWORD PTR [rsi+0x10] + vshufps xmm4,xmm8,xmm9,0x88 + vshufps xmm5,xmm8,xmm9,0xdd + vmovups xmm8,XMMWORD PTR [rsi+0x20] + vmovups xmm9,XMMWORD PTR [rsi+0x30] + vshufps xmm6,xmm8,xmm9,0x88 + vshufps xmm7,xmm8,xmm9,0xdd + vpshufd xmm6,xmm6,0x93 + vpshufd xmm7,xmm7,0x93 + mov al,0x7 +3: + vpaddd xmm0,xmm0,xmm4 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm5 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x93 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x39 + vpaddd xmm0,xmm0,xmm6 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm7 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x39 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x93 + dec al + je 3f + vshufps xmm8,xmm4,xmm5,0xd6 + vpshufd xmm9,xmm4,0xf + vpshufd xmm4,xmm8,0x39 + vshufps xmm8,xmm6,xmm7,0xfa + vpblendd xmm9,xmm9,xmm8,0xaa + vpunpcklqdq xmm8,xmm7,xmm5 + vpblendd xmm8,xmm8,xmm6,0x88 + vpshufd xmm8,xmm8,0x78 + vpunpckhdq xmm5,xmm5,xmm7 + vpunpckldq xmm6,xmm6,xmm5 + vpshufd xmm7,xmm6,0x1e + vmovdqa xmm5,xmm9 + vmovdqa xmm6,xmm8 + jmp 3b +3: + vpxor xmm0,xmm0,xmm2 + vpxor xmm1,xmm1,xmm3 + vpxor xmm2,xmm2,XMMWORD PTR [rdi] + vpxor xmm3,xmm3,XMMWORD PTR [rdi+0x10] + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + ret +.p2align 6 +2: + push rbp + mov rbp,rsp + sub rsp,0x90 + and rsp,0xffffffffffffffc0 + vpbroadcastd zmm0,ecx + shr rcx,0x20 + vpbroadcastd zmm1,ecx + vpaddd zmm2,zmm0,ZMMWORD PTR [ADD0+rip] + vpcmpltud k1,zmm2,zmm0 + vpaddd zmm1{k1},zmm1,DWORD PTR [ADD1+rip]{1to16} + vmovdqa32 ZMMWORD PTR [rsp],zmm2 + vmovdqa32 ZMMWORD PTR [rsp+0x40],zmm1 + cmp r10,0x10 + jb 2f +3: + vpbroadcastd zmm16,DWORD PTR [rsi] + vpbroadcastd zmm17,DWORD PTR [rsi+0x4] + vpbroadcastd zmm18,DWORD PTR [rsi+0x8] + vpbroadcastd zmm19,DWORD PTR [rsi+0xc] + vpbroadcastd zmm20,DWORD PTR [rsi+0x10] + vpbroadcastd zmm21,DWORD PTR [rsi+0x14] + vpbroadcastd zmm22,DWORD PTR [rsi+0x18] + vpbroadcastd zmm23,DWORD PTR [rsi+0x1c] + vpbroadcastd zmm24,DWORD PTR [rsi+0x20] + vpbroadcastd zmm25,DWORD PTR [rsi+0x24] + vpbroadcastd zmm26,DWORD PTR [rsi+0x28] + vpbroadcastd zmm27,DWORD PTR [rsi+0x2c] + vpbroadcastd zmm28,DWORD PTR [rsi+0x30] + vpbroadcastd zmm29,DWORD PTR [rsi+0x34] + vpbroadcastd zmm30,DWORD PTR [rsi+0x38] + vpbroadcastd zmm31,DWORD PTR [rsi+0x3c] + vpbroadcastd zmm0,DWORD PTR [rdi] + vpbroadcastd zmm1,DWORD PTR [rdi+0x4] + vpbroadcastd zmm2,DWORD PTR [rdi+0x8] + vpbroadcastd zmm3,DWORD PTR [rdi+0xc] + vpbroadcastd zmm4,DWORD PTR [rdi+0x10] + vpbroadcastd zmm5,DWORD PTR [rdi+0x14] + vpbroadcastd zmm6,DWORD PTR [rdi+0x18] + vpbroadcastd zmm7,DWORD PTR [rdi+0x1c] + vpbroadcastd zmm8,DWORD PTR [BLAKE3_IV_0+rip] + vpbroadcastd zmm9,DWORD PTR [BLAKE3_IV_1+rip] + vpbroadcastd zmm10,DWORD PTR [BLAKE3_IV_2+rip] + vpbroadcastd zmm11,DWORD PTR [BLAKE3_IV_3+rip] + vmovdqa32 zmm12,ZMMWORD PTR [rsp] + vmovdqa32 zmm13,ZMMWORD PTR [rsp+0x40] + vpbroadcastd zmm14,edx + vpbroadcastd zmm15,r8d + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm20 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm17 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm21 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm24 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm28 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm25 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm29 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm18 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm23 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm22 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm16 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm17 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm25 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm27 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm30 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm19 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm29 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm20 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm22 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm27 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm21 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm31 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm26 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm30 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm23 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm19 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm20 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm21 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm24 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm28 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm31 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm29 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm26 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm23 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm16 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm18 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm17 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm25 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm24 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm30 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm28 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm29 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm19 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm22 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm27 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm17 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm31 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm25 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm30 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm19 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm26 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm20 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpxord zmm0,zmm0,zmm8 + vpxord zmm1,zmm1,zmm9 + vpxord zmm2,zmm2,zmm10 + vpxord zmm3,zmm3,zmm11 + vpxord zmm4,zmm4,zmm12 + vpxord zmm5,zmm5,zmm13 + vpxord zmm6,zmm6,zmm14 + vpxord zmm7,zmm7,zmm15 + vpxord zmm8,zmm8,DWORD PTR [rdi]{1to16} + vpxord zmm9,zmm9,DWORD PTR [rdi+0x4]{1to16} + vpxord zmm10,zmm10,DWORD PTR [rdi+0x8]{1to16} + vpxord zmm11,zmm11,DWORD PTR [rdi+0xc]{1to16} + vpxord zmm12,zmm12,DWORD PTR [rdi+0x10]{1to16} + vpxord zmm13,zmm13,DWORD PTR [rdi+0x14]{1to16} + vpxord zmm14,zmm14,DWORD PTR [rdi+0x18]{1to16} + vpxord zmm15,zmm15,DWORD PTR [rdi+0x1c]{1to16} + vpunpckldq zmm16,zmm0,zmm1 + vpunpckhdq zmm17,zmm0,zmm1 + vpunpckldq zmm18,zmm2,zmm3 + vpunpckhdq zmm19,zmm2,zmm3 + vpunpckldq zmm20,zmm4,zmm5 + vpunpckhdq zmm21,zmm4,zmm5 + vpunpckldq zmm22,zmm6,zmm7 + vpunpckhdq zmm23,zmm6,zmm7 + vpunpckldq zmm24,zmm8,zmm9 + vpunpckhdq zmm25,zmm8,zmm9 + vpunpckldq zmm26,zmm10,zmm11 + vpunpckhdq zmm27,zmm10,zmm11 + vpunpckldq zmm28,zmm12,zmm13 + vpunpckhdq zmm29,zmm12,zmm13 + vpunpckldq zmm30,zmm14,zmm15 + vpunpckhdq zmm31,zmm14,zmm15 + vpunpcklqdq zmm0,zmm16,zmm18 + vpunpckhqdq zmm1,zmm16,zmm18 + vpunpcklqdq zmm2,zmm17,zmm19 + vpunpckhqdq zmm3,zmm17,zmm19 + vpunpcklqdq zmm4,zmm20,zmm22 + vpunpckhqdq zmm5,zmm20,zmm22 + vpunpcklqdq zmm6,zmm21,zmm23 + vpunpckhqdq zmm7,zmm21,zmm23 + vpunpcklqdq zmm8,zmm24,zmm26 + vpunpckhqdq zmm9,zmm24,zmm26 + vpunpcklqdq zmm10,zmm25,zmm27 + vpunpckhqdq zmm11,zmm25,zmm27 + vpunpcklqdq zmm12,zmm28,zmm30 + vpunpckhqdq zmm13,zmm28,zmm30 + vpunpcklqdq zmm14,zmm29,zmm31 + vpunpckhqdq zmm15,zmm29,zmm31 + vshufi32x4 zmm16,zmm0,zmm4,0x88 + vshufi32x4 zmm17,zmm1,zmm5,0x88 + vshufi32x4 zmm18,zmm2,zmm6,0x88 + vshufi32x4 zmm19,zmm3,zmm7,0x88 + vshufi32x4 zmm20,zmm0,zmm4,0xdd + vshufi32x4 zmm21,zmm1,zmm5,0xdd + vshufi32x4 zmm22,zmm2,zmm6,0xdd + vshufi32x4 zmm23,zmm3,zmm7,0xdd + vshufi32x4 zmm24,zmm8,zmm12,0x88 + vshufi32x4 zmm25,zmm9,zmm13,0x88 + vshufi32x4 zmm26,zmm10,zmm14,0x88 + vshufi32x4 zmm27,zmm11,zmm15,0x88 + vshufi32x4 zmm28,zmm8,zmm12,0xdd + vshufi32x4 zmm29,zmm9,zmm13,0xdd + vshufi32x4 zmm30,zmm10,zmm14,0xdd + vshufi32x4 zmm31,zmm11,zmm15,0xdd + vshufi32x4 zmm0,zmm16,zmm24,0x88 + vshufi32x4 zmm1,zmm17,zmm25,0x88 + vshufi32x4 zmm2,zmm18,zmm26,0x88 + vshufi32x4 zmm3,zmm19,zmm27,0x88 + vshufi32x4 zmm4,zmm20,zmm28,0x88 + vshufi32x4 zmm5,zmm21,zmm29,0x88 + vshufi32x4 zmm6,zmm22,zmm30,0x88 + vshufi32x4 zmm7,zmm23,zmm31,0x88 + vshufi32x4 zmm8,zmm16,zmm24,0xdd + vshufi32x4 zmm9,zmm17,zmm25,0xdd + vshufi32x4 zmm10,zmm18,zmm26,0xdd + vshufi32x4 zmm11,zmm19,zmm27,0xdd + vshufi32x4 zmm12,zmm20,zmm28,0xdd + vshufi32x4 zmm13,zmm21,zmm29,0xdd + vshufi32x4 zmm14,zmm22,zmm30,0xdd + vshufi32x4 zmm15,zmm23,zmm31,0xdd + vmovdqu32 ZMMWORD PTR [r9],zmm0 + vmovdqu32 ZMMWORD PTR [r9+0x40],zmm1 + vmovdqu32 ZMMWORD PTR [r9+0x80],zmm2 + vmovdqu32 ZMMWORD PTR [r9+0xc0],zmm3 + vmovdqu32 ZMMWORD PTR [r9+0x100],zmm4 + vmovdqu32 ZMMWORD PTR [r9+0x140],zmm5 + vmovdqu32 ZMMWORD PTR [r9+0x180],zmm6 + vmovdqu32 ZMMWORD PTR [r9+0x1c0],zmm7 + vmovdqu32 ZMMWORD PTR [r9+0x200],zmm8 + vmovdqu32 ZMMWORD PTR [r9+0x240],zmm9 + vmovdqu32 ZMMWORD PTR [r9+0x280],zmm10 + vmovdqu32 ZMMWORD PTR [r9+0x2c0],zmm11 + vmovdqu32 ZMMWORD PTR [r9+0x300],zmm12 + vmovdqu32 ZMMWORD PTR [r9+0x340],zmm13 + vmovdqu32 ZMMWORD PTR [r9+0x380],zmm14 + vmovdqu32 ZMMWORD PTR [r9+0x3c0],zmm15 + vmovdqa32 zmm0,ZMMWORD PTR [rsp] + vmovdqa32 zmm1,ZMMWORD PTR [rsp+0x40] + vpaddd zmm2,zmm0,DWORD PTR [ADD16+rip]{1to16} + vpcmpltud k1,zmm2,zmm0 + vpaddd zmm1{k1},zmm1,DWORD PTR [ADD1+rip]{1to16} + vmovdqa32 ZMMWORD PTR [rsp],zmm2 + vmovdqa32 ZMMWORD PTR [rsp+0x40],zmm1 + add r9,0x400 + sub r10,0x10 + cmp r10,0x10 + jae 3b + test r10,r10 + jne 2f +9: + vzeroupper + mov rsp,rbp + pop rbp + ret +2: + test r10,0x8 + je 2f + vpbroadcastd ymm16,DWORD PTR [rsi] + vpbroadcastd ymm17,DWORD PTR [rsi+0x4] + vpbroadcastd ymm18,DWORD PTR [rsi+0x8] + vpbroadcastd ymm19,DWORD PTR [rsi+0xc] + vpbroadcastd ymm20,DWORD PTR [rsi+0x10] + vpbroadcastd ymm21,DWORD PTR [rsi+0x14] + vpbroadcastd ymm22,DWORD PTR [rsi+0x18] + vpbroadcastd ymm23,DWORD PTR [rsi+0x1c] + vpbroadcastd ymm24,DWORD PTR [rsi+0x20] + vpbroadcastd ymm25,DWORD PTR [rsi+0x24] + vpbroadcastd ymm26,DWORD PTR [rsi+0x28] + vpbroadcastd ymm27,DWORD PTR [rsi+0x2c] + vpbroadcastd ymm28,DWORD PTR [rsi+0x30] + vpbroadcastd ymm29,DWORD PTR [rsi+0x34] + vpbroadcastd ymm30,DWORD PTR [rsi+0x38] + vpbroadcastd ymm31,DWORD PTR [rsi+0x3c] + vpbroadcastd ymm0,DWORD PTR [rdi] + vpbroadcastd ymm1,DWORD PTR [rdi+0x4] + vpbroadcastd ymm2,DWORD PTR [rdi+0x8] + vpbroadcastd ymm3,DWORD PTR [rdi+0xc] + vpbroadcastd ymm4,DWORD PTR [rdi+0x10] + vpbroadcastd ymm5,DWORD PTR [rdi+0x14] + vpbroadcastd ymm6,DWORD PTR [rdi+0x18] + vpbroadcastd ymm7,DWORD PTR [rdi+0x1c] + vpbroadcastd ymm8,DWORD PTR [BLAKE3_IV_0+rip] + vpbroadcastd ymm9,DWORD PTR [BLAKE3_IV_1+rip] + vpbroadcastd ymm10,DWORD PTR [BLAKE3_IV_2+rip] + vpbroadcastd ymm11,DWORD PTR [BLAKE3_IV_3+rip] + vmovdqa ymm12,YMMWORD PTR [rsp] + vmovdqa ymm13,YMMWORD PTR [rsp+0x40] + vpbroadcastd ymm14,edx + vpbroadcastd ymm15,r8d + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm20 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm17 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm21 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm24 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm28 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm25 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm29 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm18 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm23 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm22 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm16 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm17 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm25 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm27 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm30 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm19 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm29 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm20 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm22 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm27 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm21 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm31 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm26 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm30 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm23 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm19 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm20 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm21 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm24 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm28 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm31 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm29 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm26 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm23 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm16 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm18 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm17 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm25 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm24 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm30 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm28 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm29 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm19 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm22 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm27 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm17 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm31 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm25 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm30 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm19 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm26 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm20 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpxor ymm0,ymm0,ymm8 + vpxor ymm1,ymm1,ymm9 + vpxor ymm2,ymm2,ymm10 + vpxor ymm3,ymm3,ymm11 + vpxor ymm4,ymm4,ymm12 + vpxor ymm5,ymm5,ymm13 + vpxor ymm6,ymm6,ymm14 + vpxor ymm7,ymm7,ymm15 + vpxord ymm8,ymm8,DWORD PTR [rdi]{1to8} + vpxord ymm9,ymm9,DWORD PTR [rdi+0x4]{1to8} + vpxord ymm10,ymm10,DWORD PTR [rdi+0x8]{1to8} + vpxord ymm11,ymm11,DWORD PTR [rdi+0xc]{1to8} + vpxord ymm12,ymm12,DWORD PTR [rdi+0x10]{1to8} + vpxord ymm13,ymm13,DWORD PTR [rdi+0x14]{1to8} + vpxord ymm14,ymm14,DWORD PTR [rdi+0x18]{1to8} + vpxord ymm15,ymm15,DWORD PTR [rdi+0x1c]{1to8} + vpunpckldq ymm16,ymm0,ymm1 + vpunpckhdq ymm17,ymm0,ymm1 + vpunpckldq ymm18,ymm2,ymm3 + vpunpckhdq ymm19,ymm2,ymm3 + vpunpckldq ymm20,ymm4,ymm5 + vpunpckhdq ymm21,ymm4,ymm5 + vpunpckldq ymm22,ymm6,ymm7 + vpunpckhdq ymm23,ymm6,ymm7 + vpunpckldq ymm24,ymm8,ymm9 + vpunpckhdq ymm25,ymm8,ymm9 + vpunpckldq ymm26,ymm10,ymm11 + vpunpckhdq ymm27,ymm10,ymm11 + vpunpckldq ymm28,ymm12,ymm13 + vpunpckhdq ymm29,ymm12,ymm13 + vpunpckldq ymm30,ymm14,ymm15 + vpunpckhdq ymm31,ymm14,ymm15 + vpunpcklqdq ymm0,ymm16,ymm18 + vpunpckhqdq ymm1,ymm16,ymm18 + vpunpcklqdq ymm2,ymm17,ymm19 + vpunpckhqdq ymm3,ymm17,ymm19 + vpunpcklqdq ymm4,ymm20,ymm22 + vpunpckhqdq ymm5,ymm20,ymm22 + vpunpcklqdq ymm6,ymm21,ymm23 + vpunpckhqdq ymm7,ymm21,ymm23 + vpunpcklqdq ymm8,ymm24,ymm26 + vpunpckhqdq ymm9,ymm24,ymm26 + vpunpcklqdq ymm10,ymm25,ymm27 + vpunpckhqdq ymm11,ymm25,ymm27 + vpunpcklqdq ymm12,ymm28,ymm30 + vpunpckhqdq ymm13,ymm28,ymm30 + vpunpcklqdq ymm14,ymm29,ymm31 + vpunpckhqdq ymm15,ymm29,ymm31 + vshufi32x4 ymm16,ymm0,ymm4,0x0 + vshufi32x4 ymm17,ymm8,ymm12,0x0 + vshufi32x4 ymm18,ymm1,ymm5,0x0 + vshufi32x4 ymm19,ymm9,ymm13,0x0 + vshufi32x4 ymm20,ymm2,ymm6,0x0 + vshufi32x4 ymm21,ymm10,ymm14,0x0 + vshufi32x4 ymm22,ymm3,ymm7,0x0 + vshufi32x4 ymm23,ymm11,ymm15,0x0 + vshufi32x4 ymm24,ymm0,ymm4,0x3 + vshufi32x4 ymm25,ymm8,ymm12,0x3 + vshufi32x4 ymm26,ymm1,ymm5,0x3 + vshufi32x4 ymm27,ymm9,ymm13,0x3 + vshufi32x4 ymm28,ymm2,ymm6,0x3 + vshufi32x4 ymm29,ymm10,ymm14,0x3 + vshufi32x4 ymm30,ymm3,ymm7,0x3 + vshufi32x4 ymm31,ymm11,ymm15,0x3 + vmovdqu32 YMMWORD PTR [r9],ymm16 + vmovdqu32 YMMWORD PTR [r9+0x20],ymm17 + vmovdqu32 YMMWORD PTR [r9+0x40],ymm18 + vmovdqu32 YMMWORD PTR [r9+0x60],ymm19 + vmovdqu32 YMMWORD PTR [r9+0x80],ymm20 + vmovdqu32 YMMWORD PTR [r9+0xa0],ymm21 + vmovdqu32 YMMWORD PTR [r9+0xc0],ymm22 + vmovdqu32 YMMWORD PTR [r9+0xe0],ymm23 + vmovdqu32 YMMWORD PTR [r9+0x100],ymm24 + vmovdqu32 YMMWORD PTR [r9+0x120],ymm25 + vmovdqu32 YMMWORD PTR [r9+0x140],ymm26 + vmovdqu32 YMMWORD PTR [r9+0x160],ymm27 + vmovdqu32 YMMWORD PTR [r9+0x180],ymm28 + vmovdqu32 YMMWORD PTR [r9+0x1a0],ymm29 + vmovdqu32 YMMWORD PTR [r9+0x1c0],ymm30 + vmovdqu32 YMMWORD PTR [r9+0x1e0],ymm31 + vmovdqa ymm0,YMMWORD PTR [rsp+0x20] + vmovdqa ymm1,YMMWORD PTR [rsp+0x60] + vmovdqa YMMWORD PTR [rsp],ymm0 + vmovdqa YMMWORD PTR [rsp+0x40],ymm1 + add r9,0x200 + sub r10,0x8 +2: + test r10,0x4 + je 2f + vbroadcasti32x4 zmm0,XMMWORD PTR [rdi] + vbroadcasti32x4 zmm1,XMMWORD PTR [rdi+0x10] + vbroadcasti32x4 zmm2,XMMWORD PTR [BLAKE3_IV+rip] + vmovdqa xmm12,XMMWORD PTR [rsp] + vmovdqa xmm13,XMMWORD PTR [rsp+0x40] + vpunpckldq xmm14,xmm12,xmm13 + vpunpckhdq xmm15,xmm12,xmm13 + vpermq ymm14,ymm14,0xdc + vpermq ymm15,ymm15,0xdc + vpbroadcastd zmm12,edx + vinserti64x4 zmm13,zmm14,ymm15,0x1 + mov eax,0x4444 + kmovw k2,eax + vpblendmd zmm13{k2},zmm13,zmm12 + vpbroadcastd zmm15,r8d + mov eax,0x8888 + kmovw k4,eax + vpblendmd zmm3{k4},zmm13,zmm15 + mov eax,0xaaaa + kmovw k3,eax + vbroadcasti32x4 zmm8,XMMWORD PTR [rsi] + vbroadcasti32x4 zmm9,XMMWORD PTR [rsi+0x10] + vshufps zmm4,zmm8,zmm9,0x88 + vshufps zmm5,zmm8,zmm9,0xdd + vbroadcasti32x4 zmm8,XMMWORD PTR [rsi+0x20] + vbroadcasti32x4 zmm9,XMMWORD PTR [rsi+0x30] + vshufps zmm6,zmm8,zmm9,0x88 + vshufps zmm7,zmm8,zmm9,0xdd + vpshufd zmm6,zmm6,0x93 + vpshufd zmm7,zmm7,0x93 + mov al,0x7 +3: + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x10 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0xc + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0x7 + vpshufd zmm0,zmm0,0x93 + vpshufd zmm3,zmm3,0x4e + vpshufd zmm2,zmm2,0x39 + vpaddd zmm0,zmm0,zmm6 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x10 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0xc + vpaddd zmm0,zmm0,zmm7 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0x7 + vpshufd zmm0,zmm0,0x39 + vpshufd zmm3,zmm3,0x4e + vpshufd zmm2,zmm2,0x93 + dec al + je 3f + vshufps zmm8,zmm4,zmm5,0xd6 + vpshufd zmm9,zmm4,0xf + vpshufd zmm4,zmm8,0x39 + vshufps zmm8,zmm6,zmm7,0xfa + vpblendmd zmm9{k3},zmm9,zmm8 + vpunpcklqdq zmm8,zmm7,zmm5 + vpblendmd zmm8{k4},zmm8,zmm6 + vpshufd zmm8,zmm8,0x78 + vpunpckhdq zmm5,zmm5,zmm7 + vpunpckldq zmm6,zmm6,zmm5 + vpshufd zmm7,zmm6,0x1e + vmovdqa32 zmm5,zmm9 + vmovdqa32 zmm6,zmm8 + jmp 3b +3: + vpxord zmm0,zmm0,zmm2 + vpxord zmm1,zmm1,zmm3 + vbroadcasti32x4 zmm8,XMMWORD PTR [rdi] + vbroadcasti32x4 zmm9,XMMWORD PTR [rdi+0x10] + vpxord zmm2,zmm2,zmm8 + vpxord zmm3,zmm3,zmm9 + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + vextracti128 XMMWORD PTR [r9+0x40],ymm0,0x1 + vextracti128 XMMWORD PTR [r9+0x50],ymm1,0x1 + vextracti128 XMMWORD PTR [r9+0x60],ymm2,0x1 + vextracti128 XMMWORD PTR [r9+0x70],ymm3,0x1 + vextracti32x4 XMMWORD PTR [r9+0x80],zmm0,0x2 + vextracti32x4 XMMWORD PTR [r9+0x90],zmm1,0x2 + vextracti32x4 XMMWORD PTR [r9+0xa0],zmm2,0x2 + vextracti32x4 XMMWORD PTR [r9+0xb0],zmm3,0x2 + vextracti32x4 XMMWORD PTR [r9+0xc0],zmm0,0x3 + vextracti32x4 XMMWORD PTR [r9+0xd0],zmm1,0x3 + vextracti32x4 XMMWORD PTR [r9+0xe0],zmm2,0x3 + vextracti32x4 XMMWORD PTR [r9+0xf0],zmm3,0x3 + vmovdqa xmm0,XMMWORD PTR [rsp+0x10] + vmovdqa xmm1,XMMWORD PTR [rsp+0x50] + vmovdqa XMMWORD PTR [rsp],xmm0 + vmovdqa XMMWORD PTR [rsp+0x40],xmm1 + add r9,0x100 + sub r10,0x4 +2: + test r10,0x2 + je 2f + vbroadcasti128 ymm0,XMMWORD PTR [rdi] + vbroadcasti128 ymm1,XMMWORD PTR [rdi+0x10] + vmovd xmm13,DWORD PTR [rsp] + vpinsrd xmm13,xmm13,DWORD PTR [rsp+0x40],0x1 + vpinsrd xmm13,xmm13,edx,0x2 + vmovd xmm14,DWORD PTR [rsp+0x4] + vpinsrd xmm14,xmm14,DWORD PTR [rsp+0x44],0x1 + vpinsrd xmm14,xmm14,edx,0x2 + vinserti128 ymm13,ymm13,xmm14,0x1 + vbroadcasti128 ymm2,XMMWORD PTR [BLAKE3_IV+rip] + vpbroadcastd ymm8,r8d + vpblendd ymm3,ymm13,ymm8,0x88 + vbroadcasti128 ymm8,XMMWORD PTR [rsi] + vbroadcasti128 ymm9,XMMWORD PTR [rsi+0x10] + vshufps ymm4,ymm8,ymm9,0x88 + vshufps ymm5,ymm8,ymm9,0xdd + vbroadcasti128 ymm8,XMMWORD PTR [rsi+0x20] + vbroadcasti128 ymm9,XMMWORD PTR [rsi+0x30] + vshufps ymm6,ymm8,ymm9,0x88 + vshufps ymm7,ymm8,ymm9,0xdd + vpshufd ymm6,ymm6,0x93 + vpshufd ymm7,ymm7,0x93 + mov al,0x7 +3: + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x10 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0xc + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x8 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0x7 + vpshufd ymm0,ymm0,0x93 + vpshufd ymm3,ymm3,0x4e + vpshufd ymm2,ymm2,0x39 + vpaddd ymm0,ymm0,ymm6 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x10 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0xc + vpaddd ymm0,ymm0,ymm7 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x8 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0x7 + vpshufd ymm0,ymm0,0x39 + vpshufd ymm3,ymm3,0x4e + vpshufd ymm2,ymm2,0x93 + dec al + je 3f + vshufps ymm8,ymm4,ymm5,0xd6 + vpshufd ymm9,ymm4,0xf + vpshufd ymm4,ymm8,0x39 + vshufps ymm8,ymm6,ymm7,0xfa + vpblendd ymm9,ymm9,ymm8,0xaa + vpunpcklqdq ymm8,ymm7,ymm5 + vpblendd ymm8,ymm8,ymm6,0x88 + vpshufd ymm8,ymm8,0x78 + vpunpckhdq ymm5,ymm5,ymm7 + vpunpckldq ymm6,ymm6,ymm5 + vpshufd ymm7,ymm6,0x1e + vmovdqa ymm5,ymm9 + vmovdqa ymm6,ymm8 + jmp 3b +3: + vpxor ymm0,ymm0,ymm2 + vpxor ymm1,ymm1,ymm3 + vbroadcasti128 ymm8,XMMWORD PTR [rdi] + vbroadcasti128 ymm9,XMMWORD PTR [rdi+0x10] + vpxor ymm2,ymm2,ymm8 + vpxor ymm3,ymm3,ymm9 + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + vextracti128 XMMWORD PTR [r9+0x40],ymm0,0x1 + vextracti128 XMMWORD PTR [r9+0x50],ymm1,0x1 + vextracti128 XMMWORD PTR [r9+0x60],ymm2,0x1 + vextracti128 XMMWORD PTR [r9+0x70],ymm3,0x1 + vmovdqu xmm0,XMMWORD PTR [rsp+0x8] + vmovdqu xmm1,XMMWORD PTR [rsp+0x48] + vmovdqa XMMWORD PTR [rsp],xmm0 + vmovdqa XMMWORD PTR [rsp+0x40],xmm1 + add r9,0x80 + sub r10,0x2 +2: + test r10,0x1 + je 9b + vmovdqu xmm0,XMMWORD PTR [rdi] + vmovdqu xmm1,XMMWORD PTR [rdi+0x10] + vmovd xmm14,DWORD PTR [rsp] + vpinsrd xmm14,xmm14,DWORD PTR [rsp+0x40],0x1 + vpinsrd xmm14,xmm14,edx,0x2 + vmovdqa xmm2,XMMWORD PTR [BLAKE3_IV+rip] + vpinsrd xmm3,xmm14,r8d,0x3 + vmovups xmm8,XMMWORD PTR [rsi] + vmovups xmm9,XMMWORD PTR [rsi+0x10] + vshufps xmm4,xmm8,xmm9,0x88 + vshufps xmm5,xmm8,xmm9,0xdd + vmovups xmm8,XMMWORD PTR [rsi+0x20] + vmovups xmm9,XMMWORD PTR [rsi+0x30] + vshufps xmm6,xmm8,xmm9,0x88 + vshufps xmm7,xmm8,xmm9,0xdd + vpshufd xmm6,xmm6,0x93 + vpshufd xmm7,xmm7,0x93 + mov al,0x7 +3: + vpaddd xmm0,xmm0,xmm4 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm5 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x93 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x39 + vpaddd xmm0,xmm0,xmm6 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm7 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x39 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x93 + dec al + je 3f + vshufps xmm8,xmm4,xmm5,0xd6 + vpshufd xmm9,xmm4,0xf + vpshufd xmm4,xmm8,0x39 + vshufps xmm8,xmm6,xmm7,0xfa + vpblendd xmm9,xmm9,xmm8,0xaa + vpunpcklqdq xmm8,xmm7,xmm5 + vpblendd xmm8,xmm8,xmm6,0x88 + vpshufd xmm8,xmm8,0x78 + vpunpckhdq xmm5,xmm5,xmm7 + vpunpckldq xmm6,xmm6,xmm5 + vpshufd xmm7,xmm6,0x1e + vmovdqa xmm5,xmm9 + vmovdqa xmm6,xmm8 + jmp 3b +3: + vpxor xmm0,xmm0,xmm2 + vpxor xmm1,xmm1,xmm3 + vpxor xmm2,xmm2,XMMWORD PTR [rdi] + vpxor xmm3,xmm3,XMMWORD PTR [rdi+0x10] + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + jmp 9b + + #ifdef __APPLE__ .static_data #else diff --git a/src/third_party/blake3/asm/blake3_avx512_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_avx512_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/c/blake3_dispatch.c b/src/third_party/blake3/blake3_dispatch.c similarity index 85% rename from src/third_party/blake3/c/blake3_dispatch.c rename to src/third_party/blake3/blake3_dispatch.c index 88f6bc88d..0092a36c0 100644 --- a/src/third_party/blake3/c/blake3_dispatch.c +++ b/src/third_party/blake3/blake3_dispatch.c @@ -4,43 +4,23 @@ #include "blake3_impl.h" -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#elif defined(__GNUC__) -#include -#else -#undef IS_X86 /* Unimplemented! */ -#endif -#endif +#if defined(IS_X86) -#if !defined(BLAKE3_ATOMICS) -#if defined(__has_include) -#if __has_include() && !defined(_MSC_VER) -#define BLAKE3_ATOMICS 1 +#if defined(__GNUC__) || defined(__clang__) +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) __atomic_load_n(&x, __ATOMIC_RELAXED) +#define ATOMIC_STORE(x, y) __atomic_store_n(&x, y, __ATOMIC_RELAXED) #else -#define BLAKE3_ATOMICS 0 -#endif /* __has_include() && !defined(_MSC_VER) */ -#else -#define BLAKE3_ATOMICS 0 -#endif /* defined(__has_include) */ -#endif /* BLAKE3_ATOMICS */ +#include +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) __iso_volatile_load32(&x) +#define ATOMIC_STORE(x, y) __iso_volatile_store32(&x, y) +#endif -#if BLAKE3_ATOMICS -#define ATOMIC_INT _Atomic int -#define ATOMIC_LOAD(x) x -#define ATOMIC_STORE(x, y) x = y -#elif defined(_MSC_VER) -#define ATOMIC_INT long -#define ATOMIC_LOAD(x) _InterlockedOr(&x, 0) -#define ATOMIC_STORE(x, y) _InterlockedExchange(&x, y) -#else -#define ATOMIC_INT int -#define ATOMIC_LOAD(x) x -#define ATOMIC_STORE(x, y) x = y #endif + #define MAYBE_UNUSED(x) (void)((x)) #if defined(IS_X86) @@ -86,7 +66,6 @@ static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { #endif } -#endif enum cpu_feature { SSE2 = 1 << 0, @@ -112,9 +91,9 @@ static get_cpu_features(void) { /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ - long features = ATOMIC_LOAD(g_cpu_features); + enum cpu_feature features = ATOMIC_LOAD(g_cpu_features); if (features != UNDEFINED) { - return (enum cpu_feature)features; + return features; } else { #if defined(IS_X86) uint32_t regs[4] = {0}; @@ -154,13 +133,14 @@ static } } ATOMIC_STORE(g_cpu_features, features); - return (enum cpu_feature)features; + return features; #else /* How to detect NEON? */ return 0; #endif } } +#endif void blake3_compress_in_place(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], @@ -220,6 +200,30 @@ void blake3_compress_xof(const uint32_t cv[8], blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); } + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks) { + if (outblocks == 0) { + // The current assembly implementation always outputs at least 1 block. + return; + } +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); + return; + } +#endif +#endif + for(size_t i = 0; i < outblocks; ++i) { + blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i); + } +} + void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, diff --git a/src/third_party/blake3/asm/blake3_impl.h b/src/third_party/blake3/blake3_impl.h similarity index 84% rename from src/third_party/blake3/asm/blake3_impl.h rename to src/third_party/blake3/blake3_impl.h index beab5cf53..88e71e41e 100644 --- a/src/third_party/blake3/asm/blake3_impl.h +++ b/src/third_party/blake3/blake3_impl.h @@ -9,6 +9,10 @@ #include "blake3.h" +#ifdef __cplusplus +extern "C" { +#endif + // internal flags enum blake3_flags { CHUNK_START = 1 << 0, @@ -28,7 +32,13 @@ enum blake3_flags { #define INLINE static inline __attribute__((always_inline)) #endif -#if defined(__x86_64__) || defined(_M_X64) +#ifdef __cplusplus +#define NOEXCEPT noexcept +#else +#define NOEXCEPT +#endif + +#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC) #define IS_X86 #define IS_X86_64 #endif @@ -38,7 +48,7 @@ enum blake3_flags { #define IS_X86_32 #endif -#if defined(__aarch64__) || defined(_M_ARM64) +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) #define IS_AARCH64 #endif @@ -162,6 +172,13 @@ INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], key_words[7] = load32(&key[7 * 4]); } +INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN], + uint32_t block_words[16]) { + for (size_t i = 0; i < 16; i++) { + block_words[i] = load32(&block[i * 4]); + } +} + INLINE void store32(void *dst, uint32_t w) { uint8_t *p = (uint8_t *)dst; p[0] = (uint8_t)(w >> 0); @@ -191,6 +208,11 @@ void blake3_compress_xof(const uint32_t cv[8], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]); +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks); + void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, @@ -198,6 +220,22 @@ void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blake3_simd_degree(void); +BLAKE3_PRIVATE size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb); + +#if defined(BLAKE3_USE_TBB) +BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb( + // shared params + const uint32_t key[8], uint8_t flags, bool use_tbb, + // left-hand side params + const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter, + uint8_t *l_cvs, size_t *l_n, + // right-hand side params + const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter, + uint8_t *r_cvs, size_t *r_n) NOEXCEPT; +#endif // Declarations for implementation-specific functions. void blake3_compress_in_place_portable(uint32_t cv[8], @@ -270,6 +308,13 @@ void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +#if !defined(_WIN32) && !defined(__CYGWIN__) +void blake3_xof_many_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t* out, size_t outblocks); +#endif #endif #endif @@ -281,5 +326,8 @@ void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, uint8_t flags_end, uint8_t *out); #endif +#ifdef __cplusplus +} +#endif #endif /* BLAKE3_IMPL_H */ diff --git a/src/third_party/blake3/asm/blake3_neon.c b/src/third_party/blake3/blake3_neon.c similarity index 96% rename from src/third_party/blake3/asm/blake3_neon.c rename to src/third_party/blake3/blake3_neon.c index 8a818fc78..794ea80f4 100644 --- a/src/third_party/blake3/asm/blake3_neon.c +++ b/src/third_party/blake3/blake3_neon.c @@ -10,14 +10,12 @@ INLINE uint32x4_t loadu_128(const uint8_t src[16]) { // vld1q_u32 has alignment requirements. Don't use it. - uint32x4_t x; - memcpy(&x, src, 16); - return x; + return vreinterpretq_u32_u8(vld1q_u8(src)); } INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { // vst1q_u32 has alignment requirements. Don't use it. - memcpy(dest, &src, 16); + vst1q_u8(dest, vreinterpretq_u8_u32(src)); } INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { @@ -36,7 +34,7 @@ INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { } INLINE uint32x4_t rot16_128(uint32x4_t x) { - // The straightfoward implementation would be two shifts and an or, but that's + // The straightforward implementation would be two shifts and an or, but that's // slower on microarchitectures we've tested. See // https://github.com/BLAKE3-team/BLAKE3/pull/319. // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); @@ -245,10 +243,11 @@ INLINE void load_counters4(uint64_t counter, bool increment_counter, counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); } -void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { +static void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { uint32x4_t h_vecs[8] = { set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), diff --git a/src/third_party/blake3/asm/blake3_portable.c b/src/third_party/blake3/blake3_portable.c similarity index 100% rename from src/third_party/blake3/asm/blake3_portable.c rename to src/third_party/blake3/blake3_portable.c diff --git a/src/third_party/blake3/asm/blake3_sse2_x86-64_unix.S b/src/third_party/blake3/blake3_sse2_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_sse2_x86-64_unix.S rename to src/third_party/blake3/blake3_sse2_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_sse2_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_sse2_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/asm/blake3_sse41_x86-64_unix.S b/src/third_party/blake3/blake3_sse41_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_sse41_x86-64_unix.S rename to src/third_party/blake3/blake3_sse41_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_sse41_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_sse41_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_sse41_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_sse41_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/c/LICENSE b/src/third_party/blake3/c/LICENSE deleted file mode 100644 index f5892efc3..000000000 --- a/src/third_party/blake3/c/LICENSE +++ /dev/null @@ -1,330 +0,0 @@ -This work is released into the public domain with CC0 1.0. Alternatively, it is -licensed under the Apache License 2.0. - -------------------------------------------------------------------------------- - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2019 Jack O'Connor and Samuel Neves - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/third_party/blake3/c/blake3.c b/src/third_party/blake3/c/blake3.c deleted file mode 100644 index 692f4b021..000000000 --- a/src/third_party/blake3/c/blake3.c +++ /dev/null @@ -1,616 +0,0 @@ -#include -#include -#include - -#include "blake3.h" -#include "blake3_impl.h" - -const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } - -INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], - uint8_t flags) { - memcpy(self->cv, key, BLAKE3_KEY_LEN); - self->chunk_counter = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - self->buf_len = 0; - self->blocks_compressed = 0; - self->flags = flags; -} - -INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], - uint64_t chunk_counter) { - memcpy(self->cv, key, BLAKE3_KEY_LEN); - self->chunk_counter = chunk_counter; - self->blocks_compressed = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - self->buf_len = 0; -} - -INLINE size_t chunk_state_len(const blake3_chunk_state *self) { - return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + - ((size_t)self->buf_len); -} - -INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, - const uint8_t *input, size_t input_len) { - size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); - if (take > input_len) { - take = input_len; - } - uint8_t *dest = self->buf + ((size_t)self->buf_len); - memcpy(dest, input, take); - self->buf_len += (uint8_t)take; - return take; -} - -INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { - if (self->blocks_compressed == 0) { - return CHUNK_START; - } else { - return 0; - } -} - -typedef struct { - uint32_t input_cv[8]; - uint64_t counter; - uint8_t block[BLAKE3_BLOCK_LEN]; - uint8_t block_len; - uint8_t flags; -} output_t; - -INLINE output_t make_output(const uint32_t input_cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - output_t ret; - memcpy(ret.input_cv, input_cv, 32); - memcpy(ret.block, block, BLAKE3_BLOCK_LEN); - ret.block_len = block_len; - ret.counter = counter; - ret.flags = flags; - return ret; -} - -// Chaining values within a given chunk (specifically the compress_in_place -// interface) are represented as words. This avoids unnecessary bytes<->words -// conversion overhead in the portable implementation. However, the hash_many -// interface handles both user input and parent node blocks, so it accepts -// bytes. For that reason, chaining values in the CV stack are represented as -// bytes. -INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { - uint32_t cv_words[8]; - memcpy(cv_words, self->input_cv, 32); - blake3_compress_in_place(cv_words, self->block, self->block_len, - self->counter, self->flags); - store_cv_words(cv, cv_words); -} - -INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, - size_t out_len) { - uint64_t output_block_counter = seek / 64; - size_t offset_within_block = seek % 64; - uint8_t wide_buf[64]; - while (out_len > 0) { - blake3_compress_xof(self->input_cv, self->block, self->block_len, - output_block_counter, self->flags | ROOT, wide_buf); - size_t available_bytes = 64 - offset_within_block; - size_t memcpy_len; - if (out_len > available_bytes) { - memcpy_len = available_bytes; - } else { - memcpy_len = out_len; - } - memcpy(out, wide_buf + offset_within_block, memcpy_len); - out += memcpy_len; - out_len -= memcpy_len; - output_block_counter += 1; - offset_within_block = 0; - } -} - -INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, - size_t input_len) { - if (self->buf_len > 0) { - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; - if (input_len > 0) { - blake3_compress_in_place( - self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, - self->flags | chunk_state_maybe_start_flag(self)); - self->blocks_compressed += 1; - self->buf_len = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - } - } - - while (input_len > BLAKE3_BLOCK_LEN) { - blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, - self->chunk_counter, - self->flags | chunk_state_maybe_start_flag(self)); - self->blocks_compressed += 1; - input += BLAKE3_BLOCK_LEN; - input_len -= BLAKE3_BLOCK_LEN; - } - - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; -} - -INLINE output_t chunk_state_output(const blake3_chunk_state *self) { - uint8_t block_flags = - self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; - return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, - block_flags); -} - -INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], - const uint32_t key[8], uint8_t flags) { - return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); -} - -// Given some input larger than one chunk, return the number of bytes that -// should go in the left subtree. This is the largest power-of-2 number of -// chunks that leaves at least 1 byte for the right subtree. -INLINE size_t left_len(size_t content_len) { - // Subtract 1 to reserve at least one byte for the right side. content_len - // should always be greater than BLAKE3_CHUNK_LEN. - size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; - return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; -} - -// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time -// on a single thread. Write out the chunk chaining values and return the -// number of chunks hashed. These chunks are never the root and never empty; -// those cases use a different codepath. -INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, - uint8_t *out) { -#if defined(BLAKE3_TESTING) - assert(0 < input_len); - assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); -#endif - - const uint8_t *chunks_array[MAX_SIMD_DEGREE]; - size_t input_position = 0; - size_t chunks_array_len = 0; - while (input_len - input_position >= BLAKE3_CHUNK_LEN) { - chunks_array[chunks_array_len] = &input[input_position]; - input_position += BLAKE3_CHUNK_LEN; - chunks_array_len += 1; - } - - blake3_hash_many(chunks_array, chunks_array_len, - BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, - true, flags, CHUNK_START, CHUNK_END, out); - - // Hash the remaining partial chunk, if there is one. Note that the empty - // chunk (meaning the empty message) is a different codepath. - if (input_len > input_position) { - uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; - blake3_chunk_state chunk_state; - chunk_state_init(&chunk_state, key, flags); - chunk_state.chunk_counter = counter; - chunk_state_update(&chunk_state, &input[input_position], - input_len - input_position); - output_t output = chunk_state_output(&chunk_state); - output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]); - return chunks_array_len + 1; - } else { - return chunks_array_len; - } -} - -// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time -// on a single thread. Write out the parent chaining values and return the -// number of parents hashed. (If there's an odd input chaining value left over, -// return it as an additional output.) These parents are never the root and -// never empty; those cases use a different codepath. -INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, - size_t num_chaining_values, - const uint32_t key[8], uint8_t flags, - uint8_t *out) { -#if defined(BLAKE3_TESTING) - assert(2 <= num_chaining_values); - assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); -#endif - - const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; - size_t parents_array_len = 0; - while (num_chaining_values - (2 * parents_array_len) >= 2) { - parents_array[parents_array_len] = - &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; - parents_array_len += 1; - } - - blake3_hash_many(parents_array, parents_array_len, 1, key, - 0, // Parents always use counter 0. - false, flags | PARENT, - 0, // Parents have no start flags. - 0, // Parents have no end flags. - out); - - // If there's an odd child left over, it becomes an output. - if (num_chaining_values > 2 * parents_array_len) { - memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], - &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], - BLAKE3_OUT_LEN); - return parents_array_len + 1; - } else { - return parents_array_len; - } -} - -// The wide helper function returns (writes out) an array of chaining values -// and returns the length of that array. The number of chaining values returned -// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, -// if the input is shorter than that many chunks. The reason for maintaining a -// wide array of chaining values going back up the tree, is to allow the -// implementation to hash as many parents in parallel as possible. -// -// As a special case when the SIMD degree is 1, this function will still return -// at least 2 outputs. This guarantees that this function doesn't perform the -// root compression. (If it did, it would use the wrong flags, and also we -// wouldn't be able to implement extendable output.) Note that this function is -// not used when the whole input is only 1 chunk long; that's a different -// codepath. -// -// Why not just have the caller split the input on the first update(), instead -// of implementing this special rule? Because we don't want to limit SIMD or -// multi-threading parallelism for that update(). -static size_t blake3_compress_subtree_wide(const uint8_t *input, - size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, - uint8_t flags, uint8_t *out) { - // Note that the single chunk case does *not* bump the SIMD degree up to 2 - // when it is 1. If this implementation adds multi-threading in the future, - // this gives us the option of multi-threading even the 2-chunk case, which - // can help performance on smaller platforms. - if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { - return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, - out); - } - - // With more than simd_degree chunks, we need to recurse. Start by dividing - // the input into left and right subtrees. (Note that this is only optimal - // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree - // of 3 or something, we'll need a more complicated strategy.) - size_t left_input_len = left_len(input_len); - size_t right_input_len = input_len - left_input_len; - const uint8_t *right_input = &input[left_input_len]; - uint64_t right_chunk_counter = - chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); - - // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to - // account for the special case of returning 2 outputs when the SIMD degree - // is 1. - uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; - size_t degree = blake3_simd_degree(); - if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { - // The special case: We always use a degree of at least two, to make - // sure there are two outputs. Except, as noted above, at the chunk - // level, where we allow degree=1. (Note that the 1-chunk-input case is - // a different codepath.) - degree = 2; - } - uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; - - // Recurse! If this implementation adds multi-threading support in the - // future, this is where it will go. - size_t left_n = blake3_compress_subtree_wide(input, left_input_len, key, - chunk_counter, flags, cv_array); - size_t right_n = blake3_compress_subtree_wide( - right_input, right_input_len, key, right_chunk_counter, flags, right_cvs); - - // The special case again. If simd_degree=1, then we'll have left_n=1 and - // right_n=1. Rather than compressing them into a single output, return - // them directly, to make sure we always have at least two outputs. - if (left_n == 1) { - memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); - return 2; - } - - // Otherwise, do one layer of parent node compression. - size_t num_chaining_values = left_n + right_n; - return compress_parents_parallel(cv_array, num_chaining_values, key, flags, - out); -} - -// Hash a subtree with compress_subtree_wide(), and then condense the resulting -// list of chaining values down to a single parent node. Don't compress that -// last parent node, however. Instead, return its message bytes (the -// concatenated chaining values of its children). This is necessary when the -// first call to update() supplies a complete subtree, because the topmost -// parent node of that subtree could end up being the root. It's also necessary -// for extended output in the general case. -// -// As with compress_subtree_wide(), this function is not used on inputs of 1 -// chunk or less. That's a different codepath. -INLINE void compress_subtree_to_parent_node( - const uint8_t *input, size_t input_len, const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { -#if defined(BLAKE3_TESTING) - assert(input_len > BLAKE3_CHUNK_LEN); -#endif - - uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; - size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, - chunk_counter, flags, cv_array); - assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); - - // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, - // compress_subtree_wide() returns more than 2 chaining values. Condense - // them into 2 by forming parent nodes repeatedly. - uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - // The second half of this loop condition is always true, and we just - // asserted it above. But GCC can't tell that it's always true, and if NDEBUG - // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious - // warnings here. GCC 8.5 is particularly sensitive, so if you're changing - // this code, test it against that version. - while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { - num_cvs = - compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); - memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); - } - memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); -} - -INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8], - uint8_t flags) { - memcpy(self->key, key, BLAKE3_KEY_LEN); - chunk_state_init(&self->chunk, key, flags); - self->cv_stack_len = 0; -} - -void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); } - -void blake3_hasher_init_keyed(blake3_hasher *self, - const uint8_t key[BLAKE3_KEY_LEN]) { - uint32_t key_words[8]; - load_key_words(key, key_words); - hasher_init_base(self, key_words, KEYED_HASH); -} - -void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, - size_t context_len) { - blake3_hasher context_hasher; - hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT); - blake3_hasher_update(&context_hasher, context, context_len); - uint8_t context_key[BLAKE3_KEY_LEN]; - blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); - uint32_t context_key_words[8]; - load_key_words(context_key, context_key_words); - hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL); -} - -void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) { - blake3_hasher_init_derive_key_raw(self, context, strlen(context)); -} - -// As described in hasher_push_cv() below, we do "lazy merging", delaying -// merges until right before the next CV is about to be added. This is -// different from the reference implementation. Another difference is that we -// aren't always merging 1 chunk at a time. Instead, each CV might represent -// any power-of-two number of chunks, as long as the smaller-above-larger stack -// order is maintained. Instead of the "count the trailing 0-bits" algorithm -// described in the spec, we use a "count the total number of 1-bits" variant -// that doesn't require us to retain the subtree size of the CV on top of the -// stack. The principle is the same: each CV that should remain in the stack is -// represented by a 1-bit in the total number of chunks (or bytes) so far. -INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { - size_t post_merge_stack_len = (size_t)popcnt(total_len); - while (self->cv_stack_len > post_merge_stack_len) { - uint8_t *parent_node = - &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; - output_t output = parent_output(parent_node, self->key, self->chunk.flags); - output_chaining_value(&output, parent_node); - self->cv_stack_len -= 1; - } -} - -// In reference_impl.rs, we merge the new CV with existing CVs from the stack -// before pushing it. We can do that because we know more input is coming, so -// we know none of the merges are root. -// -// This setting is different. We want to feed as much input as possible to -// compress_subtree_wide(), without setting aside anything for the chunk_state. -// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once -// as a single subtree, if at all possible. -// -// This leads to two problems: -// 1) This 64 KiB input might be the only call that ever gets made to update. -// In this case, the root node of the 64 KiB subtree would be the root node -// of the whole tree, and it would need to be ROOT finalized. We can't -// compress it until we know. -// 2) This 64 KiB input might complete a larger tree, whose root node is -// similarly going to be the the root of the whole tree. For example, maybe -// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the -// node at the root of the 256 KiB subtree until we know how to finalize it. -// -// The second problem is solved with "lazy merging". That is, when we're about -// to add a CV to the stack, we don't merge it with anything first, as the -// reference impl does. Instead we do merges using the *previous* CV that was -// added, which is sitting on top of the stack, and we put the new CV -// (unmerged) on top of the stack afterwards. This guarantees that we never -// merge the root node until finalize(). -// -// Solving the first problem requires an additional tool, -// compress_subtree_to_parent_node(). That function always returns the top -// *two* chaining values of the subtree it's compressing. We then do lazy -// merging with each of them separately, so that the second CV will always -// remain unmerged. (That also helps us support extendable output when we're -// hashing an input all-at-once.) -INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], - uint64_t chunk_counter) { - hasher_merge_cv_stack(self, chunk_counter); - memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, - BLAKE3_OUT_LEN); - self->cv_stack_len += 1; -} - -void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len) { - // Explicitly checking for zero avoids causing UB by passing a null pointer - // to memcpy. This comes up in practice with things like: - // std::vector v; - // blake3_hasher_update(&hasher, v.data(), v.size()); - if (input_len == 0) { - return; - } - - const uint8_t *input_bytes = (const uint8_t *)input; - - // If we have some partial chunk bytes in the internal chunk_state, we need - // to finish that chunk first. - if (chunk_state_len(&self->chunk) > 0) { - size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); - if (take > input_len) { - take = input_len; - } - chunk_state_update(&self->chunk, input_bytes, take); - input_bytes += take; - input_len -= take; - // If we've filled the current chunk and there's more coming, finalize this - // chunk and proceed. In this case we know it's not the root. - if (input_len > 0) { - output_t output = chunk_state_output(&self->chunk); - uint8_t chunk_cv[32]; - output_chaining_value(&output, chunk_cv); - hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); - chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); - } else { - return; - } - } - - // Now the chunk_state is clear, and we have more input. If there's more than - // a single chunk (so, definitely not the root chunk), hash the largest whole - // subtree we can, with the full benefits of SIMD (and maybe in the future, - // multi-threading) parallelism. Two restrictions: - // - The subtree has to be a power-of-2 number of chunks. Only subtrees along - // the right edge can be incomplete, and we don't know where the right edge - // is going to be until we get to finalize(). - // - The subtree must evenly divide the total number of chunks up until this - // point (if total is not 0). If the current incomplete subtree is only - // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have - // to complete the current subtree first. - // Because we might need to break up the input to form powers of 2, or to - // evenly divide what we already have, this part runs in a loop. - while (input_len > BLAKE3_CHUNK_LEN) { - size_t subtree_len = round_down_to_power_of_2(input_len); - uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; - // Shrink the subtree_len until it evenly divides the count so far. We know - // that subtree_len itself is a power of 2, so we can use a bitmasking - // trick instead of an actual remainder operation. (Note that if the caller - // consistently passes power-of-2 inputs of the same size, as is hopefully - // typical, this loop condition will always fail, and subtree_len will - // always be the full length of the input.) - // - // An aside: We don't have to shrink subtree_len quite this much. For - // example, if count_so_far is 1, we could pass 2 chunks to - // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still - // get the right answer in the end, and we might get to use 2-way SIMD - // parallelism. The problem with this optimization, is that it gets us - // stuck always hashing 2 chunks. The total number of chunks will remain - // odd, and we'll never graduate to higher degrees of parallelism. See - // https://github.com/BLAKE3-team/BLAKE3/issues/69. - while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { - subtree_len /= 2; - } - // The shrunken subtree_len might now be 1 chunk long. If so, hash that one - // chunk by itself. Otherwise, compress the subtree into a pair of CVs. - uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; - if (subtree_len <= BLAKE3_CHUNK_LEN) { - blake3_chunk_state chunk_state; - chunk_state_init(&chunk_state, self->key, self->chunk.flags); - chunk_state.chunk_counter = self->chunk.chunk_counter; - chunk_state_update(&chunk_state, input_bytes, subtree_len); - output_t output = chunk_state_output(&chunk_state); - uint8_t cv[BLAKE3_OUT_LEN]; - output_chaining_value(&output, cv); - hasher_push_cv(self, cv, chunk_state.chunk_counter); - } else { - // This is the high-performance happy path, though getting here depends - // on the caller giving us a long enough input. - uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; - compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, - self->chunk.chunk_counter, - self->chunk.flags, cv_pair); - hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); - hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], - self->chunk.chunk_counter + (subtree_chunks / 2)); - } - self->chunk.chunk_counter += subtree_chunks; - input_bytes += subtree_len; - input_len -= subtree_len; - } - - // If there's any remaining input less than a full chunk, add it to the chunk - // state. In that case, also do a final merge loop to make sure the subtree - // stack doesn't contain any unmerged pairs. The remaining input means we - // know these merges are non-root. This merge loop isn't strictly necessary - // here, because hasher_push_chunk_cv already does its own merge loop, but it - // simplifies blake3_hasher_finalize below. - if (input_len > 0) { - chunk_state_update(&self->chunk, input_bytes, input_len); - hasher_merge_cv_stack(self, self->chunk.chunk_counter); - } -} - -void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, - size_t out_len) { - blake3_hasher_finalize_seek(self, 0, out, out_len); -} - -void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, - uint8_t *out, size_t out_len) { - // Explicitly checking for zero avoids causing UB by passing a null pointer - // to memcpy. This comes up in practice with things like: - // std::vector v; - // blake3_hasher_finalize(&hasher, v.data(), v.size()); - if (out_len == 0) { - return; - } - - // If the subtree stack is empty, then the current chunk is the root. - if (self->cv_stack_len == 0) { - output_t output = chunk_state_output(&self->chunk); - output_root_bytes(&output, seek, out, out_len); - return; - } - // If there are any bytes in the chunk state, finalize that chunk and do a - // roll-up merge between that chunk hash and every subtree in the stack. In - // this case, the extra merge loop at the end of blake3_hasher_update - // guarantees that none of the subtrees in the stack need to be merged with - // each other first. Otherwise, if there are no bytes in the chunk state, - // then the top of the stack is a chunk hash, and we start the merge from - // that. - output_t output; - size_t cvs_remaining; - if (chunk_state_len(&self->chunk) > 0) { - cvs_remaining = self->cv_stack_len; - output = chunk_state_output(&self->chunk); - } else { - // There are always at least 2 CVs in the stack in this case. - cvs_remaining = self->cv_stack_len - 2; - output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, - self->chunk.flags); - } - while (cvs_remaining > 0) { - cvs_remaining -= 1; - uint8_t parent_block[BLAKE3_BLOCK_LEN]; - memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); - output_chaining_value(&output, &parent_block[32]); - output = parent_output(parent_block, self->key, self->chunk.flags); - } - output_root_bytes(&output, seek, out, out_len); -} - -void blake3_hasher_reset(blake3_hasher *self) { - chunk_state_reset(&self->chunk, self->key, 0); - self->cv_stack_len = 0; -} diff --git a/src/third_party/blake3/c/blake3_avx2.c b/src/third_party/blake3/c/blake3_avx2.c deleted file mode 100644 index 46b5cecfd..000000000 --- a/src/third_party/blake3/c/blake3_avx2.c +++ /dev/null @@ -1,312 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 8 - -INLINE __m256i loadu(const uint8_t src[32]) { - return _mm256_loadu_si256((const __m256i *)src); -} - -INLINE void storeu(__m256i src, uint8_t dest[16]) { - _mm256_storeu_si256((__m256i *)dest, src); -} - -INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } - -INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } - -INLINE __m256i rot16(__m256i x) { - return _mm256_shuffle_epi8( - x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, - 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); -} - -INLINE __m256i rot12(__m256i x) { - return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12)); -} - -INLINE __m256i rot8(__m256i x) { - return _mm256_shuffle_epi8( - x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, - 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); -} - -INLINE __m256i rot7(__m256i x) { - return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7)); -} - -INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m256i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high - // is 22/33/66/77. - __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); - __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); - __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); - __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); - __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); - __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); - __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); - __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); - - // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is - // 11/33. - __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); - __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); - __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); - __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); - __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); - __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); - __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); - __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); - - // Interleave 128-bit lanes. - vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); - vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); - vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); - vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); - vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); - vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); - vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); - vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m256i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]); - out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]); - out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]); - out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]); - out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]); - out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]); - out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]); - out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]); - out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]); - out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]); - out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]); - out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]); - out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]); - for (size_t i = 0; i < 8; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[8]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m256i *out_lo, __m256i *out_hi) { - const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter); - const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); - const __m256i add1 = _mm256_and_si256(mask, add0); - __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1); - __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)), - _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000))); - __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m256i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m256i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m256i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m256i block_flags_vec = set1(block_flags); - __m256i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m256i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(h_vecs); - storeu(h_vecs[0], &out[0 * sizeof(__m256i)]); - storeu(h_vecs[1], &out[1 * sizeof(__m256i)]); - storeu(h_vecs[2], &out[2 * sizeof(__m256i)]); - storeu(h_vecs[3], &out[3 * sizeof(__m256i)]); - storeu(h_vecs[4], &out[4 * sizeof(__m256i)]); - storeu(h_vecs[5], &out[5 * sizeof(__m256i)]); - storeu(h_vecs[6], &out[6 * sizeof(__m256i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m256i)]); -} - -void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } -#if !defined(BLAKE3_NO_SSE41) - blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, out); -#else - blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); -#endif -} diff --git a/src/third_party/blake3/c/blake3_avx512.c b/src/third_party/blake3/c/blake3_avx512.c deleted file mode 100644 index f969bf37f..000000000 --- a/src/third_party/blake3/c/blake3_avx512.c +++ /dev/null @@ -1,1220 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu_128(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE __m256i loadu_256(const uint8_t src[32]) { - return _mm256_loadu_si256((const __m256i *)src); -} - -INLINE __m512i loadu_512(const uint8_t src[64]) { - return _mm512_loadu_si512((const __m512i *)src); -} - -INLINE void storeu_128(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE void storeu_256(__m256i src, uint8_t dest[16]) { - _mm256_storeu_si256((__m256i *)dest, src); -} - -INLINE __m128i add_128(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -INLINE __m256i add_256(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } - -INLINE __m512i add_512(__m512i a, __m512i b) { return _mm512_add_epi32(a, b); } - -INLINE __m128i xor_128(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m256i xor_256(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } - -INLINE __m512i xor_512(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); } - -INLINE __m128i set1_128(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m256i set1_256(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } - -INLINE __m512i set1_512(uint32_t x) { return _mm512_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16_128(__m128i x) { return _mm_ror_epi32(x, 16); } - -INLINE __m256i rot16_256(__m256i x) { return _mm256_ror_epi32(x, 16); } - -INLINE __m512i rot16_512(__m512i x) { return _mm512_ror_epi32(x, 16); } - -INLINE __m128i rot12_128(__m128i x) { return _mm_ror_epi32(x, 12); } - -INLINE __m256i rot12_256(__m256i x) { return _mm256_ror_epi32(x, 12); } - -INLINE __m512i rot12_512(__m512i x) { return _mm512_ror_epi32(x, 12); } - -INLINE __m128i rot8_128(__m128i x) { return _mm_ror_epi32(x, 8); } - -INLINE __m256i rot8_256(__m256i x) { return _mm256_ror_epi32(x, 8); } - -INLINE __m512i rot8_512(__m512i x) { return _mm512_ror_epi32(x, 8); } - -INLINE __m128i rot7_128(__m128i x) { return _mm_ror_epi32(x, 7); } - -INLINE __m256i rot7_256(__m256i x) { return _mm256_ror_epi32(x, 7); } - -INLINE __m512i rot7_512(__m512i x) { return _mm512_ror_epi32(x, 7); } - -/* - * ---------------------------------------------------------------------------- - * compress_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = add_128(add_128(*row0, m), *row1); - *row3 = xor_128(*row3, *row0); - *row3 = rot16_128(*row3); - *row2 = add_128(*row2, *row3); - *row1 = xor_128(*row1, *row2); - *row1 = rot12_128(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = add_128(add_128(*row0, m), *row1); - *row3 = xor_128(*row3, *row0); - *row3 = rot8_128(*row3); - *row2 = add_128(*row2, *row3); - *row1 = xor_128(*row1, *row2); - *row1 = rot7_128(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu_128((uint8_t *)&cv[0]); - rows[1] = loadu_128((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu_128(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu_128(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu_128(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu_128(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_xof_avx512(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu_128(xor_128(rows[0], rows[2]), &out[0]); - storeu_128(xor_128(rows[1], rows[3]), &out[16]); - storeu_128(xor_128(rows[2], loadu_128((uint8_t *)&cv[0])), &out[32]); - storeu_128(xor_128(rows[3], loadu_128((uint8_t *)&cv[4])), &out[48]); -} - -void blake3_compress_in_place_avx512(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu_128(xor_128(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu_128(xor_128(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -/* - * ---------------------------------------------------------------------------- - * hash4_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn4(__m128i v[16], __m128i m[16], size_t r) { - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[15] = rot16_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot12_128(v[4]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[15] = rot8_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot7_128(v[4]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot16_128(v[15]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[4] = rot12_128(v[4]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot8_128(v[15]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - v[4] = rot7_128(v[4]); -} - -INLINE void transpose_vecs_128(__m128i vecs[4]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_128(&out[0]); - transpose_vecs_128(&out[4]); - transpose_vecs_128(&out[8]); - transpose_vecs_128(&out[12]); -} - -INLINE void load_counters4(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - uint64_t mask = (increment_counter ? ~0 : 0); - __m256i mask_vec = _mm256_set1_epi64x(mask); - __m256i deltas = _mm256_setr_epi64x(0, 1, 2, 3); - deltas = _mm256_and_si256(mask_vec, deltas); - __m256i counters = - _mm256_add_epi64(_mm256_set1_epi64x((int64_t)counter), deltas); - *out_lo = _mm256_cvtepi64_epi32(counters); - *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32)); -} - -static -void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), - set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters4(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1_128(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1_128(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn4(v, msg_vecs, 0); - round_fn4(v, msg_vecs, 1); - round_fn4(v, msg_vecs, 2); - round_fn4(v, msg_vecs, 3); - round_fn4(v, msg_vecs, 4); - round_fn4(v, msg_vecs, 5); - round_fn4(v, msg_vecs, 6); - h_vecs[0] = xor_128(v[0], v[8]); - h_vecs[1] = xor_128(v[1], v[9]); - h_vecs[2] = xor_128(v[2], v[10]); - h_vecs[3] = xor_128(v[3], v[11]); - h_vecs[4] = xor_128(v[4], v[12]); - h_vecs[5] = xor_128(v[5], v[13]); - h_vecs[6] = xor_128(v[6], v[14]); - h_vecs[7] = xor_128(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_128(&h_vecs[0]); - transpose_vecs_128(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu_128(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu_128(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu_128(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu_128(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu_128(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu_128(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu_128(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu_128(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash8_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn8(__m256i v[16], __m256i m[16], size_t r) { - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_256(v[0], v[4]); - v[1] = add_256(v[1], v[5]); - v[2] = add_256(v[2], v[6]); - v[3] = add_256(v[3], v[7]); - v[12] = xor_256(v[12], v[0]); - v[13] = xor_256(v[13], v[1]); - v[14] = xor_256(v[14], v[2]); - v[15] = xor_256(v[15], v[3]); - v[12] = rot16_256(v[12]); - v[13] = rot16_256(v[13]); - v[14] = rot16_256(v[14]); - v[15] = rot16_256(v[15]); - v[8] = add_256(v[8], v[12]); - v[9] = add_256(v[9], v[13]); - v[10] = add_256(v[10], v[14]); - v[11] = add_256(v[11], v[15]); - v[4] = xor_256(v[4], v[8]); - v[5] = xor_256(v[5], v[9]); - v[6] = xor_256(v[6], v[10]); - v[7] = xor_256(v[7], v[11]); - v[4] = rot12_256(v[4]); - v[5] = rot12_256(v[5]); - v[6] = rot12_256(v[6]); - v[7] = rot12_256(v[7]); - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_256(v[0], v[4]); - v[1] = add_256(v[1], v[5]); - v[2] = add_256(v[2], v[6]); - v[3] = add_256(v[3], v[7]); - v[12] = xor_256(v[12], v[0]); - v[13] = xor_256(v[13], v[1]); - v[14] = xor_256(v[14], v[2]); - v[15] = xor_256(v[15], v[3]); - v[12] = rot8_256(v[12]); - v[13] = rot8_256(v[13]); - v[14] = rot8_256(v[14]); - v[15] = rot8_256(v[15]); - v[8] = add_256(v[8], v[12]); - v[9] = add_256(v[9], v[13]); - v[10] = add_256(v[10], v[14]); - v[11] = add_256(v[11], v[15]); - v[4] = xor_256(v[4], v[8]); - v[5] = xor_256(v[5], v[9]); - v[6] = xor_256(v[6], v[10]); - v[7] = xor_256(v[7], v[11]); - v[4] = rot7_256(v[4]); - v[5] = rot7_256(v[5]); - v[6] = rot7_256(v[6]); - v[7] = rot7_256(v[7]); - - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_256(v[0], v[5]); - v[1] = add_256(v[1], v[6]); - v[2] = add_256(v[2], v[7]); - v[3] = add_256(v[3], v[4]); - v[15] = xor_256(v[15], v[0]); - v[12] = xor_256(v[12], v[1]); - v[13] = xor_256(v[13], v[2]); - v[14] = xor_256(v[14], v[3]); - v[15] = rot16_256(v[15]); - v[12] = rot16_256(v[12]); - v[13] = rot16_256(v[13]); - v[14] = rot16_256(v[14]); - v[10] = add_256(v[10], v[15]); - v[11] = add_256(v[11], v[12]); - v[8] = add_256(v[8], v[13]); - v[9] = add_256(v[9], v[14]); - v[5] = xor_256(v[5], v[10]); - v[6] = xor_256(v[6], v[11]); - v[7] = xor_256(v[7], v[8]); - v[4] = xor_256(v[4], v[9]); - v[5] = rot12_256(v[5]); - v[6] = rot12_256(v[6]); - v[7] = rot12_256(v[7]); - v[4] = rot12_256(v[4]); - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_256(v[0], v[5]); - v[1] = add_256(v[1], v[6]); - v[2] = add_256(v[2], v[7]); - v[3] = add_256(v[3], v[4]); - v[15] = xor_256(v[15], v[0]); - v[12] = xor_256(v[12], v[1]); - v[13] = xor_256(v[13], v[2]); - v[14] = xor_256(v[14], v[3]); - v[15] = rot8_256(v[15]); - v[12] = rot8_256(v[12]); - v[13] = rot8_256(v[13]); - v[14] = rot8_256(v[14]); - v[10] = add_256(v[10], v[15]); - v[11] = add_256(v[11], v[12]); - v[8] = add_256(v[8], v[13]); - v[9] = add_256(v[9], v[14]); - v[5] = xor_256(v[5], v[10]); - v[6] = xor_256(v[6], v[11]); - v[7] = xor_256(v[7], v[8]); - v[4] = xor_256(v[4], v[9]); - v[5] = rot7_256(v[5]); - v[6] = rot7_256(v[6]); - v[7] = rot7_256(v[7]); - v[4] = rot7_256(v[4]); -} - -INLINE void transpose_vecs_256(__m256i vecs[8]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high - // is 22/33/66/77. - __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); - __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); - __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); - __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); - __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); - __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); - __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); - __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); - - // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is - // 11/33. - __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); - __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); - __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); - __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); - __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); - __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); - __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); - __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); - - // Interleave 128-bit lanes. - vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); - vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); - vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); - vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); - vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); - vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); - vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); - vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); -} - -INLINE void transpose_msg_vecs8(const uint8_t *const *inputs, - size_t block_offset, __m256i out[16]) { - out[0] = loadu_256(&inputs[0][block_offset + 0 * sizeof(__m256i)]); - out[1] = loadu_256(&inputs[1][block_offset + 0 * sizeof(__m256i)]); - out[2] = loadu_256(&inputs[2][block_offset + 0 * sizeof(__m256i)]); - out[3] = loadu_256(&inputs[3][block_offset + 0 * sizeof(__m256i)]); - out[4] = loadu_256(&inputs[4][block_offset + 0 * sizeof(__m256i)]); - out[5] = loadu_256(&inputs[5][block_offset + 0 * sizeof(__m256i)]); - out[6] = loadu_256(&inputs[6][block_offset + 0 * sizeof(__m256i)]); - out[7] = loadu_256(&inputs[7][block_offset + 0 * sizeof(__m256i)]); - out[8] = loadu_256(&inputs[0][block_offset + 1 * sizeof(__m256i)]); - out[9] = loadu_256(&inputs[1][block_offset + 1 * sizeof(__m256i)]); - out[10] = loadu_256(&inputs[2][block_offset + 1 * sizeof(__m256i)]); - out[11] = loadu_256(&inputs[3][block_offset + 1 * sizeof(__m256i)]); - out[12] = loadu_256(&inputs[4][block_offset + 1 * sizeof(__m256i)]); - out[13] = loadu_256(&inputs[5][block_offset + 1 * sizeof(__m256i)]); - out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]); - out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]); - for (size_t i = 0; i < 8; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_256(&out[0]); - transpose_vecs_256(&out[8]); -} - -INLINE void load_counters8(uint64_t counter, bool increment_counter, - __m256i *out_lo, __m256i *out_hi) { - uint64_t mask = (increment_counter ? ~0 : 0); - __m512i mask_vec = _mm512_set1_epi64(mask); - __m512i deltas = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); - deltas = _mm512_and_si512(mask_vec, deltas); - __m512i counters = - _mm512_add_epi64(_mm512_set1_epi64((int64_t)counter), deltas); - *out_lo = _mm512_cvtepi64_epi32(counters); - *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32)); -} - -static -void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m256i h_vecs[8] = { - set1_256(key[0]), set1_256(key[1]), set1_256(key[2]), set1_256(key[3]), - set1_256(key[4]), set1_256(key[5]), set1_256(key[6]), set1_256(key[7]), - }; - __m256i counter_low_vec, counter_high_vec; - load_counters8(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m256i block_len_vec = set1_256(BLAKE3_BLOCK_LEN); - __m256i block_flags_vec = set1_256(block_flags); - __m256i msg_vecs[16]; - transpose_msg_vecs8(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m256i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_256(IV[0]), set1_256(IV[1]), set1_256(IV[2]), set1_256(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn8(v, msg_vecs, 0); - round_fn8(v, msg_vecs, 1); - round_fn8(v, msg_vecs, 2); - round_fn8(v, msg_vecs, 3); - round_fn8(v, msg_vecs, 4); - round_fn8(v, msg_vecs, 5); - round_fn8(v, msg_vecs, 6); - h_vecs[0] = xor_256(v[0], v[8]); - h_vecs[1] = xor_256(v[1], v[9]); - h_vecs[2] = xor_256(v[2], v[10]); - h_vecs[3] = xor_256(v[3], v[11]); - h_vecs[4] = xor_256(v[4], v[12]); - h_vecs[5] = xor_256(v[5], v[13]); - h_vecs[6] = xor_256(v[6], v[14]); - h_vecs[7] = xor_256(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_256(h_vecs); - storeu_256(h_vecs[0], &out[0 * sizeof(__m256i)]); - storeu_256(h_vecs[1], &out[1 * sizeof(__m256i)]); - storeu_256(h_vecs[2], &out[2 * sizeof(__m256i)]); - storeu_256(h_vecs[3], &out[3 * sizeof(__m256i)]); - storeu_256(h_vecs[4], &out[4 * sizeof(__m256i)]); - storeu_256(h_vecs[5], &out[5 * sizeof(__m256i)]); - storeu_256(h_vecs[6], &out[6 * sizeof(__m256i)]); - storeu_256(h_vecs[7], &out[7 * sizeof(__m256i)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash16_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn16(__m512i v[16], __m512i m[16], size_t r) { - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_512(v[0], v[4]); - v[1] = add_512(v[1], v[5]); - v[2] = add_512(v[2], v[6]); - v[3] = add_512(v[3], v[7]); - v[12] = xor_512(v[12], v[0]); - v[13] = xor_512(v[13], v[1]); - v[14] = xor_512(v[14], v[2]); - v[15] = xor_512(v[15], v[3]); - v[12] = rot16_512(v[12]); - v[13] = rot16_512(v[13]); - v[14] = rot16_512(v[14]); - v[15] = rot16_512(v[15]); - v[8] = add_512(v[8], v[12]); - v[9] = add_512(v[9], v[13]); - v[10] = add_512(v[10], v[14]); - v[11] = add_512(v[11], v[15]); - v[4] = xor_512(v[4], v[8]); - v[5] = xor_512(v[5], v[9]); - v[6] = xor_512(v[6], v[10]); - v[7] = xor_512(v[7], v[11]); - v[4] = rot12_512(v[4]); - v[5] = rot12_512(v[5]); - v[6] = rot12_512(v[6]); - v[7] = rot12_512(v[7]); - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_512(v[0], v[4]); - v[1] = add_512(v[1], v[5]); - v[2] = add_512(v[2], v[6]); - v[3] = add_512(v[3], v[7]); - v[12] = xor_512(v[12], v[0]); - v[13] = xor_512(v[13], v[1]); - v[14] = xor_512(v[14], v[2]); - v[15] = xor_512(v[15], v[3]); - v[12] = rot8_512(v[12]); - v[13] = rot8_512(v[13]); - v[14] = rot8_512(v[14]); - v[15] = rot8_512(v[15]); - v[8] = add_512(v[8], v[12]); - v[9] = add_512(v[9], v[13]); - v[10] = add_512(v[10], v[14]); - v[11] = add_512(v[11], v[15]); - v[4] = xor_512(v[4], v[8]); - v[5] = xor_512(v[5], v[9]); - v[6] = xor_512(v[6], v[10]); - v[7] = xor_512(v[7], v[11]); - v[4] = rot7_512(v[4]); - v[5] = rot7_512(v[5]); - v[6] = rot7_512(v[6]); - v[7] = rot7_512(v[7]); - - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_512(v[0], v[5]); - v[1] = add_512(v[1], v[6]); - v[2] = add_512(v[2], v[7]); - v[3] = add_512(v[3], v[4]); - v[15] = xor_512(v[15], v[0]); - v[12] = xor_512(v[12], v[1]); - v[13] = xor_512(v[13], v[2]); - v[14] = xor_512(v[14], v[3]); - v[15] = rot16_512(v[15]); - v[12] = rot16_512(v[12]); - v[13] = rot16_512(v[13]); - v[14] = rot16_512(v[14]); - v[10] = add_512(v[10], v[15]); - v[11] = add_512(v[11], v[12]); - v[8] = add_512(v[8], v[13]); - v[9] = add_512(v[9], v[14]); - v[5] = xor_512(v[5], v[10]); - v[6] = xor_512(v[6], v[11]); - v[7] = xor_512(v[7], v[8]); - v[4] = xor_512(v[4], v[9]); - v[5] = rot12_512(v[5]); - v[6] = rot12_512(v[6]); - v[7] = rot12_512(v[7]); - v[4] = rot12_512(v[4]); - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_512(v[0], v[5]); - v[1] = add_512(v[1], v[6]); - v[2] = add_512(v[2], v[7]); - v[3] = add_512(v[3], v[4]); - v[15] = xor_512(v[15], v[0]); - v[12] = xor_512(v[12], v[1]); - v[13] = xor_512(v[13], v[2]); - v[14] = xor_512(v[14], v[3]); - v[15] = rot8_512(v[15]); - v[12] = rot8_512(v[12]); - v[13] = rot8_512(v[13]); - v[14] = rot8_512(v[14]); - v[10] = add_512(v[10], v[15]); - v[11] = add_512(v[11], v[12]); - v[8] = add_512(v[8], v[13]); - v[9] = add_512(v[9], v[14]); - v[5] = xor_512(v[5], v[10]); - v[6] = xor_512(v[6], v[11]); - v[7] = xor_512(v[7], v[8]); - v[4] = xor_512(v[4], v[9]); - v[5] = rot7_512(v[5]); - v[6] = rot7_512(v[6]); - v[7] = rot7_512(v[7]); - v[4] = rot7_512(v[4]); -} - -// 0b10001000, or lanes a0/a2/b0/b2 in little-endian order -#define LO_IMM8 0x88 - -INLINE __m512i unpack_lo_128(__m512i a, __m512i b) { - return _mm512_shuffle_i32x4(a, b, LO_IMM8); -} - -// 0b11011101, or lanes a1/a3/b1/b3 in little-endian order -#define HI_IMM8 0xdd - -INLINE __m512i unpack_hi_128(__m512i a, __m512i b) { - return _mm512_shuffle_i32x4(a, b, HI_IMM8); -} - -INLINE void transpose_vecs_512(__m512i vecs[16]) { - // Interleave 32-bit lanes. The _0 unpack is lanes - // 0/0/1/1/4/4/5/5/8/8/9/9/12/12/13/13, and the _2 unpack is lanes - // 2/2/3/3/6/6/7/7/10/10/11/11/14/14/15/15. - __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]); - __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]); - __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]); - __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]); - __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]); - __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]); - __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]); - __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]); - __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]); - __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]); - __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]); - __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]); - __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]); - __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]); - __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]); - __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]); - - // Interleave 64-bit lanes. The _0 unpack is lanes - // 0/0/0/0/4/4/4/4/8/8/8/8/12/12/12/12, the _1 unpack is lanes - // 1/1/1/1/5/5/5/5/9/9/9/9/13/13/13/13, the _2 unpack is lanes - // 2/2/2/2/6/6/6/6/10/10/10/10/14/14/14/14, and the _3 unpack is lanes - // 3/3/3/3/7/7/7/7/11/11/11/11/15/15/15/15. - __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0); - __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0); - __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2); - __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2); - __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0); - __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0); - __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2); - __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2); - __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0); - __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0); - __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2); - __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2); - __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0); - __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0); - __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2); - __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2); - - // Interleave 128-bit lanes. The _0 unpack is - // 0/0/0/0/8/8/8/8/0/0/0/0/8/8/8/8, the _1 unpack is - // 1/1/1/1/9/9/9/9/1/1/1/1/9/9/9/9, and so on. - __m512i abcdefgh_0 = unpack_lo_128(abcd_0, efgh_0); - __m512i abcdefgh_1 = unpack_lo_128(abcd_1, efgh_1); - __m512i abcdefgh_2 = unpack_lo_128(abcd_2, efgh_2); - __m512i abcdefgh_3 = unpack_lo_128(abcd_3, efgh_3); - __m512i abcdefgh_4 = unpack_hi_128(abcd_0, efgh_0); - __m512i abcdefgh_5 = unpack_hi_128(abcd_1, efgh_1); - __m512i abcdefgh_6 = unpack_hi_128(abcd_2, efgh_2); - __m512i abcdefgh_7 = unpack_hi_128(abcd_3, efgh_3); - __m512i ijklmnop_0 = unpack_lo_128(ijkl_0, mnop_0); - __m512i ijklmnop_1 = unpack_lo_128(ijkl_1, mnop_1); - __m512i ijklmnop_2 = unpack_lo_128(ijkl_2, mnop_2); - __m512i ijklmnop_3 = unpack_lo_128(ijkl_3, mnop_3); - __m512i ijklmnop_4 = unpack_hi_128(ijkl_0, mnop_0); - __m512i ijklmnop_5 = unpack_hi_128(ijkl_1, mnop_1); - __m512i ijklmnop_6 = unpack_hi_128(ijkl_2, mnop_2); - __m512i ijklmnop_7 = unpack_hi_128(ijkl_3, mnop_3); - - // Interleave 128-bit lanes again for the final outputs. - vecs[0] = unpack_lo_128(abcdefgh_0, ijklmnop_0); - vecs[1] = unpack_lo_128(abcdefgh_1, ijklmnop_1); - vecs[2] = unpack_lo_128(abcdefgh_2, ijklmnop_2); - vecs[3] = unpack_lo_128(abcdefgh_3, ijklmnop_3); - vecs[4] = unpack_lo_128(abcdefgh_4, ijklmnop_4); - vecs[5] = unpack_lo_128(abcdefgh_5, ijklmnop_5); - vecs[6] = unpack_lo_128(abcdefgh_6, ijklmnop_6); - vecs[7] = unpack_lo_128(abcdefgh_7, ijklmnop_7); - vecs[8] = unpack_hi_128(abcdefgh_0, ijklmnop_0); - vecs[9] = unpack_hi_128(abcdefgh_1, ijklmnop_1); - vecs[10] = unpack_hi_128(abcdefgh_2, ijklmnop_2); - vecs[11] = unpack_hi_128(abcdefgh_3, ijklmnop_3); - vecs[12] = unpack_hi_128(abcdefgh_4, ijklmnop_4); - vecs[13] = unpack_hi_128(abcdefgh_5, ijklmnop_5); - vecs[14] = unpack_hi_128(abcdefgh_6, ijklmnop_6); - vecs[15] = unpack_hi_128(abcdefgh_7, ijklmnop_7); -} - -INLINE void transpose_msg_vecs16(const uint8_t *const *inputs, - size_t block_offset, __m512i out[16]) { - out[0] = loadu_512(&inputs[0][block_offset]); - out[1] = loadu_512(&inputs[1][block_offset]); - out[2] = loadu_512(&inputs[2][block_offset]); - out[3] = loadu_512(&inputs[3][block_offset]); - out[4] = loadu_512(&inputs[4][block_offset]); - out[5] = loadu_512(&inputs[5][block_offset]); - out[6] = loadu_512(&inputs[6][block_offset]); - out[7] = loadu_512(&inputs[7][block_offset]); - out[8] = loadu_512(&inputs[8][block_offset]); - out[9] = loadu_512(&inputs[9][block_offset]); - out[10] = loadu_512(&inputs[10][block_offset]); - out[11] = loadu_512(&inputs[11][block_offset]); - out[12] = loadu_512(&inputs[12][block_offset]); - out[13] = loadu_512(&inputs[13][block_offset]); - out[14] = loadu_512(&inputs[14][block_offset]); - out[15] = loadu_512(&inputs[15][block_offset]); - for (size_t i = 0; i < 16; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_512(out); -} - -INLINE void load_counters16(uint64_t counter, bool increment_counter, - __m512i *out_lo, __m512i *out_hi) { - const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter); - const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - const __m512i masked_deltas = _mm512_and_si512(deltas, mask); - const __m512i low_words = _mm512_add_epi32( - _mm512_set1_epi32((int32_t)counter), - masked_deltas); - // The carry bit is 1 if the high bit of the word was 1 before addition and is - // 0 after. - // NOTE: It would be a bit more natural to use _mm512_cmp_epu32_mask to - // compute the carry bits here, and originally we did, but that intrinsic is - // broken under GCC 5.4. See https://github.com/BLAKE3-team/BLAKE3/issues/271. - const __m512i carries = _mm512_srli_epi32( - _mm512_andnot_si512( - low_words, // 0 after (gets inverted by andnot) - _mm512_set1_epi32((int32_t)counter)), // and 1 before - 31); - const __m512i high_words = _mm512_add_epi32( - _mm512_set1_epi32((int32_t)(counter >> 32)), - carries); - *out_lo = low_words; - *out_hi = high_words; -} - -static -void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, - uint8_t *out) { - __m512i h_vecs[8] = { - set1_512(key[0]), set1_512(key[1]), set1_512(key[2]), set1_512(key[3]), - set1_512(key[4]), set1_512(key[5]), set1_512(key[6]), set1_512(key[7]), - }; - __m512i counter_low_vec, counter_high_vec; - load_counters16(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m512i block_len_vec = set1_512(BLAKE3_BLOCK_LEN); - __m512i block_flags_vec = set1_512(block_flags); - __m512i msg_vecs[16]; - transpose_msg_vecs16(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m512i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_512(IV[0]), set1_512(IV[1]), set1_512(IV[2]), set1_512(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn16(v, msg_vecs, 0); - round_fn16(v, msg_vecs, 1); - round_fn16(v, msg_vecs, 2); - round_fn16(v, msg_vecs, 3); - round_fn16(v, msg_vecs, 4); - round_fn16(v, msg_vecs, 5); - round_fn16(v, msg_vecs, 6); - h_vecs[0] = xor_512(v[0], v[8]); - h_vecs[1] = xor_512(v[1], v[9]); - h_vecs[2] = xor_512(v[2], v[10]); - h_vecs[3] = xor_512(v[3], v[11]); - h_vecs[4] = xor_512(v[4], v[12]); - h_vecs[5] = xor_512(v[5], v[13]); - h_vecs[6] = xor_512(v[6], v[14]); - h_vecs[7] = xor_512(v[7], v[15]); - - block_flags = flags; - } - - // transpose_vecs_512 operates on a 16x16 matrix of words, but we only have 8 - // state vectors. Pad the matrix with zeros. After transposition, store the - // lower half of each vector. - __m512i padded[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_512(0), set1_512(0), set1_512(0), set1_512(0), - set1_512(0), set1_512(0), set1_512(0), set1_512(0), - }; - transpose_vecs_512(padded); - _mm256_mask_storeu_epi32(&out[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0])); - _mm256_mask_storeu_epi32(&out[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1])); - _mm256_mask_storeu_epi32(&out[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2])); - _mm256_mask_storeu_epi32(&out[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3])); - _mm256_mask_storeu_epi32(&out[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4])); - _mm256_mask_storeu_epi32(&out[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5])); - _mm256_mask_storeu_epi32(&out[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6])); - _mm256_mask_storeu_epi32(&out[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7])); - _mm256_mask_storeu_epi32(&out[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8])); - _mm256_mask_storeu_epi32(&out[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9])); - _mm256_mask_storeu_epi32(&out[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10])); - _mm256_mask_storeu_epi32(&out[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11])); - _mm256_mask_storeu_epi32(&out[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12])); - _mm256_mask_storeu_epi32(&out[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13])); - _mm256_mask_storeu_epi32(&out[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14])); - _mm256_mask_storeu_epi32(&out[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15])); -} - -/* - * ---------------------------------------------------------------------------- - * hash_many_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void hash_one_avx512(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= 16) { - blake3_hash16_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 16; - } - inputs += 16; - num_inputs -= 16; - out = &out[16 * BLAKE3_OUT_LEN]; - } - while (num_inputs >= 8) { - blake3_hash8_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 8; - } - inputs += 8; - num_inputs -= 8; - out = &out[8 * BLAKE3_OUT_LEN]; - } - while (num_inputs >= 4) { - blake3_hash4_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 4; - } - inputs += 4; - num_inputs -= 4; - out = &out[4 * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_impl.h b/src/third_party/blake3/c/blake3_impl.h deleted file mode 100644 index beab5cf53..000000000 --- a/src/third_party/blake3/c/blake3_impl.h +++ /dev/null @@ -1,285 +0,0 @@ -#ifndef BLAKE3_IMPL_H -#define BLAKE3_IMPL_H - -#include -#include -#include -#include -#include - -#include "blake3.h" - -// internal flags -enum blake3_flags { - CHUNK_START = 1 << 0, - CHUNK_END = 1 << 1, - PARENT = 1 << 2, - ROOT = 1 << 3, - KEYED_HASH = 1 << 4, - DERIVE_KEY_CONTEXT = 1 << 5, - DERIVE_KEY_MATERIAL = 1 << 6, -}; - -// This C implementation tries to support recent versions of GCC, Clang, and -// MSVC. -#if defined(_MSC_VER) -#define INLINE static __forceinline -#else -#define INLINE static inline __attribute__((always_inline)) -#endif - -#if defined(__x86_64__) || defined(_M_X64) -#define IS_X86 -#define IS_X86_64 -#endif - -#if defined(__i386__) || defined(_M_IX86) -#define IS_X86 -#define IS_X86_32 -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#define IS_AARCH64 -#endif - -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#endif -#endif - -#if !defined(BLAKE3_USE_NEON) - // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness - #if defined(IS_AARCH64) - #if defined(__ARM_BIG_ENDIAN) - #define BLAKE3_USE_NEON 0 - #else - #define BLAKE3_USE_NEON 1 - #endif - #else - #define BLAKE3_USE_NEON 0 - #endif -#endif - -#if defined(IS_X86) -#define MAX_SIMD_DEGREE 16 -#elif BLAKE3_USE_NEON == 1 -#define MAX_SIMD_DEGREE 4 -#else -#define MAX_SIMD_DEGREE 1 -#endif - -// There are some places where we want a static size that's equal to the -// MAX_SIMD_DEGREE, but also at least 2. -#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) - -static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, - 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, - 0x1F83D9ABUL, 0x5BE0CD19UL}; - -static const uint8_t MSG_SCHEDULE[7][16] = { - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, - {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, - {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, - {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, - {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, - {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, -}; - -/* Find index of the highest set bit */ -/* x is assumed to be nonzero. */ -static unsigned int highest_one(uint64_t x) { -#if defined(__GNUC__) || defined(__clang__) - return 63 ^ (unsigned int)__builtin_clzll(x); -#elif defined(_MSC_VER) && defined(IS_X86_64) - unsigned long index; - _BitScanReverse64(&index, x); - return index; -#elif defined(_MSC_VER) && defined(IS_X86_32) - if(x >> 32) { - unsigned long index; - _BitScanReverse(&index, (unsigned long)(x >> 32)); - return 32 + index; - } else { - unsigned long index; - _BitScanReverse(&index, (unsigned long)x); - return index; - } -#else - unsigned int c = 0; - if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } - if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } - if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } - if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } - if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } - if(x & 0x0000000000000002ULL) { c += 1; } - return c; -#endif -} - -// Count the number of 1 bits. -INLINE unsigned int popcnt(uint64_t x) { -#if defined(__GNUC__) || defined(__clang__) - return (unsigned int)__builtin_popcountll(x); -#else - unsigned int count = 0; - while (x != 0) { - count += 1; - x &= x - 1; - } - return count; -#endif -} - -// Largest power of two less than or equal to x. As a special case, returns 1 -// when x is 0. -INLINE uint64_t round_down_to_power_of_2(uint64_t x) { - return 1ULL << highest_one(x | 1); -} - -INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } - -INLINE uint32_t counter_high(uint64_t counter) { - return (uint32_t)(counter >> 32); -} - -INLINE uint32_t load32(const void *src) { - const uint8_t *p = (const uint8_t *)src; - return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | - ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); -} - -INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], - uint32_t key_words[8]) { - key_words[0] = load32(&key[0 * 4]); - key_words[1] = load32(&key[1 * 4]); - key_words[2] = load32(&key[2 * 4]); - key_words[3] = load32(&key[3 * 4]); - key_words[4] = load32(&key[4 * 4]); - key_words[5] = load32(&key[5 * 4]); - key_words[6] = load32(&key[6 * 4]); - key_words[7] = load32(&key[7 * 4]); -} - -INLINE void store32(void *dst, uint32_t w) { - uint8_t *p = (uint8_t *)dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); -} - -INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { - store32(&bytes_out[0 * 4], cv_words[0]); - store32(&bytes_out[1 * 4], cv_words[1]); - store32(&bytes_out[2 * 4], cv_words[2]); - store32(&bytes_out[3 * 4], cv_words[3]); - store32(&bytes_out[4 * 4], cv_words[4]); - store32(&bytes_out[5 * 4], cv_words[5]); - store32(&bytes_out[6 * 4], cv_words[6]); - store32(&bytes_out[7 * 4], cv_words[7]); -} - -void blake3_compress_in_place(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags, - uint8_t out[64]); - -void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out); - -size_t blake3_simd_degree(void); - - -// Declarations for implementation-specific functions. -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof_portable(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); - -void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); - -#if defined(IS_X86) -#if !defined(BLAKE3_NO_SSE2) -void blake3_compress_in_place_sse2(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); -void blake3_compress_xof_sse2(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); -void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_SSE41) -void blake3_compress_in_place_sse41(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); -void blake3_compress_xof_sse41(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); -void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_AVX2) -void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_AVX512) -void blake3_compress_in_place_avx512(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof_avx512(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); - -void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#endif - -#if BLAKE3_USE_NEON == 1 -void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif - - -#endif /* BLAKE3_IMPL_H */ diff --git a/src/third_party/blake3/c/blake3_neon.c b/src/third_party/blake3/c/blake3_neon.c deleted file mode 100644 index 8a818fc78..000000000 --- a/src/third_party/blake3/c/blake3_neon.c +++ /dev/null @@ -1,368 +0,0 @@ -#include "blake3_impl.h" - -#include - -#ifdef __ARM_BIG_ENDIAN -#error "This implementation only supports little-endian ARM." -// It might be that all we need for big-endian support here is to get the loads -// and stores right, but step zero would be finding a way to test it in CI. -#endif - -INLINE uint32x4_t loadu_128(const uint8_t src[16]) { - // vld1q_u32 has alignment requirements. Don't use it. - uint32x4_t x; - memcpy(&x, src, 16); - return x; -} - -INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { - // vst1q_u32 has alignment requirements. Don't use it. - memcpy(dest, &src, 16); -} - -INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { - return vaddq_u32(a, b); -} - -INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) { - return veorq_u32(a, b); -} - -INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); } - -INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - uint32_t array[4] = {a, b, c, d}; - return vld1q_u32(array); -} - -INLINE uint32x4_t rot16_128(uint32x4_t x) { - // The straightfoward implementation would be two shifts and an or, but that's - // slower on microarchitectures we've tested. See - // https://github.com/BLAKE3-team/BLAKE3/pull/319. - // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); - return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))); -} - -INLINE uint32x4_t rot12_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12)); - return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12); -} - -INLINE uint32x4_t rot8_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8)); -#if defined(__clang__) - return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12)); -#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700 - static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12}; - return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8)); -#else - return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8); -#endif -} - -INLINE uint32x4_t rot7_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7)); - return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7); -} - -// TODO: compress_neon - -// TODO: hash2_neon - -/* - * ---------------------------------------------------------------------------- - * hash4_neon - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) { - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[15] = rot16_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot12_128(v[4]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[15] = rot8_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot7_128(v[4]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot16_128(v[15]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[4] = rot12_128(v[4]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot8_128(v[15]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - v[4] = rot7_128(v[4]); -} - -INLINE void transpose_vecs_128(uint32x4_t vecs[4]) { - // Individually transpose the four 2x2 sub-matrices in each corner. - uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]); - uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]); - - // Swap the top-right and bottom-left 2x2s (which just got transposed). - vecs[0] = - vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0])); - vecs[1] = - vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1])); - vecs[2] = - vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0])); - vecs[3] = - vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1])); -} - -INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, - size_t block_offset, uint32x4_t out[16]) { - out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]); - out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]); - out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]); - out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]); - out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]); - out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]); - out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]); - out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]); - out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]); - out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]); - out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]); - out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]); - out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]); - out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]); - out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]); - out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]); - transpose_vecs_128(&out[0]); - transpose_vecs_128(&out[4]); - transpose_vecs_128(&out[8]); - transpose_vecs_128(&out[12]); -} - -INLINE void load_counters4(uint64_t counter, bool increment_counter, - uint32x4_t *out_low, uint32x4_t *out_high) { - uint64_t mask = (increment_counter ? ~0 : 0); - *out_low = set4( - counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)), - counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3))); - *out_high = set4( - counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)), - counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); -} - -void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - uint32x4_t h_vecs[8] = { - set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), - set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), - }; - uint32x4_t counter_low_vec, counter_high_vec; - load_counters4(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN); - uint32x4_t block_flags_vec = set1_128(block_flags); - uint32x4_t msg_vecs[16]; - transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - uint32x4_t v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn4(v, msg_vecs, 0); - round_fn4(v, msg_vecs, 1); - round_fn4(v, msg_vecs, 2); - round_fn4(v, msg_vecs, 3); - round_fn4(v, msg_vecs, 4); - round_fn4(v, msg_vecs, 5); - round_fn4(v, msg_vecs, 6); - h_vecs[0] = xor_128(v[0], v[8]); - h_vecs[1] = xor_128(v[1], v[9]); - h_vecs[2] = xor_128(v[2], v[10]); - h_vecs[3] = xor_128(v[3], v[11]); - h_vecs[4] = xor_128(v[4], v[12]); - h_vecs[5] = xor_128(v[5], v[13]); - h_vecs[6] = xor_128(v[6], v[14]); - h_vecs[7] = xor_128(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_128(&h_vecs[0]); - transpose_vecs_128(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash_many_neon - * ---------------------------------------------------------------------------- - */ - -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -INLINE void hash_one_neon(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, uint8_t flags_end, - uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - // TODO: Implement compress_neon. However note that according to - // https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227, - // compress_neon might not be any faster than compress_portable. - blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= 4) { - blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 4; - } - inputs += 4; - num_inputs -= 4; - out = &out[4 * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_portable.c b/src/third_party/blake3/c/blake3_portable.c deleted file mode 100644 index 062dd1b47..000000000 --- a/src/third_party/blake3/c/blake3_portable.c +++ /dev/null @@ -1,160 +0,0 @@ -#include "blake3_impl.h" -#include - -INLINE uint32_t rotr32(uint32_t w, uint32_t c) { - return (w >> c) | (w << (32 - c)); -} - -INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, - uint32_t x, uint32_t y) { - state[a] = state[a] + state[b] + x; - state[d] = rotr32(state[d] ^ state[a], 16); - state[c] = state[c] + state[d]; - state[b] = rotr32(state[b] ^ state[c], 12); - state[a] = state[a] + state[b] + y; - state[d] = rotr32(state[d] ^ state[a], 8); - state[c] = state[c] + state[d]; - state[b] = rotr32(state[b] ^ state[c], 7); -} - -INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { - // Select the message schedule based on the round. - const uint8_t *schedule = MSG_SCHEDULE[round]; - - // Mix the columns. - g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); - g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); - g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); - g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); - - // Mix the rows. - g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); - g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); - g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); - g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); -} - -INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - uint32_t block_words[16]; - block_words[0] = load32(block + 4 * 0); - block_words[1] = load32(block + 4 * 1); - block_words[2] = load32(block + 4 * 2); - block_words[3] = load32(block + 4 * 3); - block_words[4] = load32(block + 4 * 4); - block_words[5] = load32(block + 4 * 5); - block_words[6] = load32(block + 4 * 6); - block_words[7] = load32(block + 4 * 7); - block_words[8] = load32(block + 4 * 8); - block_words[9] = load32(block + 4 * 9); - block_words[10] = load32(block + 4 * 10); - block_words[11] = load32(block + 4 * 11); - block_words[12] = load32(block + 4 * 12); - block_words[13] = load32(block + 4 * 13); - block_words[14] = load32(block + 4 * 14); - block_words[15] = load32(block + 4 * 15); - - state[0] = cv[0]; - state[1] = cv[1]; - state[2] = cv[2]; - state[3] = cv[3]; - state[4] = cv[4]; - state[5] = cv[5]; - state[6] = cv[6]; - state[7] = cv[7]; - state[8] = IV[0]; - state[9] = IV[1]; - state[10] = IV[2]; - state[11] = IV[3]; - state[12] = counter_low(counter); - state[13] = counter_high(counter); - state[14] = (uint32_t)block_len; - state[15] = (uint32_t)flags; - - round_fn(state, &block_words[0], 0); - round_fn(state, &block_words[0], 1); - round_fn(state, &block_words[0], 2); - round_fn(state, &block_words[0], 3); - round_fn(state, &block_words[0], 4); - round_fn(state, &block_words[0], 5); - round_fn(state, &block_words[0], 6); -} - -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - uint32_t state[16]; - compress_pre(state, cv, block, block_len, counter, flags); - cv[0] = state[0] ^ state[8]; - cv[1] = state[1] ^ state[9]; - cv[2] = state[2] ^ state[10]; - cv[3] = state[3] ^ state[11]; - cv[4] = state[4] ^ state[12]; - cv[5] = state[5] ^ state[13]; - cv[6] = state[6] ^ state[14]; - cv[7] = state[7] ^ state[15]; -} - -void blake3_compress_xof_portable(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - uint32_t state[16]; - compress_pre(state, cv, block, block_len, counter, flags); - - store32(&out[0 * 4], state[0] ^ state[8]); - store32(&out[1 * 4], state[1] ^ state[9]); - store32(&out[2 * 4], state[2] ^ state[10]); - store32(&out[3 * 4], state[3] ^ state[11]); - store32(&out[4 * 4], state[4] ^ state[12]); - store32(&out[5 * 4], state[5] ^ state[13]); - store32(&out[6 * 4], state[6] ^ state[14]); - store32(&out[7 * 4], state[7] ^ state[15]); - store32(&out[8 * 4], state[8] ^ cv[0]); - store32(&out[9 * 4], state[9] ^ cv[1]); - store32(&out[10 * 4], state[10] ^ cv[2]); - store32(&out[11 * 4], state[11] ^ cv[3]); - store32(&out[12 * 4], state[12] ^ cv[4]); - store32(&out[13 * 4], state[13] ^ cv[5]); - store32(&out[14 * 4], state[14] ^ cv[6]); - store32(&out[15 * 4], state[15] ^ cv[7]); -} - -INLINE void hash_one_portable(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - store_cv_words(out, cv); -} - -void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs > 0) { - hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_sse2.c b/src/third_party/blake3/c/blake3_sse2.c deleted file mode 100644 index 86bb17aba..000000000 --- a/src/third_party/blake3/c/blake3_sse2.c +++ /dev/null @@ -1,566 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 4 - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE void storeu(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16(__m128i x) { - return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1); -} - -INLINE __m128i rot12(__m128i x) { - return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); -} - -INLINE __m128i rot8(__m128i x) { - return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8)); -} - -INLINE __m128i rot7(__m128i x) { - return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); -} - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot16(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot12(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot8(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot7(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) { - const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01); - __m128i mask = _mm_set1_epi16(imm8); - mask = _mm_and_si128(mask, bits); - mask = _mm_cmpeq_epi16(mask, bits); - return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu((uint8_t *)&cv[0]); - rows[1] = loadu((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_in_place_sse2(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -void blake3_compress_xof_sse2(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), &out[0]); - storeu(xorv(rows[1], rows[3]), &out[16]); - storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); - storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); -} - -INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m128i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[4]); - transpose_vecs(&out[8]); - transpose_vecs(&out[12]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); - const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); - const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); - __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), - _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(&h_vecs[0]); - transpose_vecs(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -INLINE void hash_one_sse2(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_sse41.c b/src/third_party/blake3/c/blake3_sse41.c deleted file mode 100644 index b28ae13b9..000000000 --- a/src/third_party/blake3/c/blake3_sse41.c +++ /dev/null @@ -1,560 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 4 - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE void storeu(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16(__m128i x) { - return _mm_shuffle_epi8( - x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); -} - -INLINE __m128i rot12(__m128i x) { - return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); -} - -INLINE __m128i rot8(__m128i x) { - return _mm_shuffle_epi8( - x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); -} - -INLINE __m128i rot7(__m128i x) { - return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); -} - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot16(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot12(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot8(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot7(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu((uint8_t *)&cv[0]); - rows[1] = loadu((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_in_place_sse41(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -void blake3_compress_xof_sse41(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), &out[0]); - storeu(xorv(rows[1], rows[3]), &out[16]); - storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); - storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); -} - -INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m128i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[4]); - transpose_vecs(&out[8]); - transpose_vecs(&out[12]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); - const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); - const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); - __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), - _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(&h_vecs[0]); - transpose_vecs(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -INLINE void hash_one_sse41(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/torture/tests/torture_coff_tests.c b/src/torture/tests/torture_coff_tests.c new file mode 100644 index 000000000..8e2c2f006 --- /dev/null +++ b/src/torture/tests/torture_coff_tests.c @@ -0,0 +1,519 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal String8 +t_coff_test_encode(Arena *arena, TestCtx *ctx, String8 file_name, String8 source, T_Result *result_out) +{ + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, file_name, source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + *result_out = result; + return value != 0 ? value->data : str8_zero(); +} + +TEST(coff_writer_bigobj) +{ + // Exercise the standard-COFF boundary and an associative parent above 16 bits. + U32 section_counts[] = {3, 0xfeff, 0xff00, 0x10002}; + for EachElement(case_idx, section_counts) { + U32 section_count = section_counts[case_idx]; + COFF_ObjWriter *writer = coff_obj_writer_alloc(0x12345678, COFF_MachineType_X64); + for (U32 i = 0; i < section_count - 2; i += 1) { + coff_obj_writer_push_section(writer, str8_lit(".empty"), COFF_SectionFlag_LnkRemove, str8_zero()); + } + COFF_SectionFlags flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_LnkCOMDAT; + COFF_ObjSection *head = coff_obj_writer_push_section(writer, str8_lit(".long_head_section"), flags, str8_lit("head")); + COFF_ObjSection *assoc = coff_obj_writer_push_section(writer, str8_lit(".assoc"), flags, str8(push_array(arena, U8, 8), 8)); + coff_obj_writer_push_symbol_secdef(writer, head, COFF_ComdatSelect_Any); + COFF_ObjSymbol *target = coff_obj_writer_push_symbol_extern(writer, str8_lit("long_target_symbol"), 0, head); + COFF_ObjSymbol *weak = coff_obj_writer_push_symbol_weak(writer, str8_lit("weak"), COFF_WeakExt_SearchAlias, target); + COFF_ObjSymbol *absolute = coff_obj_writer_push_symbol_abs(writer, str8_lit("absolute"), 17, COFF_SymStorageClass_External); + COFF_ObjSymbol *assoc_def = coff_obj_writer_push_symbol_associative(writer, assoc, head); + COFF_ObjSymbol *undef = coff_obj_writer_push_symbol_undef(writer, str8_lit("undef")); + COFF_ObjSymbol *common = coff_obj_writer_push_symbol_common(writer, str8_lit("common"), 32); + coff_obj_writer_section_push_reloc_addr(writer, assoc, 0, weak); + + String8 data = coff_obj_writer_serialize(arena, writer); + COFF_FileHeaderInfo info = coff_file_header_info_from_data(data); + T_Ok(info.is_big_obj == (section_count > 0xfeff)); + T_Ok(info.section_count_no_null == section_count); + T_Ok(info.symbol_size == (info.is_big_obj ? sizeof(COFF_Symbol32) : sizeof(COFF_Symbol16))); + T_Ok(info.symbol_count == 10); + String8 symbols = str8_substr(data, info.symbol_table_range); + String8 strings = str8_substr(data, info.string_table_range); + COFF_ParsedSymbol parsed_target = coff_parse_symbol(info, strings, symbols, target->idx); + T_Ok(parsed_target.section_number == section_count - 1); + T_Ok(str8_match(parsed_target.name, target->name, 0)); + COFF_ParsedSymbol parsed_assoc = coff_parse_symbol(info, strings, symbols, assoc_def->idx); + U32 parent = 0; + COFF_ComdatSelectType selection = 0; + coff_parse_secdef(parsed_assoc, info.is_big_obj, &selection, &parent, 0, 0); + T_Ok(parsed_assoc.section_number == section_count); + T_Ok(selection == COFF_ComdatSelect_Associative && parent == section_count - 1); + COFF_ParsedSymbol parsed_weak = coff_parse_symbol(info, strings, symbols, weak->idx); + COFF_SymbolWeakExt *weak_aux = coff_parse_weak_tag(parsed_weak, info.is_big_obj); + T_Ok(weak_aux->tag_index == target->idx && weak_aux->characteristics == COFF_WeakExt_SearchAlias); + COFF_ParsedSymbol parsed_absolute = coff_parse_symbol(info, strings, symbols, absolute->idx); + T_Ok(parsed_absolute.section_number == COFF_Symbol_AbsSection32 && parsed_absolute.value == 17); + T_Ok(coff_parse_symbol(info, strings, symbols, undef->idx).section_number == COFF_Symbol_UndefinedSection); + COFF_ParsedSymbol parsed_common = coff_parse_symbol(info, strings, symbols, common->idx); + T_Ok(parsed_common.section_number == COFF_Symbol_UndefinedSection && parsed_common.value == 32); + COFF_SectionHeader *sections = (COFF_SectionHeader *)(data.str + info.section_table_range.min); + COFF_Reloc *reloc = (COFF_Reloc *)(data.str + sections[section_count - 1].relocs_foff); + T_Ok(sections[section_count - 1].reloc_count == 1 && reloc->isymbol == weak->idx); + T_Ok(t_write_file(str8f(arena, "writer_%u.obj", section_count), data)); + coff_obj_writer_release(&writer); + } +} + +TEST(coff_codec_object_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { object: { machine: x64, sections: { text: { name: \".text\", permissions: (read, execute), content: code, " + "data: { hex: \"00000000c3\" }, relocations: { ref: { type: Addr32Nb, offset: 0, symbol: undef } } }, head: { name: \".head\", permissions: (read), content: " + "initialized_data, " + "flags: (link_comdat), data: { text: \"head\" } }, assoc: { name: \".assoc\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"assoc\" " + "} } }, symbols: { " + "fallback: { kind: absolute, name: \"fallback\", value: 17, storage: external }, entry: { kind: external, name: \"entry\", section: text, value: 0 }, func: { kind: " + "external_function, " + "name: \"func\", section: text, value: 1 }, local: { kind: static, name: \"local\", section: text, value: 2 }, head_def: { kind: section_definition, section: head, " + "selection: Any }, " + "assoc_def: { kind: section_definition, section: assoc, selection: Associative, associate: head }, weak: { kind: weak, name: \"weak\", fallback: fallback, search: alias }, " + "abs_static: { kind: absolute, " + "name: \"abs_static\", value: 34, storage: static }, undef: { kind: undefined, name: \"undef\" }, undef_func: { kind: undefined_function, name: \"undef_func\" }, " + "undef_section: { kind: undefined_section, " + "name: \"undef_section\", value: 51 }, section_symbol: { kind: section, name: \"section_symbol\", section: text }, common: { kind: common, name: \"common\", size: 64 } }, " + "directives: { directive: \"/export:entry\" } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("object_parity.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + String8 expected = + t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\0\0\0\0\xc3"), .flags = "rx:code", + .relocs = (T_COFF_DefReloc[]){T_COFF_DefReloc(X64_Addr32Nb, 0, "undef"), {0}}}, + {"head", ".head", str8_lit_comp("head"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"assoc", ".assoc", str8_lit_comp("assoc"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_AbsExtern("fallback", 17), + T_COFF_DefSymbol_Extern("entry", "text", 0), + T_COFF_DefSymbol_ExternFunc("func", "text", 1), + T_COFF_DefSymbol_Static("local", "text", 2), + T_COFF_DefSymbol_Secdef("head", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Associative("assoc", "head"), + T_COFF_DefSymbol_Weak("weak", COFF_WeakExt_SearchAlias, "fallback"), + T_COFF_DefSymbol_AbsStatic("abs_static", 34), + T_COFF_DefSymbol_Undef("undef"), + T_COFF_DefSymbol_UndefFunc("undef_func"), + T_COFF_DefSymbol_UndefSec("undef_section", 51), + T_COFF_DefSymbol_Sect("section_symbol", "text"), + T_COFF_DefSymbol_Common("common", 64), + {0}}, + .directives = (char *[]){"/export:entry", 0}, + }); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(value != 0 && str8_match(value->data, expected, 0)); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("legacy")))); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("script")))); + T_Ok(t_write_file(str8_lit("legacy/value.obj"), expected)); + T_Ok(t_write_file(str8_lit("script/value.obj"), value->data)); + T_Ok(t_match_folders(t_make_file_path(arena, str8_lit("legacy")), t_make_file_path(arena, str8_lit("script")))); + MD_Node *semantic = 0; + result = value->codec->decode(&script, value, &semantic); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + if (semantic != 0) { + MD_Node *object = t_child_from_string(semantic, "object"); + MD_Node *sections = t_child_from_string(object, "sections"); + MD_Node *symbols = t_child_from_string(object, "symbols"); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(object, "machine")), str8_lit("Amd64"), 0)); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(t_child_from_string(sections, "section_1"), "data")), str8_lit("00000000c3"), 0)); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(t_child_from_string(symbols, "symbol_0"), "name")), str8_lit("fallback"), 0)); + } +} + +TEST(coff_codec_library_parity) +{ + String8 source = str8_lit("test: { artifacts: { value: { coff: { library: { timestamp: 0, mode: 0, second_linker_member: true, members: { implementation: " + "{ path: \"member.obj\", object: { machine: x64, " + "sections: { text: { name: \".text\", permissions: (read, execute), content: code, data: { hex: \"c3\" } } }, symbols: { func: { kind: " + "external_function, name: \"func\", section: text, value: 0 } } } }, " + "imported: { import: { dll: \"foo.dll\", name: \"foo\", machine: x64, timestamp: 4294967295, type: code, lookup: name, hint: 0 } }, scaffold: { " + "dll_import: { name: \"bar.dll\", machine: x64, timestamp: 0 } } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("library_parity.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + String8 expected = t_coff_from_def_lib( + arena, (T_COFF_DefLib){ + .emit_second_member = 1, + .members = (T_COFF_DefLibMember[]){{.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), + .path = str8_lit_comp("member.obj"), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("func", "text", 0), {0}}}}, + {.type = T_COFF_DefLibMember_Import, + .import = {.dll = "foo.dll", + .name = "foo", + .import_by = COFF_ImportBy_Name, + .type = COFF_ImportHeader_Code, + .hit_or_ordinal = 0, + .time_stamp = T_COFF_DefSetTimeStamp(~0u), + .machine = T_COFF_DefSetMachine(X64)}}, + {.type = T_COFF_DefLibMember_DllImportStatic, .dll_import = {.name = "bar.dll", .machine = T_COFF_DefSetMachine(X64)}}, + {0}}, + }); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(value != 0 && str8_match(value->data, expected, 0)); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("legacy")))); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("script")))); + T_Ok(t_write_file(str8_lit("legacy/value.lib"), expected)); + T_Ok(t_write_file(str8_lit("script/value.lib"), value->data)); + T_Ok(t_match_folders(t_make_file_path(arena, str8_lit("legacy")), t_make_file_path(arena, str8_lit("script")))); + MD_Node *semantic = 0; + result = value->codec->decode(&script, value, &semantic); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + if (semantic != 0) { + MD_Node *library = t_child_from_string(semantic, "library"); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(library, "member_count")), str8_lit("5"), 0)); + T_Ok(!md_node_is_nil(t_child_from_string(t_child_from_string(t_child_from_string(library, "members"), "member_1"), "import"))); + } +} + +TEST(coff_codec_object_layout_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { object: { machine: x64, timestamp: 305419896, sections: { " + "write: { name: \".write\", permissions: (read, write), content: initialized_data, alignment: 32, data: { text: \"write\" } }, " + "bss: { name: \".bss\", permissions: (read, write), content: uninitialized_data, data: { zero: 4 } }, " + "long: { name: \".very_long_section_name\", permissions: (read), content: initialized_data, raw_flags: 8, data: { text: \"long\" } }, " + "empty: { name: \".empty\", permissions: (read), content: initialized_data, data: { zero: 0 } } }, " + "directives: { directive: \"/defaultlib:first\", directive: \"/include:middle\", directive: \"/export:last\" } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("object_layout_parity.tst"), source, &result); + String8 expected = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .time_stamp = T_COFF_DefSetTimeStamp(305419896), + .sections = (T_COFF_DefSection[]){ + {"write", ".write", str8_lit_comp("write"), .flags = "rw:data@32"}, + {"bss", ".bss", str8_lit_comp("\0\0\0\0"), .flags = "rw:bss"}, + {"long", ".very_long_section_name", str8_lit_comp("long"), .flags = "r:data", .raw_flags = COFF_SectionFlag_TypeNoPad}, + {"empty", ".empty", str8_zero(), .flags = "r:data"}, + {0}}, + .directives = (char *[]){"/defaultlib:first", "/include:middle", "/export:last", 0}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_comdat_weak_aux_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { object: { machine: x64, sections: { " + "null: { name: \".null\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"0\" } }, " + "nodup: { name: \".nodup\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"1\" } }, " + "any: { name: \".any\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"2\" } }, " + "same: { name: \".same\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"3\" } }, " + "exact: { name: \".exact\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"4\" } }, " + "assoc: { name: \".assoc\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"5\" } }, " + "largest: { name: \".largest\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"6\" } }, " + "relocs: { name: \".relocs\", permissions: (read), content: initialized_data, data: { zero: 4 }, relocations: { " + "late: { type: Addr32Nb, offset: 0, symbol: late } } } }, symbols: { " + "fallback: { kind: absolute, name: \"fallback\", value: 1, storage: external }, " + "null_def: { kind: section_definition, section: null, selection: Null }, " + "nodup_def: { kind: section_definition, section: nodup, selection: NoDuplicates }, " + "any_def: { kind: section_definition, section: any, selection: Any }, " + "same_def: { kind: section_definition, section: same, selection: SameSize }, " + "exact_def: { kind: section_definition, section: exact, selection: ExactMatch }, " + "assoc_def: { kind: section_definition, section: assoc, selection: Associative, associate: any }, " + "largest_def: { kind: section_definition, section: largest, selection: Largest }, " + "weak_no_library: { kind: weak, name: \"weak_no_library\", fallback: fallback, search: no_library }, " + "weak_search_library: { kind: weak, name: \"weak_search_library\", fallback: fallback, search: search_library }, " + "weak_alias: { kind: weak, name: \"weak_alias\", fallback: fallback, search: alias }, " + "weak_anti_dependency: { kind: weak, name: \"weak_anti_dependency\", fallback: fallback, search: anti_dependency }, " + "late: { kind: undefined, name: \"relocation_target_after_aux_records\" } } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("comdat_weak_aux_parity.tst"), source, &result); + String8 expected = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + {"null", ".null", str8_lit_comp("0"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"nodup", ".nodup", str8_lit_comp("1"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"any", ".any", str8_lit_comp("2"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"same", ".same", str8_lit_comp("3"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"exact", ".exact", str8_lit_comp("4"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"assoc", ".assoc", str8_lit_comp("5"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"largest", ".largest", str8_lit_comp("6"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"relocs", ".relocs", str8_lit_comp("\0\0\0\0"), .flags = "r:data", + .relocs = (T_COFF_DefReloc[]){T_COFF_DefReloc(X64_Addr32Nb, 0, "relocation_target_after_aux_records"), {0}}}, + {0}}, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_AbsExtern("fallback", 1), + T_COFF_DefSymbol_Secdef("null", COFF_ComdatSelect_Null), + T_COFF_DefSymbol_Secdef("nodup", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("any", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("same", COFF_ComdatSelect_SameSize), + T_COFF_DefSymbol_Secdef("exact", COFF_ComdatSelect_ExactMatch), + T_COFF_DefSymbol_Associative("assoc", "any"), + T_COFF_DefSymbol_Secdef("largest", COFF_ComdatSelect_Largest), + T_COFF_DefSymbol_Weak("weak_no_library", COFF_WeakExt_NoLibrary, "fallback"), + T_COFF_DefSymbol_Weak("weak_search_library", COFF_WeakExt_SearchLibrary, "fallback"), + T_COFF_DefSymbol_Weak("weak_alias", COFF_WeakExt_SearchAlias, "fallback"), + T_COFF_DefSymbol_Weak("weak_anti_dependency", COFF_WeakExt_AntiDependency, "fallback"), + T_COFF_DefSymbol_Undef("relocation_target_after_aux_records"), + {0}}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_relocation_variants_parity) +{ + struct RelocationCase + { + String8 machine_name; + COFF_MachineType machine; + } cases[] = { + {str8_lit_comp("x64"), COFF_MachineType_X64}, + {str8_lit_comp("x86"), COFF_MachineType_X86}, + {str8_lit_comp("arm"), COFF_MachineType_Arm}, + {str8_lit_comp("arm64"), COFF_MachineType_Arm64}, + }; + for EachElement(case_idx, cases) + { + U64 relocation_count = 0; + for EachIndex(type, 256) { relocation_count += coff_string_from_reloc(cases[case_idx].machine, (COFF_RelocType)type).size != 0; } + COFF_RelocType *types = push_array(arena, COFF_RelocType, relocation_count); + T_COFF_DefReloc *relocations = push_array(arena, T_COFF_DefReloc, relocation_count + 1); + String8List source = {0}; + str8_list_pushf(arena, &source, + "test: { artifacts: { value: { coff: { object: { machine: %S, sections: { data: { name: \".data\", " + "permissions: (read), content: initialized_data, data: { zero: %llu }, relocations: {", + cases[case_idx].machine_name, relocation_count); + U64 relocation_idx = 0; + for EachIndex(type, 256) + { + String8 type_name = coff_string_from_reloc(cases[case_idx].machine, (COFF_RelocType)type); + if (type_name.size != 0) + { + types[relocation_idx] = (COFF_RelocType)type; + relocations[relocation_idx].type = &types[relocation_idx]; + relocations[relocation_idx].apply_off = relocation_idx; + relocations[relocation_idx].symbol = "target"; + str8_list_pushf(arena, &source, " r%llu: { type: %S, offset: %llu, symbol: target }", relocation_idx, type_name, relocation_idx); + relocation_idx += 1; + } + } + str8_list_pushf(arena, &source, " } } }, symbols: { target: { kind: undefined, name: \"target\" } } } } } } }"); + String8 file_name = push_str8f(arena, "relocation_variants_%S.tst", cases[case_idx].machine_name); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, file_name, str8_list_join(arena, &source, 0), &result); + String8 expected = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = &cases[case_idx].machine, + .sections = (T_COFF_DefSection[]){{"data", ".data", str8(push_array(arena, U8, relocation_count), relocation_count), .flags = "r:data", .relocs = relocations}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_Undef("target"), {0}}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); + } +} + +TEST(coff_codec_library_object_paths_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { library: { timestamp: 0, mode: 420, second_linker_member: false, members: { " + "short: { path: \"a.obj\", object: { machine: x64, sections: { text: { name: \".text\", permissions: (read, execute), content: code, data: { hex: \"c3\" } } }, " + "symbols: { a: { kind: external_function, name: \"a\", section: text, value: 0 } } } }, " + "middle: { path: \"middle.obj\", object: { machine: x64, sections: { data: { name: \".data\", permissions: (read, write), content: initialized_data, data: { text: \"middle\" } } }, " + "symbols: { middle: { kind: external, name: \"middle\", section: data, value: 0 } } } }, " + "long: { path: \"objects/this_is_a_long_member_name.obj\", object: { machine: x64, sections: { text: { name: \".text\", permissions: (read, execute), content: code, data: { hex: \"c3\" } } }, " + "symbols: { z: { kind: external_function, name: \"z\", section: text, value: 0 } } } } } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("library_object_paths_parity.tst"), source, &result); + String8 expected = t_coff_from_def_lib(arena, (T_COFF_DefLib){ + .mode = 420, + .emit_second_member = 0, + .members = (T_COFF_DefLibMember[]){ + {.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), .path = str8_lit_comp("a.obj"), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("a", "text", 0), {0}}}}, + {.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), .path = str8_lit_comp("middle.obj"), + .sections = (T_COFF_DefSection[]){{"data", ".data", str8_lit_comp("middle"), .flags = "rw:data"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_Extern("middle", "data", 0), {0}}}}, + {.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), .path = str8_lit_comp("objects/this_is_a_long_member_name.obj"), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("z", "text", 0), {0}}}}, + {0}}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_import_variants_parity) +{ + struct ImportTypeCase + { + String8 name; + COFF_ImportType type; + } type_cases[] = { + {str8_lit_comp("code"), COFF_ImportHeader_Code}, + {str8_lit_comp("data"), COFF_ImportHeader_Data}, + {str8_lit_comp("const"), COFF_ImportHeader_Const}, + }; + struct ImportByCase + { + String8 name; + COFF_ImportByType type; + } import_by_cases[] = { + {str8_lit_comp("ordinal"), COFF_ImportBy_Ordinal}, + {str8_lit_comp("name"), COFF_ImportBy_Name}, + {str8_lit_comp("name_no_prefix"), COFF_ImportBy_NameNoPrefix}, + {str8_lit_comp("undecorate"), COFF_ImportBy_Undecorate}, + }; + struct ImportMachineCase + { + String8 name; + COFF_MachineType type; + } machine_cases[] = { + {str8_lit_comp("x86"), COFF_MachineType_X86}, + {str8_lit_comp("x64"), COFF_MachineType_X64}, + }; + U64 member_count = ArrayCount(type_cases) * ArrayCount(import_by_cases) * ArrayCount(machine_cases); + T_COFF_DefLibMember *members = push_array(arena, T_COFF_DefLibMember, member_count + 1); + COFF_MachineType *machines = push_array(arena, COFF_MachineType, member_count); + COFF_TimeStamp *timestamps = push_array(arena, COFF_TimeStamp, member_count); + String8List source = {0}; + str8_list_pushf(arena, &source, + "test: { artifacts: { value: { coff: { library: { timestamp: 0, mode: 0, second_linker_member: false, members: {"); + U64 member_idx = 0; + for EachElement(machine_idx, machine_cases) + { + for EachElement(type_idx, type_cases) + { + for EachElement(import_by_idx, import_by_cases) + { + U16 hint_or_ordinal = (U16)(100 + member_idx); + COFF_TimeStamp timestamp = (COFF_TimeStamp)(1000 + member_idx); + String8 name = machine_cases[machine_idx].type == COFF_MachineType_X86 ? push_str8f(arena, "_codec_import_%llu@4", member_idx) + : push_str8f(arena, "codec_import_%llu", member_idx); + char *number_field = import_by_cases[import_by_idx].type == COFF_ImportBy_Ordinal ? "ordinal" : "hint"; + str8_list_pushf(arena, &source, + " m%llu: { import: { dll: \"imports.dll\", name: \"%S\", machine: %S, timestamp: %u, type: %S, lookup: %S, %s: %u } }", + member_idx, name, machine_cases[machine_idx].name, timestamp, type_cases[type_idx].name, import_by_cases[import_by_idx].name, number_field, + hint_or_ordinal); + machines[member_idx] = machine_cases[machine_idx].type; + timestamps[member_idx] = timestamp; + members[member_idx].type = T_COFF_DefLibMember_Import; + members[member_idx].import.dll = "imports.dll"; + members[member_idx].import.name = (char *)name.str; + members[member_idx].import.import_by = import_by_cases[import_by_idx].type; + members[member_idx].import.type = type_cases[type_idx].type; + members[member_idx].import.hit_or_ordinal = hint_or_ordinal; + members[member_idx].import.time_stamp = ×tamps[member_idx]; + members[member_idx].import.machine = &machines[member_idx]; + member_idx += 1; + } + } + } + str8_list_pushf(arena, &source, " } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("import_variants_parity.tst"), str8_list_join(arena, &source, 0), &result); + String8 expected = t_coff_from_def_lib(arena, (T_COFF_DefLib){.emit_second_member = 0, .members = members}); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_validation_precedes_encoding) +{ + String8 source = str8_lit("test: { artifacts: { value: { coff: { object: { machine: x64, sections: { text: { name: \".text\", permissionz: (read), content: " + "code, data: { hex: \"c3\" } } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("invalid_coff.tst"), source, &script); + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(value != 0 && value->data.size == 0 && value->state == T_ArtifactState_Failed); +} + +TEST(coff_codec_const_import) +{ + String8 source = str8_lit("test: { artifacts: { value: { coff: { library: { members: { imported: { import: { dll: \"foo.dll\", name: \"foo\", machine: x64, " + "type: const, lookup: undecorate } } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("const_import.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + MD_Node *semantic = 0; + if (t_result_is_ok(result)) { result = value->codec->decode(&script, value, &semantic); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + if (semantic != 0) { + MD_Node *members = t_child_from_string(t_child_from_string(semantic, "library"), "members"); + MD_Node *import = t_child_from_string(t_child_from_string(members, "member_0"), "import"); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(import, "type")), str8_lit("Const"), 0)); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(import, "lookup")), str8_lit("3"), 0)); + } +} + +TEST(coff_codec_rejects_variant_fields) +{ + String8 source = + str8_lit("test: { artifacts: { value: { coff: { object: { machine: x64, sections: { text: { name: \".text\", permissions: (read), content: code, " + "data: { hex: \"c3\" } } }, " + "symbols: { bad_external: { kind: external, name: \"bad\", section: text, size: 4 }, bad_absolute: { kind: absolute, name: \"abs\", value: 1 } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("invalid_variant.tst"), source, &script); + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(result.diagnostics.count >= 2); + T_Ok(value != 0 && value->data.size == 0 && value->state == T_ArtifactState_Failed); +} + +TEST(coff_codec_rejects_invalid_section_name_offset) +{ + String8 data = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + }); + COFF_FileHeaderInfo header = coff_file_header_info_from_data(data); + COFF_SectionHeader *section = (COFF_SectionHeader *)(data.str + header.section_table_range.min); + MemoryZeroArray(section->name); + MemoryCopy(section->name, "/999999", 7); + T_Context decode = {.arena = arena}; + T_Artifact artifact = {.data = data, .definition = &md_nil_node}; + MD_Node *semantic = 0; + T_Result result = t_coff_decode(&decode, &artifact, &semantic); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(semantic == 0); +} + +TEST(coff_codec_rejects_truncated_import) +{ + COFF_LibWriter *writer = coff_lib_writer_alloc(); + coff_lib_writer_push_import(writer, COFF_MachineType_X64, 0, str8_lit("foo.dll"), COFF_ImportBy_Name, str8_lit("foo"), 0, COFF_ImportHeader_Code); + String8 data = coff_lib_writer_serialize(arena, writer, 0, 0, 1); + coff_lib_writer_release(&writer); + U64 offset = coff_regular_archive_member_iter_init(data); + COFF_ArchiveMember member = {0}; + while (coff_regular_archive_member_iter_next(data, &offset, &member)) { + if (coff_is_import(member.data)) { + COFF_ImportHeader *header = (COFF_ImportHeader *)member.data.str; + header->data_size += 1; + break; + } + } + T_Context decode = {.arena = arena}; + T_Artifact artifact = {.data = data, .definition = &md_nil_node}; + MD_Node *semantic = 0; + T_Result result = t_coff_decode(&decode, &artifact, &semantic); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(semantic == 0); +} diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c new file mode 100644 index 000000000..008cba9d8 --- /dev/null +++ b/src/torture/tests/torture_linker_tests.c @@ -0,0 +1,576 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +typedef struct T_SharedPoolTask +{ + U64 visits[64]; + U32 delay_ms; + B32 use_barrier; +} T_SharedPoolTask; + +internal THREAD_POOL_TASK_FUNC(t_shared_pool_task) +{ + T_SharedPoolTask *task = raw_task; + ins_atomic_u64_inc_eval(&task->visits[task_id]); + if (task->delay_ms) { sleep_ms(task->delay_ms); } + if (task->use_barrier) { barrier_wait(tp->barrier); } +} + +TEST(shared_pool_governor_repeated_passes) +{ +#if OS_WINDOWS + // An isolated budget must not borrow from (or disturb) running production links. + String8 name = str8f(arena, "radlink-torture-governor-%u-%llu", GetCurrentProcessId(), now_time_us()); + TP_Context *pool = tp_alloc(arena, 4, 4, name); + T_SharedPoolTask task = {0}; + tp_for_parallel(pool, 0, 64, t_shared_pool_task, &task); + + // Measure, but do not assert a timing threshold on a shared/busy test machine. + // The old governor burns a core while these already-assigned tasks sleep. + W32_Entity *governor = (W32_Entity *)PtrFromInt(pool->governor_handle.u64[0]); + FILETIME created, exited, kernel_before, user_before, kernel_after, user_after; + B32 before_ok = GetThreadTimes(governor->thread.handle, &created, &exited, &kernel_before, &user_before); + MemoryZeroStruct(&task); + task.delay_ms = 200; + tp_for_parallel(pool, 0, 3, t_shared_pool_task, &task); + B32 after_ok = GetThreadTimes(governor->thread.handle, &created, &exited, &kernel_after, &user_after); + for EachIndex(i, 3) { T_Ok(task.visits[i] == 1); } + T_Ok(before_ok && after_ok); + if (before_ok && after_ok) { + U64 cpu_before = (((U64)kernel_before.dwHighDateTime << 32) | kernel_before.dwLowDateTime) + + (((U64)user_before.dwHighDateTime << 32) | user_before.dwLowDateTime); + U64 cpu_after = (((U64)kernel_after.dwHighDateTime << 32) | kernel_after.dwLowDateTime) + + (((U64)user_after.dwHighDateTime << 32) | user_after.dwLowDateTime); + fprintf(stdout, "[shared governor] covered-pass CPU: %.3f ms\n", (F64)(cpu_after - cpu_before) / 10000.0); + } + + // Exercise wake coalescing, tiny/queued passes, and path-A/path-B transitions. + U64 counts[] = {1, 2, 3, 5, 31, 64}; + for EachIndex(pass, 96) { + MemoryZeroStruct(&task); + U64 count = counts[pass % ArrayCount(counts)]; + task.delay_ms = (pass % 12 == 0) ? 1 : 0; + tp_for_parallel(pool, 0, count, t_shared_pool_task, &task); + for EachIndex(i, count) { T_Ok(task.visits[i] == 1); } + T_Ok(ins_atomic_u64_eval(&pool->granted) == 0); + + MemoryZeroStruct(&task); + task.use_barrier = 1; + U64 cohort = tp_barrier_begin(pool); + tp_for_parallel_reserve(pool, 0, cohort, t_shared_pool_task, &task); + tp_barrier_end(pool); + for EachIndex(i, cohort) { T_Ok(task.visits[i] == 1); } + } + + // Join before releasing test-owned storage/synchronization. thread_join consumes + // each handle; clear it before the common release path detaches remaining handles. + pool->is_live = 0; + semaphore_drop_if_room(pool->governor_semaphore); + for (U64 i = 1; i < pool->worker_count; ++i) { semaphore_drop(pool->wake_semaphore); } + for (U64 i = 1; i < pool->worker_count; ++i) { + T_Ok(thread_join(pool->worker_arr[i].handle, max_U64)); + MemoryZeroStruct(&pool->worker_arr[i].handle); + } + T_Ok(thread_join(pool->governor_handle, max_U64)); + MemoryZeroStruct(&pool->governor_handle); + tp_release(pool); +#else + TestSkip(); +#endif +} + +internal String8 +t_codec_test_pdb_data(Arena *arena) +{ + PDB_Context *pdb = pdb_alloc(MSF_DEFAULT_PAGE_SIZE, COFF_MachineType_X64, 123, 1, (Guid){0}); + String8 data = data_from_pdb(arena, pdb); + pdb_release(pdb); + return data; +} + +TEST(generic_run_operation) +{ +#if OS_WINDOWS + T_Ok(copy_file_path(t_make_file_path(arena, str8_lit("run_fixture.exe")), get_process_info()->binary_file_path)); + + String8 source = str8_lit( + "test:\n" + "{\n" + " steps:\n" + " {\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_ok\" }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_nonzero\", expect_exit: 7 }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_nonzero\", expect_exit: nonzero }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_nonzero\", expect_exit: any }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_ok\",\n" + " stdout_matches: \"*genericrunstdout*\", stderr_matches: \"*genericrunstderr*\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_run_operation.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + + String8 timeout_source = str8_lit( + "test: { steps: { run: { path: \"run_fixture.exe\", args: \"-run_fixture_sleep\", timeout_ms: 10 } } }"); + T_Context timeout_script = {0}; + result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_run_timeout.tst"), timeout_source, &timeout_script); + U64 begin_us = now_time_us(); + if (t_result_is_ok(result)) { result = t_script_execute(&timeout_script); } + U64 elapsed_us = now_time_us() - begin_us; + T_Ok(result.code == T_ResultCode_Mismatch); + T_Ok(g_last_exit_code == 999); + T_Ok(elapsed_us < TIMEOUT_SEC(5)); +#else + TestSkip(); +#endif +} + +TEST(generic_run_rejects_unsafe_path) +{ + String8 source = str8_lit( + "test:\n" + "{\n" + " steps:\n" + " {\n" + " run: { path: \"../cmd.exe\" }\n" + " run: { path: \"..\\cmd.exe\" }\n" + " run: { path: \"C:\\Windows\\System32\\cmd.exe\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_run_unsafe_path.tst"), source, &script); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(result.diagnostics.count == 3); +} + +TEST(generic_build_default_output) +{ +#if OS_WINDOWS + String8 source = str8_lit( + "test:\n" + "{\n" + " artifacts:\n" + " {\n" + " source: { file_name: \"main.c\", text: { data: \"int main(void) { return 0; }\" } }\n" + " }\n" + " build:\n" + " {\n" + " windows: { compile_link: \"/nologo main.c\" }\n" + " linux: { compile_link: \"main.c\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_build.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(file_path_exists(t_make_file_path(arena, str8_lit("main.exe")))); +#else + TestSkip(); +#endif +} + +TEST(generic_build_output_with_spaces) +{ +#if OS_WINDOWS + String8 source = str8_lit( + "test:\n" + "{\n" + " artifacts:\n" + " {\n" + " source: { file_name: \"main.c\", text: { data: \"int main(void) { return 0; }\" } }\n" + " }\n" + " build:\n" + " {\n" + " compile_link: { args: \"/nologo main.c\", output: \"main image.exe\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_build_spaces.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(file_path_exists(t_make_file_path(arena, str8_lit("main image.exe")))); +#else + TestSkip(); +#endif +} + +TEST(generic_build_previous_exit_condition) +{ +#if OS_WINDOWS + String8 source = str8_lit( + "test:\n" + "{\n" + " build:\n" + " {\n" + " compile: { args: \"missing.c\", output: none, expect_exit: any }\n" + " compile: { args: \"also_missing.c\", output: none, when_previous_exit: 0 }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_build_previous_exit.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); +#else + TestSkip(); +#endif +} + +TEST(expect_pdb_operation) +{ + String8 source = str8_lit( + "test: { steps: { expect_pdb: { path: \"generated.pdb\", expected: { pdb: { fixed_streams: { info: { present: true }, tpi: { present: true }, " + "dbi: { present: true }, ipi: { present: true } }, tpi: { leaf_count: 0, header_only: true }, ipi: { leaf_count: 0, header_only: true } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("expect_pdb_operation.tst"), source, &script); + T_Ok(t_result_is_ok(result)); + + // The operation must not snapshot the side file during script parsing. + if (t_result_is_ok(result)) { + String8 pdb = t_codec_test_pdb_data(arena); + T_Ok(t_write_file(str8_lit("generated.pdb"), pdb)); + result = t_script_execute(&script); + } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); +} + +TEST(expect_pdb_rejects_unsafe_path) +{ + String8 source = str8_lit("test: { steps: { expect_pdb: { path: \"generated.pdb\", expected: { pdb: { stream_count: 0 } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("unsafe_pdb.tst"), source, &script); + T_Ok(t_result_is_ok(result) && script.command_count == 1); + if (t_result_is_ok(result) && script.command_count == 1) { + t_codec_child(script.commands[0].arguments, "path")->first->string = str8_lit(".."); + script.result = (T_Result){0}; + T_ParseContext parse = {arena, &script, &t_codec_script_suite, script.file_path, script.source, str8_lit("expect_pdb")}; + result = t_codec_expect_pdb_validate(&parse, script.commands[0].arguments); + } + T_Ok(result.code == T_ResultCode_ValidationError); +} + +TEST(pdb_codec_rejects_malformed_tpi) +{ + String8 data = t_codec_test_pdb_data(arena); + MSF_RawStreamTable *streams = msf_raw_stream_table_from_data(arena, data); + T_Ok(streams != 0 && streams->stream_count > PDB_FixedStream_Tpi && streams->streams[PDB_FixedStream_Tpi].page_count > 0); + if (streams != 0 && streams->stream_count > PDB_FixedStream_Tpi && streams->streams[PDB_FixedStream_Tpi].page_count > 0) { + U32 page = streams->streams[PDB_FixedStream_Tpi].u.page_indices_u32[0]; + PDB_TpiHeader *header = (PDB_TpiHeader *)(data.str + (U64)page * streams->page_size); + header->leaf_data_size = max_U32; + } + T_Context decode = {.arena = arena}; + MD_Node *semantic = 0; + T_Result result = t_codec_pdb_decode(&decode, data, &semantic); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(semantic == 0); +} + +TEST(expect_pdb_indexes_linker_output) +{ + String8 raw_symbol = cv_make_symbol(arena, CV_SymKind_GDATA32, + cv_make_data32(arena, (CV_SymData32){.off = 1, .sec = 1}, str8_lit("indexed_data"))); + CV_Symbol symbol = cv_symbol_from_ptr(raw_symbol.str); + CV_DebugS debug_s = {0}; + str8_list_push(arena, cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols), cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + String8List debug_s_data = cv_data_from_debug_s_c13(arena, &debug_s, 1); + T_Ok(t_write_def_obj("indexed.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("x"), .flags = "rx:code"}, + {"debug", ".debug$S", str8_list_join(arena, &debug_s_data, 0), .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("public_func", "text", 0), {0}}, + })); + T_Ok(t_write_entry_obj()); + t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:indexed.exe /pdbstripped:indexed.stripped.pdb entry.obj indexed.obj"); + T_Ok(g_last_exit_code == 0); + + String8 source = str8_lit( + "test: { steps: { expect_pdb: { path: \"indexed.pdb\", expected: { pdb: { " + "gsi: { symbols: { indexed_data: { kind: S_GDATA32 } } }, psi: { symbols: { public_func: { kind: S_PUB32, flags: 2 } } }, " + "global_symbols: { kind_counts: { S_GDATA32: 1 } }, dbi: { modules: { module_1: { object_file_name: \"indexed.obj\" } } } } } }, " + "expect_pdb: { path: \"indexed.stripped.pdb\", expected: { pdb: { tpi: { header_only: true }, ipi: { header_only: true }, " + "global_symbols: { non_public_or_proc_ref_symbol_count: 0 } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("indexed_pdb.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); +} + +TEST(link_large_import_object) +{ + // Each named import creates three sections in the synthesized DLL object. + // Add a direct function reference as well to exercise its jump thunk. + U32 import_count = 22000; + COFF_LibWriter *lib = coff_lib_writer_alloc(); + COFF_ObjWriter *obj = coff_obj_writer_alloc(0, COFF_MachineType_X64); + COFF_ObjSection *refs = coff_obj_writer_push_section(obj, str8_lit(".data"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemWrite|COFF_SectionFlag_Align8Bytes, + str8(push_array(arena, U8, (import_count + 1)*8), (import_count + 1)*8)); + for (U32 i = 0; i < import_count; i += 1) { + String8 name = str8f(arena, "import_%05u", i); + coff_lib_writer_push_import(lib, COFF_MachineType_X64, 0, str8_lit("large.dll"), COFF_ImportBy_Name, name, 0, COFF_ImportHeader_Code); + COFF_ObjSymbol *symbol = coff_obj_writer_push_symbol_undef(obj, str8f(arena, "__imp_%S", name)); + coff_obj_writer_section_push_reloc_addr(obj, refs, i*8, symbol); + } + COFF_ObjSymbol *func = coff_obj_writer_push_symbol_undef_func(obj, str8f(arena, "import_%05u", import_count - 1)); + coff_obj_writer_section_push_reloc_addr(obj, refs, import_count*8, func); + T_Ok(t_write_file(str8_lit("refs.obj"), coff_obj_writer_serialize(arena, obj))); + T_Ok(t_write_file(str8_lit("large.lib"), coff_lib_writer_serialize(arena, lib, 0, 0, 1))); + coff_obj_writer_release(&obj); + coff_lib_writer_release(&lib); + T_Ok(t_write_entry_obj()); + U64 link_begin_us = now_time_us(); + t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /opt:ref /rad_log:summary /out:large.exe entry.obj refs.obj large.lib"); + U64 link_elapsed_ms = (now_time_us() - link_begin_us) / 1000 + 1; + T_Ok(g_last_exit_code == 0); + // A mismatched phase begin/end used to report system uptime for modules. + // Use the enclosing invocation, not a fixed performance threshold. + U64 mod_pos = str8_find_needle(g_output, 0, str8_lit(" mod="), 0); + T_Ok(mod_pos < g_output.size); + if (mod_pos < g_output.size) { + String8 mod = str8_skip(g_output, mod_pos + 5); + mod = str8_prefix(mod, str8_find_needle(mod, 0, str8_lit("/"), 0)); + U64 mod_ms = 0; + T_Ok(try_u64_from_str8_c_rules(mod, &mod_ms)); + T_Ok(mod_ms <= link_elapsed_ms); + } + String8 image = t_read_file(arena, str8_lit("large.exe")); + PE_BinInfo bin = pe_bin_info_from_data(arena, image); + T_Ok(bin.data_dir_count > PE_DataDirectoryIndex_IMPORT); + COFF_SectionHeader *sections = (COFF_SectionHeader *)(image.str + bin.section_table_range.min); + PE_ParsedStaticImportTable imports = pe_static_imports_from_data(arena, bin.is_pe32, bin.section_count, sections, + image, bin.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); + T_Ok(imports.count == 1); + T_Ok(str8_match(imports.v[0].name, str8_lit("large.dll"), 0)); + T_Ok(imports.v[0].import_count == import_count); + for (U32 i = 0; i < import_count; i += 1) { + PE_ParsedImport *import = &imports.v[0].imports[i]; + T_Ok(import->type == PE_ParsedImport_Name); + T_Ok(str8_match(import->u.name.string, str8f(arena, "import_%05u", i), 0)); + } +} + +TEST(compressed_debug_reloc_parity) +{ +#if OS_WINDOWS + // The optional Oodle compressor is built with `rad_obj_compress`; ordinary + // torture builds do not require its SDK. + String8 compressor = str8f(arena, "%S/build/rad_obj_compress.exe", t_cwd_path()); + if (!file_path_exists(compressor)) { TestSkip(); } + + // Two relocations per record: overflow COFF's 16-bit relocation count, + // cross compressed-segment boundaries, and finish with a partial segment. + U32 record_count = 32769; + U32 *record_offsets = push_array(arena, U32, record_count); + CV_DebugS debug_s = {0}; + String8List *symbols = cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols); + for EachIndex(i, record_count) { + record_offsets[i] = sizeof(CV_Signature) + sizeof(CV_C13SubSectionHeader) + safe_cast_u32(symbols->total_size); + String8 raw = cv_make_symbol(arena, CV_SymKind_GDATA32, + cv_make_data32(arena, (CV_SymData32){.itype = 0x74, .off = (U32)i*4}, str8f(arena, "relocated_%05u", (U32)i))); + CV_Symbol symbol = cv_symbol_from_ptr(raw.str); + str8_list_push(arena, symbols, cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + } + String8List debug_data = cv_data_from_debug_s_c13(arena, &debug_s, 1); + COFF_ObjWriter *writer = coff_obj_writer_alloc(0, COFF_MachineType_X64); + COFF_ObjSection *data = coff_obj_writer_push_section(writer, str8_lit(".data"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemWrite, + str8(push_array(arena, U8, record_count*4 + 16), record_count*4 + 16)); + COFF_ObjSymbol *target = coff_obj_writer_push_symbol_static(writer, str8_lit("target"), 16, data); + coff_obj_writer_push_section(writer, str8_lit(".debug$S"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemDiscardable, + str8_list_join(arena, &debug_data, 0)); + // Many small relocation tables share compressed segments. Exercise their + // decoding interleaved with section-data reads. + for EachIndex(i, 512) { + CV_DebugS small_debug_s = {0}; + String8 raw = cv_make_symbol(arena, CV_SymKind_GDATA32, + cv_make_data32(arena, (CV_SymData32){.itype = 0x74, .off = (U32)i*4}, str8f(arena, "small_%04u", (U32)i))); + CV_Symbol symbol = cv_symbol_from_ptr(raw.str); + str8_list_push(arena, cv_sub_section_ptr_from_debug_s(&small_debug_s, CV_C13SubSectionKind_Symbols), + cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + String8List small_data = cv_data_from_debug_s_c13(arena, &small_debug_s, 1); + COFF_ObjSection *small_section = coff_obj_writer_push_section(writer, str8_lit(".debug$S"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemDiscardable, + str8_list_join(arena, &small_data, 0)); + U32 off = sizeof(CV_Signature) + sizeof(CV_C13SubSectionHeader) + sizeof(CV_SymbolHeader); + coff_obj_writer_section_push_reloc(writer, small_section, off + OffsetOf(CV_SymData32, sec), target, COFF_Reloc_X64_Section); + coff_obj_writer_section_push_reloc(writer, small_section, off + OffsetOf(CV_SymData32, off), target, COFF_Reloc_X64_SecRel); + } + String8 obj = coff_obj_writer_serialize(arena, writer); + // The fixture writer only emits 16-bit relocation counts. Append an explicit + // overflow table after its ordinary object data; symbol indices are now final. + U32 reloc_count = record_count*2; + COFF_Reloc *relocs = push_array(arena, COFF_Reloc, reloc_count + 1); + relocs[0].apply_off = reloc_count + 1; + U32 reloc_idx = 1; + // Deliberately reversed, so the copy must preserve records and their sort keys. + for (U32 i = record_count; i-- > 0;) { + U32 off = record_offsets[i] + sizeof(CV_SymbolHeader); + relocs[reloc_idx++] = (COFF_Reloc){off + OffsetOf(CV_SymData32, sec), target->idx, COFF_Reloc_X64_Section}; + relocs[reloc_idx++] = (COFF_Reloc){off + OffsetOf(CV_SymData32, off), target->idx, COFF_Reloc_X64_SecRel}; + } + coff_obj_writer_release(&writer); + COFF_FileHeaderInfo header = coff_file_header_info_from_data(obj); + COFF_SectionHeader *sections = (COFF_SectionHeader *)(obj.str + header.section_table_range.min); + sections[1].relocs_foff = safe_cast_u32(obj.size); + sections[1].reloc_count = max_U16; + sections[1].flags |= COFF_SectionFlag_LnkNRelocOvfl; + String8List obj_parts = {0}; + str8_list_push(arena, &obj_parts, obj); + str8_list_push(arena, &obj_parts, str8_array(relocs, reloc_count + 1)); + obj = str8_list_join(arena, &obj_parts, 0); + T_Ok(t_write_file(str8_lit("raw.obj"), obj)); + T_Ok(t_write_file(str8_lit("reloc_input.obj"), obj)); + T_Ok(t_write_entry_obj()); + char *args = "/subsystem:console /entry:entry /debug:full /rad_time_stamp:0 /rad_workers:1 /out:reloc.exe /pdbaltpath:reloc.pdb entry.obj reloc_input.obj"; + t_invoke_linkerf("%s", args); + T_Ok(g_last_exit_code == 0); + String8 expected_image = t_read_file(arena, str8_lit("reloc.exe")); + String8 expected_pdb = t_read_file(arena, str8_lit("reloc.pdb")); + T_Ok(t_invoke(compressor, str8_lit("raw.obj compressed.obj 64 kraken 256 fast"), TIMEOUT_SEC(30))); + T_Ok(g_last_exit_code == 0); + T_Ok(t_write_file(str8_lit("reloc_input.obj"), t_read_file(arena, str8_lit("compressed.obj")))); + t_invoke_linkerf("%s", args); + T_Ok(g_last_exit_code == 0); + T_Ok(str8_match(expected_image, t_read_file(arena, str8_lit("reloc.exe")), 0)); + T_Ok(str8_match(expected_pdb, t_read_file(arena, str8_lit("reloc.pdb")), 0)); +#else + TestSkip(); +#endif +} + +TEST(compressed_icf_debug_record_selection) +{ +#if OS_WINDOWS + String8 compressor = str8f(arena, "%S/build/rad_obj_compress.exe", t_cwd_path()); + if (!file_path_exists(compressor)) { TestSkip(); } + // Three folded pairs: no locals, locals at different source lines, and locals + // at the same source line. Only the second pair needs both full record trees. + String8 source = str8_lit( + "__declspec(noinline) int empty_a(void) { return 3; }\n" + "__declspec(noinline) int empty_b(void) { return 3; }\n" + "__declspec(noinline) int different_a(int x) { int a = x+1; return a; }\n" + "__declspec(noinline) int different_b(int x) { int b = x+1; return b; }\n" + "#line 200 \"shared.h\"\n" + "__declspec(noinline) int same_a(int x) { int a = x+2; return a; }\n" + "#line 200 \"shared.h\"\n" + "__declspec(noinline) int same_b(int x) { int b = x+2; return b; }\n" + "int (*volatile functions[])(int) = {different_a,different_b,same_a,same_b};\n" + "int (*volatile empty[])(void) = {empty_a,empty_b};\n" + "int entry(void) { return functions[0] != functions[1] || functions[2] != functions[3] || empty[0] != empty[1]; }\n"); + T_Ok(t_write_file(str8_lit("icf_source.c"), source)); + // Permit ICF despite the address comparisons used to verify the folds below. + T_Ok(t_invoke(t_clang_path(), str8_lit("--target=x86_64-pc-windows-msvc -c -O0 -g -gcodeview -ffunction-sections -fno-addrsig -o raw.obj icf_source.c"), TIMEOUT_SEC(30))); + T_Ok(g_last_exit_code == 0); + String8 raw_obj = t_read_file(arena, str8_lit("raw.obj")); + T_Ok(t_invoke(compressor, str8_lit("raw.obj compressed.obj 64 kraken 256 fast"), TIMEOUT_SEC(30))); + T_Ok(g_last_exit_code == 0); + String8 inputs[] = {raw_obj, t_read_file(arena, str8_lit("compressed.obj"))}; + String8 expected_image = {0}, expected_pdb = {0}; + for EachElement(input_idx, inputs) { + T_Ok(t_write_file(str8_lit("icf_input.obj"), inputs[input_idx])); + t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /debug:full /opt:ref,icf /rad_time_stamp:0 /rad_workers:1 /out:icf.exe /pdbaltpath:icf.pdb icf_input.obj"); + T_Ok(g_last_exit_code == 0); + T_Ok(t_invoke(t_make_file_path(arena, str8_lit("icf.exe")), str8_zero(), TIMEOUT_SEC(5))); + T_Ok(g_last_exit_code == 0); // prove all three function pairs actually folded + String8 image_data = t_read_file(arena, str8_lit("icf.exe")); + String8 pdb_data = t_read_file(arena, str8_lit("icf.pdb")); + if (input_idx == 0) { + expected_image = image_data; + expected_pdb = pdb_data; + } else { + T_Ok(str8_match(expected_image, image_data, 0)); + T_Ok(str8_match(expected_pdb, pdb_data, 0)); + } + + MSF_Parsed *msf = msf_parsed_from_data(arena, pdb_data); + T_Ok(msf != 0); + if (!msf) { continue; } + PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, msf_data_from_stream(msf, PDB_FixedStream_Dbi)); + PDB_CompUnitArray *modules = pdb_comp_unit_array_from_data(arena, pdb_data_from_dbi_range(dbi, PDB_DbiRange_ModuleInfo)); + U32 empty_count = 0, different_count = 0, same_count = 0; + for EachIndex(module_idx, modules->count) { + String8 symbols = pdb_data_from_unit_range(msf, modules->units[module_idx], PDB_DbiCompUnitRange_Symbols); + for (U64 cursor = 0; cursor < symbols.size;) { + CV_Symbol symbol = {0}; U64 read_size = 0; String8 error = {0}; + B32 ok = t_codec_pdb_read_symbol(symbols, cursor, PDB_SYMBOL_ALIGN, &symbol, &read_size, &error); + T_Ok(ok); + if (!ok) { break; } + if (CV_IsProc32(symbol.kind)) { + String8 name = cv_name_from_symbol(symbol.kind, symbol.data); + if (str8_match(name, str8_lit("empty_a"), 0) || str8_match(name, str8_lit("empty_b"), 0)) { empty_count += 1; } + if (str8_match(name, str8_lit("different_a"), 0) || str8_match(name, str8_lit("different_b"), 0)) { different_count += 1; } + if (str8_match(name, str8_lit("same_a"), 0) || str8_match(name, str8_lit("same_b"), 0)) { same_count += 1; } + } + cursor += read_size; + } + } + T_Ok(empty_count == 1); + T_Ok(different_count == 2); + T_Ok(same_count == 1); + } +#else + TestSkip(); +#endif +} + +TEST(pdbstripped_coff_artifact_parity) +{ + String8 path = str8f(arena, "%S/linker/tests/pdbstripped.tst", t_src_path()); + String8 source = data_from_file_path(arena, path); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, path, source, &script); + for (T_Artifact *artifact = script.first_artifact; t_result_is_ok(result) && artifact != 0; artifact = artifact->next) { + if (artifact->codec->encode != 0) { result = artifact->codec->encode(&script, artifact); } + } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + + String8 raw_symbols[] = { + cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), + cv_make_symbol(arena, CV_SymKind_GPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("global_proc"))), + cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), + + cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("global_typedef"))), + + cv_make_symbol(arena, CV_SymKind_LPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("local_proc"))), + cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("local_typedef"))), + cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), + }; + CV_DebugS debug_s = {0}; + for EachElement(i, raw_symbols) { + CV_Symbol symbol = cv_symbol_from_ptr(raw_symbols[i].str); + str8_list_push(arena, cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols), cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + } + String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); + String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); + String8 expected_debug = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable}, {0}}, + }); + String8 expected_pub = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("FOOBAR"), .flags = "rx:code"}, {"data", ".data", str8_lit_comp("QWE"), .flags = "rw:data"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("global_func", "text", 1), T_COFF_DefSymbol_Extern("global_var", "data", 1), + T_COFF_DefSymbol_Static("static_var", "data", 1), {0}}, + }); + + T_Artifact *debug = t_artifact_from_name(&script, str8_lit("debug_obj")); + T_Artifact *pub = t_artifact_from_name(&script, str8_lit("pub_obj")); + T_Artifact *entry = t_artifact_from_name(&script, str8_lit("entry_obj")); + if (debug != 0 && !str8_match(debug->data, expected_debug, 0)) { + U64 mismatch = 0; + while (mismatch < Min(debug->data.size, expected_debug.size) && debug->data.str[mismatch] == expected_debug.str[mismatch]) { mismatch += 1; } + test_outf("debug.obj differs at byte %llu (script size %llu, helper size %llu, helper section size %llu)\n", mismatch, debug->data.size, expected_debug.size, raw_debug_s.size); + } + T_Ok(debug != 0 && str8_match(debug->data, expected_debug, 0)); + T_Ok(pub != 0 && str8_match(pub->data, expected_pub, 0)); + T_Ok(entry != 0 && str8_match(entry->data, t_make_entry_obj(arena), 0)); +} diff --git a/src/torture/tests/torture_script_tests.c b/src/torture/tests/torture_script_tests.c new file mode 100644 index 000000000..32a85c2a1 --- /dev/null +++ b/src/torture/tests/torture_script_tests.c @@ -0,0 +1,18 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal void +t_script_test_log_result(Arena *arena, TestCtx *ctx, T_Result result) +{ + for EachNode(diagnostic, T_Diagnostic, result.diagnostics.first) { + String8 operation = diagnostic->operation.size == 0 ? str8_zero() : str8f(arena, " [%S]", diagnostic->operation); + test_outf("%S:%lld:%lld%S: %S\n", diagnostic->file_path, diagnostic->location.line, diagnostic->location.column, operation, diagnostic->message); + } +} + +TEST(script_path_identity) +{ + T_Ok(str8_match(test_layer_from_file_path(str8_lit("C:/repo/src/linker/tests/foo.tst")), str8_lit("linker"), 0)); + T_Ok(str8_match(test_layer_from_file_path(str8_lit("C:\\repo\\src\\torture\\tests\\foo.c")), str8_lit("torture"), 0)); + T_Ok(test_layer_from_file_path(str8_lit("C:/repo/tests/foo.tst")).size == 0); +} diff --git a/src/torture/tests/torture_subprocess_tests.c b/src/torture/tests/torture_subprocess_tests.c new file mode 100644 index 000000000..329a3eaa3 --- /dev/null +++ b/src/torture/tests/torture_subprocess_tests.c @@ -0,0 +1,147 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#include + +internal void +t_controller_fixture(void) +{ + fprintf(stdout, "stdout:ready\n"); + fprintf(stderr, "stderr:ready\n"); + fflush(stdout); + fflush(stderr); + + char line[4096]; + while (fgets(line, sizeof(line), stdin) != 0) { + U64 size = strlen(line); + while (size != 0 && (line[size - 1] == '\r' || line[size - 1] == '\n')) { line[--size] = 0; } + if (strcmp(line, "exit") == 0) { break; } + fprintf(stdout, "stdout:%s\n", line); + fprintf(stderr, "stderr:%s\n", line); + fflush(stdout); + fflush(stderr); + } +} + +global volatile sig_atomic_t t_controller_fixture_interrupted; + +internal void +t_controller_fixture_signal_handler(int signal) +{ + t_controller_fixture_interrupted = signal == SIGINT; +} + +internal void +t_controller_interrupt_fixture(void) +{ + signal(SIGINT, t_controller_fixture_signal_handler); + fprintf(stdout, "interrupt:ready\n"); + fflush(stdout); + while (!t_controller_fixture_interrupted) { sleep_ms(1); } + fprintf(stdout, "interrupt:received\n"); + fflush(stdout); +} + +internal void +t_run_operation_fixture(B32 sleep, U64 exit_code) +{ + fprintf(stdout, "GenericRunStdout\n"); + fprintf(stderr, "GenericRunStderr\n"); + fflush(stdout); + fflush(stderr); + if (sleep) { sleep_ms(30000); } + exit((int)exit_code); +} + +internal T_ControllerResult +t_controller_test_launch(Arena *arena, T_Controller *controller, B32 merge_outputs) +{ + String8List cmd_line = {0}; + str8_list_push(arena, &cmd_line, get_process_info()->binary_file_path); + str8_list_push(arena, &cmd_line, str8_lit("-controller_fixture")); + T_ControllerLaunchParams params = { + .cmd_line = cmd_line, + .path = get_process_info()->binary_path, + .inherit_env = 1, + .consoleless = 1, + .merge_outputs = merge_outputs, + }; + return t_controller_launch(arena, controller, ¶ms); +} + +TEST(controller_interactive_separate_output) +{ + T_Controller controller = {0}; + B32 is_ok = 0; + + if (t_controller_test_launch(arena, &controller, 0).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stderr, str8_lit("stderr:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait_until_quiet(&controller, TIMEOUT_MS(10), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("hello")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:hello"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stderr, str8_lit("stderr:hello"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("missing"), TIMEOUT_MS(10)).code != T_ControllerResultCode_Timeout) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("exit")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait(&controller, TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok || controller.exit_code != 0) { goto exit; } + is_ok = 1; + +exit:; + t_controller_close(&controller); + if (!is_ok) { + test_outf("stdout:\n%S\n", t_controller_stdout(arena, &controller)); + test_outf("stderr:\n%S\n", t_controller_stderr(arena, &controller)); + } + T_Ok(is_ok); +} + +TEST(controller_interactive_merged_output) +{ + T_Controller controller = {0}; + B32 is_ok = 0; + + if (t_controller_test_launch(arena, &controller, 1).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stderr, str8_lit("stderr:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("merged")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:merged"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("exit")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait(&controller, TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok || controller.exit_code != 0) { goto exit; } + { + String8 output = t_controller_stdout(arena, &controller); + is_ok = str8_find_needle(output, 0, str8_lit("stderr:ready"), 0) < output.size && str8_find_needle(output, 0, str8_lit("stderr:merged"), 0) < output.size; + } + +exit:; + t_controller_close(&controller); + if (!is_ok) { test_outf("merged output:\n%S\n", t_controller_stdout(arena, &controller)); } + T_Ok(is_ok); +} + +TEST(controller_interrupt) +{ + T_Controller controller = {0}; + B32 is_ok = 0; + String8List cmd_line = {0}; + str8_list_push(arena, &cmd_line, get_process_info()->binary_file_path); + str8_list_push(arena, &cmd_line, str8_lit("-controller_interrupt_fixture")); + T_ControllerLaunchParams params = { + .cmd_line = cmd_line, + .path = get_process_info()->binary_path, + .inherit_env = 1, + .new_console = 1, + .merge_outputs = 1, + }; + + if (t_controller_launch(arena, &controller, ¶ms).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("interrupt:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_interrupt(&controller).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("interrupt:received"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait(&controller, TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok || controller.exit_code != 0) { goto exit; } + is_ok = 1; + +exit:; + t_controller_close(&controller); + if (!is_ok) { test_outf("interrupt output:\n%S\n", t_controller_stdout(arena, &controller)); } + T_Ok(is_ok); +} diff --git a/src/torture/torture.c b/src/torture/torture.c index b85f26601..d234e7cb7 100644 --- a/src/torture/torture.c +++ b/src/torture/torture.c @@ -24,7 +24,6 @@ global String8 g_errors; // tools global B32 g_gui; -global String8 g_radbin_path; global String8 g_cl_path; global String8 g_clang_path; global String8 g_gcc_path; @@ -54,18 +53,6 @@ t_test_layer_from_name(Arena *arena, String8 pattern) return matches; } -//////////////////////////////// - -internal Linker -t_id_linker(void) -{ - String8 name = str8_chop_last_dot(str8_skip_last_slash(g_linker_path)); - if (str8_match(name, str8_lit("radlink"), StringMatchFlag_CaseInsensitive)) { return Linker_radlink; } - if (str8_match(name, str8_lit("link"), StringMatchFlag_CaseInsensitive)) { return Linker_msvc; } - if (str8_match(name, str8_lit("lld-link"), StringMatchFlag_CaseInsensitive)) { return Linker_lld; } - return Linker_Null; -} - internal B32 t_write_file_list(String8 name, String8List data) { @@ -162,69 +149,68 @@ t_run_caller(void *raw_ctx) g_is_first_print = 1; - T_RunCtx *ctx = raw_ctx; - ctx->result.status = TestStatus_Pass; + T_RunCtx *run_ctx = raw_ctx; + run_ctx->result.status = TestStatus_Pass; String8List test_out = {0}; - if (ctx->test->skip) { - ctx->result.status = TestStatus_Skip; + if (run_ctx->test->skip) { + run_ctx->result.status = TestStatus_Skip; } else { - TestCtx test_ctx = - { - .cmdline = ctx->cmdline, - .exemplars_path = g_exemplar_dir, - .artifacts_path = g_wdir, + + // run test + TestCtx test_ctx = { + .cmdline = run_ctx->cmdline, + .exemplars_path = g_exemplar_dir, + .artifacts_path = g_wdir, .input_data_path = g_input_data_dir, - .result_out = &ctx->result, - .test_out = &test_out, + .user_data = run_ctx->user_data, + .result_out = &run_ctx->result, + .test_out = &test_out, }; log_scope_begin(); - ctx->test->test_fn(scratch.arena, &test_ctx); + run_ctx->test->test_fn(scratch.arena, &test_ctx); LogScopeResult log_scope_result = log_scope_end(scratch.arena); - if(log_scope_result.strings[LogMsgKind_Info].size != 0) - { - String8 current_path = str8f(scratch.arena, "%S/current", g_wdir); - String8 exemplar_path = str8f(scratch.arena, "%S/exemplar_%S_%S", g_exemplar_dir, lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)), lower_from_str8(scratch.arena, string_from_arch(Arch_CURRENT))); - String8 current = log_scope_result.strings[LogMsgKind_Info]; - String8 exemplar = data_from_file_path(scratch.arena, exemplar_path); + + // epilog for the diff runs + if (log_scope_result.strings[LogMsgKind_Info].size) { + String8 current_path = str8f(scratch.arena, "%S/current", g_wdir); + String8 exemplar_path = str8f(scratch.arena, "%S/exemplar_%S_%S", + g_exemplar_dir, + lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)), + lower_from_str8(scratch.arena, string_from_arch(Arch_CURRENT))); + String8 current = log_scope_result.strings[LogMsgKind_Info]; + String8 exemplar = data_from_file_path(scratch.arena, exemplar_path); write_data_to_file_path(current_path, current); - if(exemplar.size == 0) - { + + if (exemplar.size == 0) { make_directory(g_exemplar_dir); copy_file_path(exemplar_path, current_path); - } - else - { - // TODO(rjf): @hack, see below - TestCtx *ctx = &test_ctx; - String8List exemplar_lines = str8_split(scratch.arena, exemplar, (U8 *)"\n", 1, StringSplitFlag_KeepEmpties); + } else { + String8List exemplar_lines = str8_split(scratch.arena, exemplar, (U8 *)"\n", 1, StringSplitFlag_KeepEmpties); String8List exemplar_lines_sanitized = {0}; - for EachNode(n, String8Node, exemplar_lines.first) - { + for EachNode(n, String8Node, exemplar_lines.first) { String8 line_trimmed = n->string; - if(line_trimmed.size != 0 && line_trimmed.str[line_trimmed.size-1] == '\r') - { + if(line_trimmed.size != 0 && line_trimmed.str[line_trimmed.size-1] == '\r') { line_trimmed.size -= 1; } str8_list_push(scratch.arena, &exemplar_lines_sanitized, line_trimmed); } - StringJoin join = {.sep = s("\n"), .post = s("\n")}; - String8 exemplar_sanitized = str8_list_join(scratch.arena, &exemplar_lines_sanitized, &join); - B32 current_matches_exemplar = str8_match(exemplar_sanitized, current, 0); - if(!current_matches_exemplar) - { + + String8 exemplar_sanitized = str8_list_join(scratch.arena, &exemplar_lines_sanitized, &(StringJoin){.sep = s("\n"), .post = s("\n")}); + B32 current_matches_exemplar = str8_match(exemplar_sanitized, current, 0); + if(!current_matches_exemplar) { // TODO(rjf): @hack, because we need to do test things in the outer scope, but this is // not a test function - all test helpers should wrap a 100% parameterized layer - Arena *arena = scratch.arena; String8 diff_cmd = str8f(scratch.arena, "diff %S %S", path_normalized_from_string(scratch.arena, exemplar_path), path_normalized_from_string(scratch.arena, current_path)); - test_outf("Current log does not match exemplar; run `%S`\n", diff_cmd); + str8_list_pushf(scratch.arena, &test_out, "Current log does not match exemplar; run `%S`\n", diff_cmd); + // TODO(rjf): @hack need to hack this in, because TestCheck assumes `return` - ctx->result_out[0] = (TestResult){.fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = "current_matches_exemplar"}; - if(debugger_is_attached()) - { + test_ctx.result_out[0] = (TestResult){.fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = "current_matches_exemplar"}; + + if(debugger_is_attached()) { Trap(); } } @@ -232,16 +218,10 @@ t_run_caller(void *raw_ctx) } } - if (ctx->result.status == TestStatus_Fail || ctx->result.status == TestStatus_Crash) { - for EachNode(n, String8Node, test_out.first) { - t_errorf("%S", n->string); - } - if (g_errors.size) { - t_errorf("%S\n", g_errors); - } - if (g_output.size) { - t_errorf("%S\n", g_output); - } + if (run_ctx->result.status == TestStatus_Fail || run_ctx->result.status == TestStatus_Crash) { + for EachNode(n, String8Node, test_out.first) { t_errorf("%S", n->string); } + if (g_errors.size) { t_errorf("%S\n", g_errors); } + if (g_output.size) { t_errorf("%S\n", g_output); } } scratch_end(scratch); @@ -267,22 +247,6 @@ t_run(CmdLine *cmdline, TestInfo *test, String8 user_data) return ctx.result; } -internal String8 -t_radbin_path(void) -{ - if (g_radbin_path.size == 0) { - local_persist U8 buffer[4096]; - Arena *arena = arena_alloc_(&(ArenaParams){ .reserve_size = sizeof(buffer), .commit_size = sizeof(buffer), .optional_backing_buffer = buffer }); -#if OS_WINDOWS - g_radbin_path = full_path_from_path(arena, str8_lit("radbin.exe")); -#else - g_radbin_path = full_path_from_path(arena, str8_lit("radbin")); -#endif - } - AssertAlways(g_radbin_path.size); - return g_radbin_path; -} - internal String8 t_cl_path(void) { @@ -885,80 +849,21 @@ t_invoke_linkerf(char *fmt, ...) return is_ok; } -internal B32 -t_invoke_radbin(char *fmt, ...) -{ - Temp scratch = scratch_begin(0,0); - va_list args; - va_start(args, fmt); - String8 cmdl = push_str8fv(scratch.arena, fmt, args); - va_end(args); - B32 is_ok = t_invoke(t_radbin_path(), cmdl, max_U64); - scratch_end(scratch); - return is_ok; -} - -internal void -t_kill_all(String8 pattern) -{ - Temp scratch = scratch_begin(0,0); - DMN_ProcessIter it = {0}; - dmn_process_iter_begin(&it); - DMN_ProcessInfo info = {0}; - while (dmn_process_iter_next(scratch.arena, &it, &info)) { - if (str8_match_wildcard(info.name, pattern, StringMatchFlag_CaseInsensitive|StringMatchFlag_SlashInsensitive)) { -#if OS_WINDOWS - if (!t_invoke(str8_lit("taskkill"), str8f(scratch.arena, "/PID %u /F", info.pid), max_U64)) { fprintf(stderr, "ERROR: failed to invoke taskkill\n"); } -#elif OS_LINUX - NotImplemented; // TODO: test - if (!t_invoke(str8_lit("kill"), str8f(scratch.arena, " -9 %u", info.pid), max_U64)) { fprintf(stderr, "ERROR: failed to invoke kill\n"); } -#else -# error NotImplemented -#endif - if (g_last_exit_code != 0) { fprintf(stderr, "ERROR: failed to kill %u\n", info.pid); } - } - } - dmn_process_iter_end(&it); - scratch_end(scratch); -} - -// TODO: obsolete -internal String8 -t_chop_line(String8 *string) -{ - return str8_chop_line(string); -} -// TODO: obsolete -internal B32 -t_match_line(String8 *output, String8 expected_line) -{ - return str8_match_wildcard(t_chop_line(output), expected_line, 0); -} - -internal B32 -t_match_linef(String8 *output, char *fmt, ...) -{ - Temp scratch = scratch_begin(0, 0); - va_list args; - va_start(args, fmt); - String8 expected_line = push_str8fv(scratch.arena, fmt, args); - B32 is_match = t_match_line(output, expected_line); - va_end(args); - scratch_end(scratch); - return is_match; -} - -force_inline int -t_test_info_is_before(TestInfo **a, TestInfo **b) +internal inline int +t_test_info_is_before(void *raw_a, void *raw_b) { + TestInfo **a = raw_a; + TestInfo **b = raw_b; String8 layer_a = a[0]->layer; String8 layer_b = b[0]->layer; int cmp = str8_compar(layer_a, layer_b, 0); - if(cmp == 0) - { - cmp = u64_compar(&a[0]->decl_line, &b[0]->decl_line); + if (cmp == 0) { + cmp = str8_compar(a[0]->label, b[0]->label, 0); + if (cmp == 0) { + cmp = u64_compar(&a[0]->decl_line, &b[0]->decl_line); + } } - return cmp; + return cmp < 0; } internal String8List @@ -1073,6 +978,85 @@ t_errorf(char *fmt, ...) scratch_end(scratch); } +//////////////////////////////// + +internal void +t_run_script_test(Arena *arena, TestCtx *ctx) +{ + T_Result run_status = {0}; + + // read script file + String8 path = ctx->user_data; + String8 source = data_from_file_path(arena, path); + if (source.size == 0) { test_outf("failed to read script: '%S'\n", path); } + T_Ok(source.size != 0); + + // parse the script + T_Context script = {0}; + run_status = t_script_parse(arena, ctx, &t_codec_script_suite, path, source, &script); + if (!t_result_is_ok(run_status)) { goto exit; } + + // execute the script if parsed + run_status = t_script_execute(&script); + if (!t_result_is_ok(run_status)) { goto exit; } + + // report run diagnostics + exit:; + for EachNode(diagnostic, T_Diagnostic, run_status.diagnostics.first) { + test_outf("%S:%lld:%lld [%S]: %S\n", + diagnostic->file_path, + diagnostic->location.line, + diagnostic->location.column, + diagnostic->operation, + diagnostic->message); + } + + T_Ok(t_result_is_ok(run_status)); +} + +internal void +t_register_scripts(void) +{ + local_persist B32 registered; + if (registered) { return; } + registered = 1; + + Temp scratch = scratch_begin(0,0); + Arena *arena = arena_alloc(.name="Torture Scripts Arena"); + + FileIter *iter = file_iter_begin(scratch.arena, t_src_path(), FileIterFlag_SkipFiles | FileIterFlag_SkipHiddenFiles); + for (FileInfo file = {0}; file_iter_next(scratch.arena, iter, &file); ) { + // skip third party folder because other projects can freely reserve the test extension + if (str8_matchi(file.name, str8_lit("third_party"))) { continue; } + + String8 scripts_dir = str8f(arena, "%S/%S/tests", t_src_path(), file.name); + String8List files = t_file_paths_from_dir(arena, scripts_dir); + String8Array paths = str8_array_from_list(arena, &files); + t_sort_str8_array(paths); + + for EachIndex(i, paths.count) { + if ( ! str8_matchi(str8_skip_last_dot(paths.v[i]), str8_lit("tst"))) { continue; } + + String8 layer = test_layer_from_file_path(paths.v[i]); + if (layer.size == 0) { + layer = str8_lit(""); + } + + TestInfo *test = &test_infos[test_infos_count++]; + test->layer = layer; + test->label = str8_chop_last_dot(str8_skip_last_slash(paths.v[i])); + test->decl_line = (S64)i; + test->test_fn = t_run_script_test; + test->user_data = paths.v[i]; + } + } + file_iter_end(iter); + + scratch_end(scratch); +} + +//////////////////////////////// + internal void t_help(void) { @@ -1104,8 +1088,10 @@ internal void t_entry_point(CmdLine *cmdline) { Temp scratch = scratch_begin(0,0); + Log *log = log_alloc(); log_select(log); + U64 exit_code = max_U64; U64 dashes_size = 9999; @@ -1125,17 +1111,20 @@ t_entry_point(CmdLine *cmdline) // { B32 print_help = cmd_line_has_flag(cmdline, str8_lit("help")) || - cmd_line_has_flag(cmdline, str8_lit("h")); + cmd_line_has_flag(cmdline, str8_lit("h")); if (print_help) { t_help(); goto exit; } } + + // Register Test Scripts + t_register_scripts(); // Gather tests { for EachIndex(i, test_infos_count) { g_sorted_test_infos[i] = &test_infos[i]; } - radsort(g_sorted_test_infos, test_infos_count, (int (*)(void *, void *))t_test_info_is_before); + radsort(g_sorted_test_infos, test_infos_count, t_test_info_is_before); } // @@ -1282,14 +1271,28 @@ t_entry_point(CmdLine *cmdline) // // Run tests // - { - U64Array target_indices = u64_array_from_list(scratch.arena, &targets); + { + // sort selected tests + U64Array target_indices = {0}; + { + U64 count = 0; + TestInfo **v = push_array(scratch.arena, TestInfo*, targets.count); + for EachNode(n, U64Node, targets.first) { + v[count++] = g_sorted_test_infos[n->data]; + } + radsort(v, count, t_test_info_is_before); + + target_indices.v = push_array(scratch.arena, U64, count); + for EachIndex(i, count) { + target_indices.v[target_indices.count++] = IntFromPtr(v[i] - &test_infos[0]); + } + } U64 max_label_size = 0; U64 max_layer_size = 0; for EachIndex(i, target_indices.count) { - U64 test_idx = target_indices.v[i]; - TestInfo *test_info = g_sorted_test_infos[test_idx]; + U64 test_idx = target_indices.v[i]; + TestInfo *test_info = &test_infos[test_idx]; max_label_size = Max(max_label_size, test_info->label.size); max_layer_size = Max(max_layer_size, test_info->layer.size); } @@ -1308,7 +1311,7 @@ t_entry_point(CmdLine *cmdline) if (i == 0) { PrintHeader("Tests"); } U64 target_idx = target_indices.v[i]; - TestInfo *test = g_sorted_test_infos[target_idx]; + TestInfo *test = &test_infos[target_idx]; // print run progress U64 dots_min = 10; @@ -1351,7 +1354,7 @@ t_entry_point(CmdLine *cmdline) // run test U64 run_start_time = now_time_us(); - TestResult result = t_run(cmdline, test, str8_zero()); + TestResult result = t_run(cmdline, test, test->user_data); U64 run_end_time = now_time_us(); // update @@ -1436,8 +1439,8 @@ t_entry_point(CmdLine *cmdline) U64 layer_max = 0; for EachIndex(i, slow_count) { Slowest s = slowest[i]; - label_max = Max(g_sorted_test_infos[s.target_idx]->label.size, label_max); - layer_max = Max(g_sorted_test_infos[s.target_idx]->layer.size, layer_max); + label_max = Max(test_infos[s.target_idx].label.size, label_max); + layer_max = Max(test_infos[s.target_idx].layer.size, layer_max); } fprintf(stderr, " \nSlow Tests\n"); @@ -1445,7 +1448,7 @@ t_entry_point(CmdLine *cmdline) Slowest s = slowest[i]; if (s.target_idx >= test_infos_count) { break; } - TestInfo *test_info = g_sorted_test_infos[s.target_idx]; + TestInfo *test_info = &test_infos[s.target_idx]; String8 elapsed_time = string_from_elapsed_time(scratch.arena, date_time_from_micro_seconds(s.d)); fprintf(stderr, " %.*s %.*s/ %.*s %.*s %.*s\n", diff --git a/src/torture/torture.h b/src/torture/torture.h index 56578809d..eb5685061 100644 --- a/src/torture/torture.h +++ b/src/torture/torture.h @@ -46,7 +46,6 @@ internal void t_run_fail_handler(void *raw_ctx); internal TestResult t_run(CmdLine *cmdline, TestInfo *test, String8 user_data); // tools -internal String8 t_radbin_path(void); internal String8 t_cl_path(void); internal String8 t_clang_path(void); internal String8 t_gcc_path(void); @@ -58,17 +57,10 @@ internal String8 t_src_path(void); internal B32 t_invoke(String8 exe, String8 cmdline, U64 timeout); internal B32 t_invoke_cl(char *fmt, ...); internal B32 t_invoke_linkerf(char *fmt, ...); -internal B32 t_invoke_radbin(char *fmt, ...); -internal void t_kill_all(String8 pattern); #define t_invoke_linker_timeout(c, t) T_Ok(t_invoke(t_radlink_path(), c, t)) #define t_invoke_linker_timeoutf(t, f, ...) t_invoke_linker_timeout(push_str8f(arena, f, ##__VA_ARGS__), t) #define t_invoke_linker(c) t_invoke_linker_timeout(c, max_U64) -internal String8 t_chop_line(String8 *string); -internal B32 t_match_line(String8 *output, String8 expected_line); -internal B32 t_match_linef(String8 *output, char *fmt, ...); -#define T_MatchLinef(out, ...) T_Ok(t_match_linef(out, __VA_ARGS__)) - // files helper internal String8List t_file_paths_from_dir(Arena *arena, String8 dir); diff --git a/src/torture/torture_base.c b/src/torture/torture_base.c new file mode 100644 index 000000000..f848d9e07 --- /dev/null +++ b/src/torture/torture_base.c @@ -0,0 +1,107 @@ +//////////////////////////////// +// Generic Parser Helpers + +internal B32 +t_md_node_is_valid(MD_Node *node) +{ + return !md_node_is_nil(node); +} + +internal B32 +t_bool_is_string_true(String8 string) +{ + return str8_matchi(string, str8_lit("true")) || str8_matchi(string, str8_lit("1")); +} + +internal B32 +t_bool_is_string_false(String8 string) +{ + return str8_matchi(string, str8_lit("false")) || str8_matchi(string, str8_lit("0")); +} + +internal B32 +t_bool_from_string(String8 string) +{ + if (str8_matchi(string, str8_lit("false"))) { + return 0; + } + return (B32)u32_from_str8(string, 10); +} + +internal String8 +t_hex_from_data(Arena *arena, String8 data) +{ + U64 hex_size = data.size * 2; + U8 *hex = push_array_no_zero(arena, U8, hex_size); + for EachIndex(i, data.size) { + local_persist U8 digits[] = "0123456789abcdef"; + hex[i * 2 + 0] = digits[data.str[i] >> 4]; + hex[i * 2 + 1] = digits[data.str[i] & 15]; + } + return str8(hex, hex_size); +} + +//////////////////////////////// +// Codec Helpers + +internal MD_Node * +t_codec_child(MD_Node *node, char *name) +{ + return md_child_from_string(node, str8_cstring(name), StringMatchFlag_CaseInsensitive); +} + +internal String8 +t_codec_scalar(MD_Node *node) +{ + return md_node_is_nil(node) || md_node_is_nil(node->first) ? str8_zero() : node->first->string; +} + +internal MD_Node * +t_codec_push_node(Arena *arena, MD_Node *parent, String8 string) +{ + MD_Node *node = md_push_node(arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, string, string, 0); + if (parent != 0) { md_node_push_child(parent, node); } + return node; +} + +internal MD_Node * +t_codec_push_field(T_Context *ctx, MD_Node *parent, char *name, String8 value) +{ + MD_Node *field = t_codec_push_node(ctx->arena, parent, str8_cstring(name)); + t_codec_push_node(ctx->arena, field, str8_copy(ctx->arena, value)); + return field; +} + +internal MD_Node * +t_codec_push_u64(T_Context *ctx, MD_Node *parent, char *name, U64 value) +{ + return t_codec_push_field(ctx, parent, name, str8f(ctx->arena, "%llu", value)); +} + +internal MD_Node * +t_codec_push_u64_s8(T_Context *ctx, MD_Node *parent, String8 name, U64 value) +{ + MD_Node *field = t_codec_push_node(ctx->arena, parent, name); + t_codec_push_node(ctx->arena, field, str8f(ctx->arena, "%llu", value)); + return field; +} + +internal B32 +t_bool_from_scalar(MD_Node *node, B32 *value_out) +{ + String8 value = t_codec_scalar(node); + + if (t_bool_is_string_true(value)) { + *value_out = 1; + return 1; + } + + if (t_bool_is_string_false(value)) { + *value_out = 0; + return 1; + } + + return 0; +} + + diff --git a/src/torture/torture_build.c b/src/torture/torture_build.c new file mode 100644 index 000000000..674fd3a60 --- /dev/null +++ b/src/torture/torture_build.c @@ -0,0 +1,757 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +global T_OpSpec t_op_repeat = {str8_lit_comp("repeat"), t_op_repeat_validate, t_op_repeat_execute}; +global T_OpSpec t_op_compare_file = {str8_lit_comp("compare_file"), t_op_compare_file_validate, t_op_compare_file_execute}; + +internal B32 +t_build_string_is_bool(String8 string) +{ + return t_bool_from_string(string) || str8_matchi(string, str8_lit("false")) || str8_match(string, str8_lit("0"), 0); +} + +internal B32 +t_build_safe_relative_path(String8 path) +{ + if (path.size == 0 || path.str[0] == '/' || path.str[0] == '\\' || str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { return 0; } + for (U64 off = 0; off < path.size;) { + U64 end = off; + while (end < path.size && path.str[end] != '/' && path.str[end] != '\\') { end += 1; } + String8 part = str8_substr(path, rng_1u64(off, end)); + if (str8_match(part, str8_lit(".."), 0)) { return 0; } + off = end + 1; + } + return 1; +} + +internal T_BuildCommandKind +t_build_command_kind_from_string(String8 string) +{ + if (str8_matchi(string, str8_lit("compile"))) { return T_BuildCommandKind_Compile; } + if (str8_matchi(string, str8_lit("compile_link"))) { return T_BuildCommandKind_CompileLink; } + if (str8_matchi(string, str8_lit("link"))) { return T_BuildCommandKind_Link; } + if (str8_matchi(string, str8_lit("resource"))) { return T_BuildCommandKind_Resource; } + if (str8_matchi(string, str8_lit("run"))) { return T_BuildCommandKind_Run; } + if (str8_matchi(string, str8_lit("copy"))) { return T_BuildCommandKind_Copy; } + return T_BuildCommandKind_Null; +} + +internal String8 +t_build_default_output(Arena *arena, String8 target_name, String8 target_kind, OperatingSystem os) +{ + if (target_name.size == 0) { target_name = str8_lit("main"); } + if (str8_matchi(target_kind, str8_lit("shared_library"))) { + return os == OperatingSystem_Windows ? str8f(arena, "%S.dll", target_name) : str8f(arena, "lib%S.so", target_name); + } + return os == OperatingSystem_Windows ? str8f(arena, "%S.exe", target_name) : str8_copy(arena, target_name); +} + +internal void +t_build_declaration_list_parse(Arena *arena, MD_Node *container, T_BuildDeclarationList *list) +{ + for MD_EachNode(node, container->first) { + T_BuildDeclaration *declaration = push_array(arena, T_BuildDeclaration, 1); + declaration->name = node->string; + declaration->definition = node; + SLLQueuePush(list->first, list->last, declaration); + list->count += 1; + } +} + +internal T_BuildOutput * +t_build_output_push(Arena *arena, T_BuildVariant *variant, String8 name, String8 path, MD_Node *definition) +{ + T_BuildOutput *output = push_array(arena, T_BuildOutput, 1); + output->name = str8_copy(arena, name); + output->path = str8_copy(arena, path); + output->definition = definition; + SLLQueuePush(variant->first_output, variant->last_output, output); + variant->output_count += 1; + return output; +} + +internal T_BuildOutput * +t_build_primary_output(T_BuildVariant *variant) +{ + for (T_BuildOutput *output = variant->first_output; output != 0; output = output->next) { + if (str8_matchi(output->name, str8_lit("executable"))) { return output; } + } + return variant->first_output; +} + +internal String8 +t_build_command_scalar(MD_Node *command) +{ + if (md_node_is_nil(command->first) || !md_node_is_nil(command->first->first)) { return str8_zero(); } + return command->first->string; +} + +internal T_BuildCommand * +t_build_command_parse(T_ParseContext *ctx, MD_Node *node) +{ + T_BuildCommandKind kind = t_build_command_kind_from_string(node->string); + if (kind == T_BuildCommandKind_Null) { return 0; } + + T_BuildCommand *command = push_array(ctx->arena, T_BuildCommand, 1); + command->kind = kind; + command->definition = node; + command->repeat_count = 1; + command->timeout_ms = max_U64; + command->expected_exit = str8_lit("0"); + command->output_mode = T_BuildOutputMode_Default; + + String8 scalar = t_build_command_scalar(node); + if (scalar.size != 0) { + command->arguments = scalar; + command->inject_output = 1; + return command; + } + + char *allowed[] = {"args", "tool", "output", "artifact", "expect_exit", "when_previous_exit", "timeout_ms", "stdout_matches", "stderr_matches", "index", "repeat", "parallel", "produces", 0}; + for MD_EachNode(field, node->first) { + B32 is_allowed = 0; + for (U64 i = 0; allowed[i] != 0; i += 1) { is_allowed |= str8_matchi(field->string, str8_cstring(allowed[i])); } + if (!is_allowed) { t_parse_errorf(ctx, T_ResultCode_ValidationError, field, "unknown build command field '%S'", field->string); } + for (MD_Node *other = field->next; !md_node_is_nil(other); other = other->next) { + if (str8_matchi(field->string, other->string)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, other, "duplicate build command field '%S'", other->string); + break; + } + } + } + + MD_Node *args = t_child_from_string(node, "args"); + command->arguments = t_scalar_string_from_node(args); + command->tool = t_scalar_string_from_node(t_child_from_string(node, "tool")); + MD_Node *output = t_child_from_string(node, "output"); + if (!md_node_is_nil(output)) { + command->output = t_scalar_string_from_node(output); + command->output_mode = str8_matchi(command->output, str8_lit("none")) ? T_BuildOutputMode_None : T_BuildOutputMode_Explicit; + command->inject_output = command->output_mode != T_BuildOutputMode_None; + } + command->artifact = t_scalar_string_from_node(t_child_from_string(node, "artifact")); + if (command->artifact.size != 0 && command->output_mode != T_BuildOutputMode_Default) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "build command cannot contain both artifact and output"); + } + command->expected_exit = t_scalar_string_from_node(t_child_from_string(node, "expect_exit")); + if (command->expected_exit.size == 0) { command->expected_exit = str8_lit("0"); } + command->when_previous_exit = t_scalar_string_from_node(t_child_from_string(node, "when_previous_exit")); + command->stdout_pattern = t_scalar_string_from_node(t_child_from_string(node, "stdout_matches")); + command->stderr_pattern = t_scalar_string_from_node(t_child_from_string(node, "stderr_matches")); + command->index_name = t_scalar_string_from_node(t_child_from_string(node, "index")); + MD_Node *parallel = t_child_from_string(node, "parallel"); + String8 parallel_string = t_scalar_string_from_node(parallel); + if (!md_node_is_nil(parallel) && !t_build_string_is_bool(parallel_string)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, parallel, "build command parallel must be true or false"); + } + command->parallel = t_bool_from_string(parallel_string); + command->produces = t_child_from_string(node, "produces"); + + MD_Node *repeat = t_child_from_string(node, "repeat"); + if (!md_node_is_nil(repeat) && !try_u64_from_str8_c_rules(t_scalar_string_from_node(repeat), &command->repeat_count)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, repeat, "build command repeat must be an integer"); + } + MD_Node *timeout = t_child_from_string(node, "timeout_ms"); + if (!md_node_is_nil(timeout) && !try_u64_from_str8_c_rules(t_scalar_string_from_node(timeout), &command->timeout_ms)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, timeout, "build command timeout_ms must be an integer"); + } + if (command->arguments.size == 0 && kind != T_BuildCommandKind_Copy) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "build command '%S' requires args", node->string); + } + U64 expected_exit_code = 0; + if (!str8_matchi(command->expected_exit, str8_lit("any")) && !str8_matchi(command->expected_exit, str8_lit("nonzero")) && + !try_u64_from_str8_c_rules(command->expected_exit, &expected_exit_code)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_child_from_string(node, "expect_exit"), "invalid expected build command exit code"); + } + if (command->when_previous_exit.size != 0 && !str8_matchi(command->when_previous_exit, str8_lit("any")) && + !str8_matchi(command->when_previous_exit, str8_lit("nonzero")) && !try_u64_from_str8_c_rules(command->when_previous_exit, &expected_exit_code)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_child_from_string(node, "when_previous_exit"), "invalid previous build command exit code"); + } + if (command->tool.size != 0 && !str8_matchi(command->tool, str8_lit("cl")) && !str8_matchi(command->tool, str8_lit("msvc")) && + !str8_matchi(command->tool, str8_lit("clang")) && !str8_matchi(command->tool, str8_lit("gcc")) && !str8_matchi(command->tool, str8_lit("radlink"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_child_from_string(node, "tool"), "unknown build command tool '%S'", command->tool); + } + if (command->repeat_count == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, repeat, "build command repeat must be greater than zero"); } + if (command->repeat_count > 1 && command->index_name.size == 0) { command->index_name = str8_lit("index"); } + return command; +} + +internal T_BuildVariant * +t_build_variant_parse(T_ParseContext *ctx, MD_Node *body, OperatingSystem os, String8 target_name, String8 target_kind) +{ + T_BuildVariant *variant = push_array(ctx->arena, T_BuildVariant, 1); + variant->os = os; + variant->definition = body; + + MD_Node *output = t_child_from_string(body, "output"); + MD_Node *outputs = t_child_from_string(body, "outputs"); + if (!md_node_is_nil(output) && !md_node_is_nil(outputs)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, output, "build variant cannot contain both output and outputs"); + } else if (!md_node_is_nil(output)) { + String8 path = t_scalar_string_from_node(output); + if (str8_matchi(path, str8_lit("none"))) { variant->output_none = 1; } + else if (path.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, output, "build output requires one path"); } + else { t_build_output_push(ctx->arena, variant, str8_lit("executable"), path, output); } + } else if (!md_node_is_nil(outputs)) { + for MD_EachNode(node, outputs->first) { + String8 path = t_scalar_string_from_node(node); + if (path.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "build output '%S' requires one path", node->string); } + else { t_build_output_push(ctx->arena, variant, node->string, path, node); } + } + } else { + OperatingSystem output_os = os == OperatingSystem_Null ? OperatingSystem_CURRENT : os; + t_build_output_push(ctx->arena, variant, str8_lit("executable"), t_build_default_output(ctx->arena, target_name, target_kind, output_os), body); + } + + MD_Node *commands = t_child_from_string(body, "commands"); + MD_Node *first = md_node_is_nil(commands) ? body->first : commands->first; + for MD_EachNode(node, first) { + T_BuildCommand *command = t_build_command_parse(ctx, node); + if (command != 0) { + SLLQueuePush(variant->first_command, variant->last_command, command); + variant->command_count += 1; + } else if (!md_node_is_nil(commands)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown build command '%S'", node->string); + } else { + char *metadata[] = {"output", "outputs", "kind", "source", "sources", "inputs", "depends", "generator", "resources", "packages", "defines", "link_args", "platforms", 0}; + B32 is_metadata = 0; + for (U64 i = 0; metadata[i] != 0; i += 1) { is_metadata |= str8_matchi(node->string, str8_cstring(metadata[i])); } + if (!is_metadata) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown build declaration '%S'", node->string); } + } + } + return variant; +} + +internal void +t_build_target_parse_variants(T_ParseContext *ctx, T_BuildTarget *target, MD_Node *body) +{ + B32 has_platform = 0; + for MD_EachNode(node, body->first) { + OperatingSystem os = operating_system_from_string(node->string); + has_platform |= os == OperatingSystem_Windows || os == OperatingSystem_Linux; + } + + if (has_platform) { + B32 has_windows = 0; + B32 has_linux = 0; + for MD_EachNode(node, body->first) { + OperatingSystem os = operating_system_from_string(node->string); + if (os == OperatingSystem_Windows || os == OperatingSystem_Linux) { + B32 *seen = os == OperatingSystem_Windows ? &has_windows : &has_linux; + if (*seen) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "duplicate build platform '%S'", node->string); } + *seen = 1; + T_BuildVariant *variant = t_build_variant_parse(ctx, node, os, target->name, target->kind); + SLLQueuePush(target->first_variant, target->last_variant, variant); + target->variant_count += 1; + } else { + char *metadata[] = {"kind", "source", "sources", "inputs", "depends", "generator", "resources", "packages", "defines", "link_args", 0}; + B32 is_metadata = 0; + for (U64 i = 0; metadata[i] != 0; i += 1) { is_metadata |= str8_matchi(node->string, str8_cstring(metadata[i])); } + if (!is_metadata) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown build declaration '%S'", node->string); } + } + } + } else { + T_BuildVariant *variant = t_build_variant_parse(ctx, body, OperatingSystem_Null, target->name, target->kind); + SLLQueuePush(target->first_variant, target->last_variant, variant); + target->variant_count += 1; + } +} + +internal T_BuildTarget * +t_build_target_parse(T_ParseContext *ctx, MD_Node *node) +{ + T_BuildTarget *target = push_array(ctx->arena, T_BuildTarget, 1); + target->name = node->string; + target->definition = node; + target->kind = t_scalar_string_from_node(t_child_from_string(node, "kind")); + if (target->kind.size == 0) { target->kind = str8_lit("executable"); } + t_build_target_parse_variants(ctx, target, node); + return target; +} + +internal T_Result +t_build_parse(T_ParseContext *ctx, MD_Node *build, T_BuildPlan **plan_out) +{ + T_BuildPlan *plan = push_array(ctx->arena, T_BuildPlan, 1); + plan->definition = build; + *plan_out = plan; + + MD_Node *targets = t_child_from_string(build, "targets"); + if (md_node_is_nil(targets)) { + + T_BuildTarget *target = push_array(ctx->arena, T_BuildTarget, 1); + target->name = str8_lit("main"); + target->kind = str8_lit("executable"); + target->definition = build; + + t_build_target_parse_variants(ctx, target, build); + + plan->is_single_target = 1; + SLLQueuePush(plan->first_target, plan->last_target, target); + plan->target_count = 1; + + return ctx->run->result; + } + + char *allowed[] = {"defaults", "values", "configurations", "features", "toolchains", "packages", "resources", "generators", "targets", 0}; + for MD_EachNode(node, build->first) { + B32 is_allowed = 0; + for (U64 i = 0; allowed[i] != 0; i += 1) { is_allowed |= str8_matchi(node->string, str8_cstring(allowed[i])); } + if (!is_allowed) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown project build declaration '%S'", node->string); } + } + + plan->defaults = t_child_from_string(build, "defaults"); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "values"), &plan->values); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "configurations"), &plan->configurations); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "features"), &plan->features); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "toolchains"), &plan->toolchains); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "packages"), &plan->packages); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "resources"), &plan->resources); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "generators"), &plan->generators); + for MD_EachNode(node, targets->first) { + T_BuildTarget *target = t_build_target_parse(ctx, node); + SLLQueuePush(plan->first_target, plan->last_target, target); + plan->target_count += 1; + } + if (plan->target_count == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, targets, "project build requires at least one target"); } + return ctx->run->result; +} + +internal T_BuildVariant * +t_build_current_variant(T_BuildTarget *target) +{ + T_BuildVariant *fallback = 0; + for (T_BuildVariant *variant = target->first_variant; variant != 0; variant = variant->next) { + if (variant->os == OperatingSystem_CURRENT) { return variant; } + if (variant->os == OperatingSystem_Null) { fallback = variant; } + } + return fallback; +} + +internal String8 +t_script_expand(T_Context *ctx, MD_Node *node, String8 string, T_BuildVariant *variant, T_ScriptBinding *bindings) +{ + (void)node; + if (variant == 0 && ctx->build != 0 && ctx->build->first_target != 0) { variant = t_build_current_variant(ctx->build->first_target); } + Temp scratch = scratch_begin(&ctx->arena, 1); + HashMap variables = {0}; + for (T_ScriptBinding *binding = bindings != 0 ? bindings : ctx->bindings; binding != 0; binding = binding->next) { + lnk_env_var_push(scratch.arena, &variables, binding->name, binding->value, LNK_EnvVarRule_Current); + } + lnk_env_var_push(scratch.arena, &variables, str8_lit("source_dir"), t_src_path(), LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8_lit("work_dir"), g_wdir, LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8_lit("platform"), lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)), LNK_EnvVarRule_Current); + if (variant != 0) { + T_BuildOutput *primary = t_build_primary_output(variant); + if (primary != 0) { + lnk_env_var_push(scratch.arena, &variables, str8_lit("output"), primary->path, LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8_lit("build.output"), primary->path, LNK_EnvVarRule_Current); + } + for (T_BuildOutput *output = variant->first_output; output != 0; output = output->next) { + lnk_env_var_push(scratch.arena, &variables, str8f(scratch.arena, "outputs.%S", output->name), output->path, LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8f(scratch.arena, "build.outputs.%S", output->name), output->path, LNK_EnvVarRule_Current); + } + } + if (ctx->build != 0) { + for (T_BuildDeclaration *value = ctx->build->values.first; value != 0; value = value->next) { + lnk_env_var_push(scratch.arena, &variables, str8f(scratch.arena, "values.%S", value->name), t_scalar_string_from_node(value->definition), LNK_EnvVarRule_Current); + } + } + String8 result = lnk_expand_env_vars_windows(ctx->arena, &variables, string); + scratch_end(scratch); + return result; +} + +internal String8 +t_build_command_output(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding) +{ + if (command->output_mode == T_BuildOutputMode_None) { return str8_zero(); } + if (command->output_mode == T_BuildOutputMode_Explicit) { + return t_script_expand(ctx, command->definition, command->output, variant, binding); + } + if (command->artifact.size != 0) { + T_Artifact *artifact = t_artifact_from_name(ctx, command->artifact); + if (artifact == 0 || artifact->file_name.size == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "unknown output artifact '%S'", command->artifact); + return str8_zero(); + } + return artifact->file_name; + } + if (!command->inject_output) { return str8_zero(); } + T_BuildOutput *output = t_build_primary_output(variant); + return output == 0 ? str8_zero() : t_script_expand(ctx, output->definition, output->path, variant, binding); +} + +internal String8 +t_build_command_arguments(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding) +{ + String8 args = t_script_expand(ctx, command->definition, command->arguments, variant, binding); + String8 output = t_build_command_output(ctx, variant, command, binding); + if (output.size == 0 || !command->inject_output) { return args; } + OperatingSystem os = variant->os == OperatingSystem_Null ? OperatingSystem_CURRENT : variant->os; + B32 msvc_style = os == OperatingSystem_Windows && (command->tool.size == 0 || str8_matchi(command->tool, str8_lit("cl")) || str8_matchi(command->tool, str8_lit("msvc"))); + switch (command->kind) { + case T_BuildCommandKind_Compile: + return msvc_style ? str8f(ctx->arena, "%S /c /Fo:\"%S\"", args, output) : str8f(ctx->arena, "%S -c -o \"%S\"", args, output); + case T_BuildCommandKind_CompileLink: + return msvc_style ? str8f(ctx->arena, "%S /Fe:\"%S\"", args, output) : str8f(ctx->arena, "%S -o \"%S\"", args, output); + case T_BuildCommandKind_Link: + return str8f(ctx->arena, "%S /out:\"%S\"", args, output); + default: break; + } + return args; +} + +internal String8 +t_build_command_tool(T_BuildVariant *variant, T_BuildCommand *command) +{ + if (command->tool.size != 0) { + if (str8_matchi(command->tool, str8_lit("cl")) || str8_matchi(command->tool, str8_lit("msvc"))) { return t_cl_path(); } + if (str8_matchi(command->tool, str8_lit("clang"))) { return t_clang_path(); } + if (str8_matchi(command->tool, str8_lit("gcc"))) { return t_gcc_path(); } + if (str8_matchi(command->tool, str8_lit("radlink"))) { return t_radlink_path(); } + return str8_zero(); + } + switch (command->kind) { + case T_BuildCommandKind_Compile: + case T_BuildCommandKind_CompileLink: return OperatingSystem_CURRENT == OperatingSystem_Windows ? t_cl_path() : t_clang_path(); + case T_BuildCommandKind_Link: return t_radlink_path(); + default: break; + } + return str8_zero(); +} + +internal B32 +t_build_exit_matches(String8 expected, U64 actual) +{ + if (str8_matchi(expected, str8_lit("any"))) { return 1; } + if (str8_matchi(expected, str8_lit("nonzero"))) { return actual != 0; } + U64 value = 0; + return try_u64_from_str8_c_rules(expected, &value) && value == actual; +} + +internal T_Result +t_build_load_artifact_output(T_Context *ctx, String8 output_path) +{ + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (artifact->file_name.size != 0 && str8_match(artifact->file_name, output_path, StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive)) { + artifact->data = t_read_file(ctx->arena, output_path); + if (artifact->data.size == 0) { + artifact->state = T_ArtifactState_Failed; + return t_context_errorf(ctx, T_ResultCode_IoError, artifact->definition, str8_lit("build"), "unable to read output artifact '%S'", output_path); + } + artifact->state = T_ArtifactState_Materialized; + } + } + return ctx->result; +} + +internal T_Result +t_build_validate_command_result(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding, U64 exit_code, String8 stdout_data, String8 stderr_data) +{ + if (!t_build_exit_matches(command->expected_exit, exit_code)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "build command exited with %llu, expected %S\n%S", exit_code, + command->expected_exit, stderr_data); + } + StringMatchFlags flags = StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive; + if (command->stdout_pattern.size != 0 && !str8_match_wildcard(stdout_data, command->stdout_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "stdout does not match '%S'\n%S", command->stdout_pattern, stdout_data); + } + if (command->stderr_pattern.size != 0 && !str8_match_wildcard(stderr_data, command->stderr_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "stderr does not match '%S'\n%S", command->stderr_pattern, stderr_data); + } + String8 output = t_build_command_output(ctx, variant, command, binding); + if (output.size != 0 && exit_code == 0) { + String8 full_path = t_make_file_path(ctx->arena, output); + if (!file_path_exists(full_path)) { return t_context_errorf(ctx, T_ResultCode_IoError, command->definition, str8_lit("build"), "expected output '%S' was not produced", output); } + T_Result result = t_build_load_artifact_output(ctx, output); + if (!t_result_is_ok(result)) { return result; } + } + MD_Node *first_produced = command->produces == 0 ? &md_nil_node : command->produces->first; + for MD_EachNode(produced, first_produced) { + String8 path = t_script_expand(ctx, produced, produced->string, variant, binding); + if (!file_path_exists(t_make_file_path(ctx->arena, path))) { + return t_context_errorf(ctx, T_ResultCode_IoError, produced, str8_lit("build"), "expected output '%S' was not produced", path); + } + } + return ctx->result; +} + +internal void +t_build_remove_command_outputs(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding) +{ + String8 output = t_build_command_output(ctx, variant, command, binding); + if (output.size != 0) { + if (!t_build_safe_relative_path(output)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "output path '%S' must be work-relative", output); + return; + } + t_delete_file(output); + } + MD_Node *first_produced = command->produces == 0 ? &md_nil_node : command->produces->first; + for MD_EachNode(produced, first_produced) { + String8 path = t_script_expand(ctx, produced, produced->string, variant, binding); + if (path.size != 0) { + if (!t_build_safe_relative_path(path)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, produced, str8_lit("build"), "produced path '%S' must be work-relative", path); + return; + } + t_delete_file(path); + } + } +} + +internal T_Result +t_build_execute_command(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, U64 index, U64 *exit_code_out) +{ + T_ScriptBinding binding = {0}; + if (command->repeat_count > 1) { + binding.name = command->index_name; + binding.value = str8f(ctx->arena, "%llu", index); + } + t_build_remove_command_outputs(ctx, variant, command, &binding); + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + String8 tool = t_build_command_tool(variant, command); + if (tool.size == 0) { return t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "build command is not executable yet"); } + String8 args = t_build_command_arguments(ctx, variant, command, &binding); + U64 timeout_us = command->timeout_ms == max_U64 || command->timeout_ms > max_U64 / 1000 ? max_U64 : command->timeout_ms * 1000; + if (!t_invoke(tool, args, timeout_us)) { return t_context_errorf(ctx, T_ResultCode_IoError, command->definition, str8_lit("build"), "unable to launch '%S'", tool); } + *exit_code_out = g_last_exit_code; + if (g_last_exit_code == max_U64) { return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "build command did not exit before timeout"); } + return t_build_validate_command_result(ctx, variant, command, &binding, g_last_exit_code, g_output, g_errors); +} + +internal T_Result +t_build_execute_parallel(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, U64 *exit_code_out) +{ + U64 count = command->repeat_count; + T_Controller *controllers = push_array(ctx->arena, T_Controller, count); + T_ScriptBinding *bindings = push_array(ctx->arena, T_ScriptBinding, count); + + B32 launched_all = 1; + for EachIndex(index, count) { + bindings[index].name = command->index_name; + bindings[index].value = str8f(ctx->arena, "%llu", index); + t_build_remove_command_outputs(ctx, variant, command, &bindings[index]); + } + + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + for EachIndex(index, count) { + String8 tool = t_build_command_tool(variant, command); + String8 args = t_build_command_arguments(ctx, variant, command, &bindings[index]); + String8List cmd_line = lnk_arg_list_parse_windows_rules(ctx->arena, args); + str8_list_push_front(ctx->arena, &cmd_line, tool); + + T_ControllerLaunchParams params = { + .cmd_line = cmd_line, + .path = g_wdir, + .inherit_env = 1, + .consoleless = 1, + }; + + if (t_controller_launch(ctx->arena, &controllers[index], ¶ms).code != T_ControllerResultCode_Ok) { + launched_all = 0; + break; + } + } + + if (!launched_all) { + for EachIndex(index, count) { t_controller_close(&controllers[index]); } + return t_context_errorf(ctx, T_ResultCode_IoError, command->definition, str8_lit("build"), "unable to launch parallel build command"); + } + + U64 timeout_us = command->timeout_ms == max_U64 || command->timeout_ms > max_U64 / 1000 ? max_U64 : command->timeout_ms * 1000; + U64 endt_us = timeout_us == max_U64 ? max_U64 : now_time_us() + timeout_us; + B32 timed_out = 0; + + for (;;) { + + U64 exited_count = 0; + + for EachIndex(index, count) { + T_ControllerResult result = t_controller_pump(&controllers[index]); + if (result.code != T_ControllerResultCode_Ok) { + for EachIndex(close_index, count) { + t_controller_close(&controllers[close_index]); + } + + return t_context_errorf(ctx, + T_ResultCode_IoError, + command->definition, + str8_lit("build"), + "unable to capture parallel build command output"); + } + exited_count += controllers[index].exited; + } + + if (exited_count == count) { break; } + + // TODO: comprehensive solution to the waiting problem + if (endt_us != max_U64 && now_time_us() >= endt_us) { + timed_out = 1; + break; + } + sleep_ms(1); + } + + for EachIndex(index, count) { t_controller_close(&controllers[index]); } + + if (timed_out) { + return t_context_errorf(ctx, + T_ResultCode_Mismatch, + command->definition, + str8_lit("build"), + "parallel build command did not exit before timeout"); + } + + *exit_code_out = 0; + for EachIndex(index, count) { + if (controllers[index].exit_code != 0) { *exit_code_out = controllers[index].exit_code; } + T_Result result = t_build_validate_command_result(ctx, + variant, + command, + &bindings[index], + controllers[index].exit_code, + t_controller_stdout(ctx->arena, &controllers[index]), + t_controller_stderr(ctx->arena, &controllers[index])); + if (!t_result_is_ok(result)) { return result; } + } + + return ctx->result; +} + +internal T_Result +t_build_execute(T_Context *ctx) +{ + if (ctx->build == 0 || ctx->build->first_target == 0) { return ctx->result; } + + if (!ctx->build->is_single_target) { + return t_context_errorf(ctx, + T_ResultCode_ValidationError, + ctx->build->definition, + str8_lit("build"), + "project build declarations require target selection or a build-script codec"); + } + + T_BuildTarget *target = ctx->build->first_target; + T_BuildVariant *variant = t_build_current_variant(target); + + if (variant == 0) { + return t_context_errorf(ctx, + T_ResultCode_ValidationError, + target->definition, + str8_lit("build"), + "build target has no variant for the current platform"); + } + + U64 previous_exit = max_U64; + B32 has_previous_exit = 0; + for EachNode(command, T_BuildCommand, variant->first_command) { + if (command->when_previous_exit.size != 0) { + if (!has_previous_exit) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "conditional build command has no previous command"); + } + if (!t_build_exit_matches(command->when_previous_exit, previous_exit)) { continue; } + } + if (command->parallel && command->repeat_count > 1) { + T_Result result = t_build_execute_parallel(ctx, variant, command, &previous_exit); + if ( ! t_result_is_ok(result)) { return result; } + has_previous_exit = 1; + continue; + } + + for EachIndex(index, command->repeat_count) { + T_Result result = t_build_execute_command(ctx, variant, command, index, &previous_exit); + if ( ! t_result_is_ok(result)) { return result; } + has_previous_exit = 1; + } + } + + return ctx->result; +} + +internal T_Result +t_op_repeat_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + MD_Node *count = t_child_from_string(arguments, "count"); + MD_Node *steps = t_child_from_string(arguments, "steps"); + U64 value = 0; + if (md_node_is_nil(count) || !try_u64_from_str8_c_rules(t_scalar_string_from_node(count), &value) || value == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, count, "repeat requires a positive integer count"); + } + if (md_node_is_nil(steps)) { t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "repeat requires steps"); } + MD_Node *index = t_child_from_string(arguments, "index"); + if (!md_node_is_nil(index) && t_scalar_string_from_node(index).size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, index, "repeat index requires a name"); } + for MD_EachNode(node, steps->first) { + T_OpSpec *spec = t_op_spec_from_name(ctx->suite, node->string); + if (spec == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown repeated operation '%S'", node->string); } + else if (spec->validate != 0) { + String8 saved = ctx->operation; + ctx->operation = spec->name; + T_Result result = spec->validate(ctx, node); + t_context_absorb_result(ctx->run, result); + ctx->operation = saved; + } + } + return ctx->run->result; +} + +internal T_Result +t_op_repeat_execute(T_Context *ctx, MD_Node *arguments) +{ + U64 count = 0; + try_u64_from_str8_c_rules(t_scalar_string_from_node(t_child_from_string(arguments, "count")), &count); + String8 index_name = t_scalar_string_from_node(t_child_from_string(arguments, "index")); + if (index_name.size == 0) { index_name = str8_lit("index"); } + MD_Node *steps = t_child_from_string(arguments, "steps"); + for (U64 index = 0; index < count; index += 1) { + T_ScriptBinding binding = {.next = ctx->bindings, .name = index_name, .value = str8f(ctx->arena, "%llu", index)}; + T_ScriptBinding *saved = ctx->bindings; + ctx->bindings = &binding; + for MD_EachNode(node, steps->first) { + T_OpSpec *spec = t_op_spec_from_name(ctx->suite, node->string); + T_Result result = spec->execute(ctx, node); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { break; } + } + ctx->bindings = saved; + if (!t_result_is_ok(ctx->result)) { break; } + } + return ctx->result; +} + +internal T_Result +t_op_compare_file_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + String8 left = t_scalar_string_from_node(t_child_from_string(arguments, "left")); + String8 right = t_scalar_string_from_node(t_child_from_string(arguments, "right")); + if (left.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare_file requires left"); } + if (right.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare_file requires right"); } + return ctx->run->result; +} + +internal T_Result +t_op_compare_file_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 left = t_script_expand(ctx, arguments, t_scalar_string_from_node(t_child_from_string(arguments, "left")), 0, 0); + String8 right = t_script_expand(ctx, arguments, t_scalar_string_from_node(t_child_from_string(arguments, "right")), 0, 0); + if (!t_build_safe_relative_path(left) || !t_build_safe_relative_path(right)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, arguments, str8_lit("compare_file"), "paths must be work-relative"); + } + String8 left_path = t_make_file_path(ctx->arena, left); + String8 right_path = t_make_file_path(ctx->arena, right); + if (!file_path_exists(left_path)) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("compare_file"), "file '%S' does not exist", left); } + if (!file_path_exists(right_path)) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("compare_file"), "file '%S' does not exist", right); } + String8 left_data = t_read_file(ctx->arena, left); + String8 right_data = t_read_file(ctx->arena, right); + if (!str8_match(left_data, right_data, 0)) { + U64 mismatch = 0; + U64 common_size = Min(left_data.size, right_data.size); + while (mismatch < common_size && left_data.str[mismatch] == right_data.str[mismatch]) { mismatch += 1; } + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("compare_file"), "files '%S' and '%S' differ at byte %llu (sizes %llu and %llu)", left, right, + mismatch, left_data.size, right_data.size); + } + return ctx->result; +} diff --git a/src/torture/torture_build.h b/src/torture/torture_build.h new file mode 100644 index 000000000..916e3c149 --- /dev/null +++ b/src/torture/torture_build.h @@ -0,0 +1,134 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +typedef enum T_BuildCommandKind +{ + T_BuildCommandKind_Null, + T_BuildCommandKind_Compile, + T_BuildCommandKind_CompileLink, + T_BuildCommandKind_Link, + T_BuildCommandKind_Resource, + T_BuildCommandKind_Run, + T_BuildCommandKind_Copy, +} T_BuildCommandKind; + +typedef enum T_BuildOutputMode +{ + T_BuildOutputMode_Default, + T_BuildOutputMode_Explicit, + T_BuildOutputMode_None, +} T_BuildOutputMode; + +typedef struct T_BuildDeclaration T_BuildDeclaration; +struct T_BuildDeclaration +{ + T_BuildDeclaration *next; + String8 name; + MD_Node *definition; +}; + +typedef struct T_BuildDeclarationList T_BuildDeclarationList; +struct T_BuildDeclarationList +{ + T_BuildDeclaration *first; + T_BuildDeclaration *last; + U64 count; +}; + +typedef struct T_BuildOutput T_BuildOutput; +struct T_BuildOutput +{ + T_BuildOutput *next; + String8 name; + String8 path; + MD_Node *definition; +}; + +typedef struct T_BuildCommand T_BuildCommand; +struct T_BuildCommand +{ + T_BuildCommand *next; + T_BuildCommandKind kind; + String8 tool; + String8 arguments; + String8 artifact; + String8 output; + T_BuildOutputMode output_mode; + String8 index_name; + U64 repeat_count; + U64 timeout_ms; + String8 expected_exit; + String8 when_previous_exit; + String8 stdout_pattern; + String8 stderr_pattern; + MD_Node *produces; + MD_Node *definition; + B32 parallel; + B32 inject_output; +}; + +typedef struct T_BuildVariant T_BuildVariant; +struct T_BuildVariant +{ + T_BuildVariant *next; + OperatingSystem os; + T_BuildOutput *first_output; + T_BuildOutput *last_output; + U64 output_count; + T_BuildCommand *first_command; + T_BuildCommand *last_command; + U64 command_count; + MD_Node *definition; + B32 output_none; +}; + +typedef struct T_BuildTarget T_BuildTarget; +struct T_BuildTarget +{ + T_BuildTarget *next; + String8 name; + String8 kind; + T_BuildVariant *first_variant; + T_BuildVariant *last_variant; + U64 variant_count; + MD_Node *definition; +}; + +struct T_BuildPlan +{ + B32 is_single_target; + MD_Node *definition; + MD_Node *defaults; + T_BuildDeclarationList values; + T_BuildDeclarationList configurations; + T_BuildDeclarationList features; + T_BuildDeclarationList toolchains; + T_BuildDeclarationList packages; + T_BuildDeclarationList resources; + T_BuildDeclarationList generators; + T_BuildTarget *first_target; + T_BuildTarget *last_target; + U64 target_count; +}; + +typedef struct T_ScriptBinding T_ScriptBinding; +struct T_ScriptBinding +{ + T_ScriptBinding *next; + String8 name; + String8 value; +}; + +internal T_Result t_build_parse(T_ParseContext *ctx, MD_Node *build, T_BuildPlan **plan_out); +internal T_Result t_build_execute(T_Context *ctx); +internal String8 t_script_expand(T_Context *ctx, MD_Node *node, String8 string, T_BuildVariant *variant, T_ScriptBinding *bindings); + +internal T_Result t_op_repeat_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_op_repeat_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_op_compare_file_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_op_compare_file_execute(T_Context *ctx, MD_Node *arguments); + +global T_OpSpec t_op_repeat; +global T_OpSpec t_op_compare_file; diff --git a/src/torture/torture_coff.c b/src/torture/torture_coff.c new file mode 100644 index 000000000..4b326ee76 --- /dev/null +++ b/src/torture/torture_coff.c @@ -0,0 +1,931 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal MD_Node * +t_coff_field(T_ParseContext *ctx, MD_Node *parent, char *name) +{ + MD_Node *result = &md_nil_node; + U64 count = 0; + for + MD_EachNode(child, parent->first) + { + if (str8_match(child->string, str8_cstring(name), StringMatchFlag_CaseInsensitive)) { + if (count == 0) { result = child; } + count += 1; + } + } + if (count > 1) { t_parse_errorf(ctx, T_ResultCode_ValidationError, result, "field '%s' may only appear once", name); } + return result; +} + +internal B32 +t_coff_name_is_allowed(String8 name, char **allowed) +{ + for (char **item = allowed; *item != 0; item += 1) { + if (str8_match(name, str8_cstring(*item), StringMatchFlag_CaseInsensitive)) { return 1; } + } + return 0; +} + +internal void +t_coff_reject_unknown(T_ParseContext *ctx, MD_Node *node, char **allowed) +{ + for + MD_EachNode(child, node->first) + { + if (!t_coff_name_is_allowed(child->string, allowed)) { t_parse_errorf(ctx, T_ResultCode_ValidationError, child, "unknown field '%S'", child->string); } + } +} + +internal B32 +t_coff_scalar(T_ParseContext *ctx, MD_Node *parent, char *name, B32 required, String8 *value_out) +{ + MD_Node *field = t_coff_field(ctx, parent, name); + if (md_node_is_nil(field)) { + if (required) { t_parse_errorf(ctx, T_ResultCode_ValidationError, parent, "missing field '%s'", name); } + return !required; + } + if (md_node_is_nil(field->first) || !md_node_is_nil(field->first->next)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, field, "field '%s' requires exactly one value", name); + return 0; + } + *value_out = field->first->string; + return 1; +} + +internal B32 +t_coff_u64(T_ParseContext *ctx, MD_Node *parent, char *name, B32 required, U64 default_value, U64 max_value, U64 *value_out) +{ + if (md_node_is_nil(t_coff_field(ctx, parent, name))) { + *value_out = default_value; + if (required) { t_parse_errorf(ctx, T_ResultCode_ValidationError, parent, "missing field '%s'", name); } + return !required; + } + String8 string = {0}; + if (!t_coff_scalar(ctx, parent, name, required, &string)) { + *value_out = default_value; + return !required; + } + U64 value = 0; + if (!try_u64_from_str8_c_rules(string, &value) || value > max_value) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, parent, name), "field '%s' must be an integer no greater than %llu", name, max_value); + return 0; + } + *value_out = value; + return 1; +} + +internal B32 +t_coff_bool(T_ParseContext *ctx, MD_Node *parent, char *name, B32 default_value, B32 *value_out) +{ + if (md_node_is_nil(t_coff_field(ctx, parent, name))) { + *value_out = default_value; + return 1; + } + String8 string = {0}; + if (!t_coff_scalar(ctx, parent, name, 0, &string)) { + *value_out = default_value; + return 1; + } + if (str8_match(string, str8_lit("true"), StringMatchFlag_CaseInsensitive)) { + *value_out = 1; + } else if (str8_match(string, str8_lit("false"), StringMatchFlag_CaseInsensitive)) { + *value_out = 0; + } else { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, parent, name), "field '%s' must be true or false", name); + return 0; + } + return 1; +} + +internal B32 +t_coff_machine(T_ParseContext *ctx, MD_Node *parent, COFF_MachineType *machine_out) +{ + String8 string = {0}; + if (!t_coff_scalar(ctx, parent, "machine", 1, &string)) { return 0; } + COFF_MachineType machine = coff_machine_from_string(string); + if (machine == COFF_MachineType_Unknown && !str8_match(string, str8_lit("unknown"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, parent, "machine"), "unknown COFF machine '%S'", string); + return 0; + } + *machine_out = machine; + return 1; +} + +internal T_COFF_Section * +t_coff_section_from_id(T_COFF_Object *object, String8 id) +{ + for (T_COFF_Section *section = object->first_section; section != 0; section = section->next) { + if (str8_match(section->id, id, 0)) { return section; } + } + return 0; +} + +internal T_COFF_Symbol * +t_coff_symbol_from_id(T_COFF_Object *object, String8 id) +{ + for + EachNode(symbol, T_COFF_Symbol, object->first_symbol) + { + if (str8_match(symbol->id, id, 0)) { return symbol; } + } + return 0; +} + +internal COFF_SectionFlags +t_coff_permission_flag(String8 string) +{ + if (str8_match(string, str8_lit("read"), StringMatchFlag_CaseInsensitive)) { return COFF_SectionFlag_MemRead; } + if (str8_match(string, str8_lit("write"), StringMatchFlag_CaseInsensitive)) { return COFF_SectionFlag_MemWrite; } + if (str8_match(string, str8_lit("execute"), StringMatchFlag_CaseInsensitive)) { return COFF_SectionFlag_MemExecute; } + return 0; +} + +internal COFF_SectionFlags +t_coff_section_flag(String8 string) +{ + struct Map + { + char *name; + COFF_SectionFlags flag; + }; + local_persist struct Map map[] = { + {"discardable", COFF_SectionFlag_MemDiscardable}, {"not_cached", COFF_SectionFlag_MemNotCached}, {"not_paged", COFF_SectionFlag_MemNotPaged}, + {"shared", COFF_SectionFlag_MemShared}, {"link_comdat", COFF_SectionFlag_LnkCOMDAT}, {"link_info", COFF_SectionFlag_LnkInfo}, + {"link_remove", COFF_SectionFlag_LnkRemove}, {"link_other", COFF_SectionFlag_LnkOther}, {"link_nreloc_overflow", COFF_SectionFlag_LnkNRelocOvfl}, + {"gp_relative", COFF_SectionFlag_GpRel}, {"type_no_pad", COFF_SectionFlag_TypeNoPad}, {"16bit", COFF_SectionFlag_Mem16Bit}, + {"locked", COFF_SectionFlag_MemLocked}, {"preload", COFF_SectionFlag_MemPreload}, + }; + for + EachElement(i, map) + { + if (str8_match(string, str8_cstring(map[i].name), StringMatchFlag_CaseInsensitive)) { return map[i].flag; } + } + return 0; +} + +internal void +t_coff_parse_named_flags(T_ParseContext *ctx, MD_Node *field, B32 permissions, COFF_SectionFlags *flags_io) +{ + if (md_node_is_nil(field)) { return; } + COFF_SectionFlags seen = 0; + for + MD_EachNode(value, field->first) + { + COFF_SectionFlags flag = permissions ? t_coff_permission_flag(value->string) : t_coff_section_flag(value->string); + if (flag == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, value, "unknown COFF section %s '%S'", permissions ? "permission" : "flag", value->string); + } else if (seen & flag) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, value, "duplicate COFF section %s '%S'", permissions ? "permission" : "flag", value->string); + } else { + seen |= flag; + } + } + *flags_io |= seen; +} + +internal B32 +t_coff_reloc_type_from_string(COFF_MachineType machine, String8 string, COFF_RelocType *type_out) +{ + for + EachIndex(value, 256) + { + String8 candidate = coff_string_from_reloc(machine, (COFF_RelocType)value); + if (candidate.size != 0 && str8_match(candidate, string, StringMatchFlag_CaseInsensitive)) { + *type_out = (COFF_RelocType)value; + return 1; + } + } + return 0; +} + +internal T_COFF_SymbolKind +t_coff_symbol_kind_from_string(String8 string) +{ + struct Map + { + char *name; + T_COFF_SymbolKind kind; + }; + local_persist struct Map map[] = { + {"external", T_COFF_SymbolKind_External}, + {"external_function", T_COFF_SymbolKind_ExternalFunction}, + {"static", T_COFF_SymbolKind_Static}, + {"section_definition", T_COFF_SymbolKind_SectionDefinition}, + {"weak", T_COFF_SymbolKind_Weak}, + {"absolute", T_COFF_SymbolKind_Absolute}, + {"undefined", T_COFF_SymbolKind_Undefined}, + {"undefined_function", T_COFF_SymbolKind_UndefinedFunction}, + {"undefined_section", T_COFF_SymbolKind_UndefinedSection}, + {"section", T_COFF_SymbolKind_Section}, + {"common", T_COFF_SymbolKind_Common}, + }; + for + EachElement(i, map) + { + if (str8_match(string, str8_cstring(map[i].name), StringMatchFlag_CaseInsensitive)) { return map[i].kind; } + } + return T_COFF_SymbolKind_Null; +} + +internal COFF_ComdatSelectType +t_coff_selection_from_string(String8 string) +{ + for (U64 value = COFF_ComdatSelect_Null; value <= COFF_ComdatSelect_Largest; value += 1) { + if (str8_match(coff_string_from_comdat_select_type((COFF_ComdatSelectType)value), string, StringMatchFlag_CaseInsensitive)) { return (COFF_ComdatSelectType)value; } + } + return (COFF_ComdatSelectType)max_U32; +} + +internal COFF_WeakExtType +t_coff_weak_search_from_string(String8 string) +{ + if (str8_match(string, str8_lit("no_library"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_NoLibrary; } + if (str8_match(string, str8_lit("search_library"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_SearchLibrary; } + if (str8_match(string, str8_lit("alias"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_SearchAlias; } + if (str8_match(string, str8_lit("anti_dependency"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_AntiDependency; } + return COFF_WeakExt_Null; +} + +internal void +t_coff_parse_relocations(T_ParseContext *ctx, T_COFF_Object *object, T_COFF_Section *section, MD_Node *list) +{ + if (md_node_is_nil(list)) { return; } + char *allowed[] = {"type", "offset", "symbol", 0}; + for + MD_EachNode(node, list->first) + { + if (section->relocation_count >= max_U16) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "section '%S' exceeds the COFF relocation limit", section->id); + continue; + } + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_Relocation *relocation = push_array(ctx->arena, T_COFF_Relocation, 1); + relocation->node = node; + relocation->id = node->string; + String8 type = {0}; + U64 offset = 0; + t_coff_scalar(ctx, node, "type", 1, &type); + t_coff_u64(ctx, node, "offset", 1, 0, max_U32, &offset); + t_coff_scalar(ctx, node, "symbol", 1, &relocation->symbol_id); + relocation->offset = (U32)offset; + if (!t_coff_reloc_type_from_string(object->machine, type, &relocation->type)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "type"), "relocation '%S' is invalid for machine %S", type, + coff_string_from_machine_type(object->machine)); + } + SLLQueuePush(section->first_relocation, section->last_relocation, relocation); + section->relocation_count += 1; + } +} + +internal void +t_coff_parse_sections(T_ParseContext *ctx, T_COFF_Object *object, MD_Node *list) +{ + if (md_node_is_nil(list)) { return; } + char *allowed[] = {"name", "permissions", "content", "alignment", "flags", "raw_flags", "data", "relocations", 0}; + for + MD_EachNode(node, list->first) + { + if (object->section_count >= max_U16) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "object exceeds the COFF section limit"); + continue; + } + t_coff_reject_unknown(ctx, node, allowed); + if (node->string.size == 0 || t_coff_section_from_id(object, node->string) != 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "section IDs must be non-empty and unique"); + continue; + } + T_COFF_Section *section = push_array(ctx->arena, T_COFF_Section, 1); + section->node = node; + section->id = node->string; + t_coff_scalar(ctx, node, "name", 1, §ion->name); + t_coff_parse_named_flags(ctx, t_coff_field(ctx, node, "permissions"), 1, §ion->flags); + String8 content = {0}; + if (t_coff_scalar(ctx, node, "content", 1, &content)) { + if (str8_match(content, str8_lit("code"), StringMatchFlag_CaseInsensitive)) { + section->flags |= COFF_SectionFlag_CntCode; + } else if (str8_match(content, str8_lit("initialized_data"), StringMatchFlag_CaseInsensitive)) { + section->flags |= COFF_SectionFlag_CntInitializedData; + } else if (str8_match(content, str8_lit("uninitialized_data"), StringMatchFlag_CaseInsensitive)) { + section->flags |= COFF_SectionFlag_CntUninitializedData; + } else { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "content"), "unknown COFF section content kind '%S'", content); + } + } + U64 alignment = 0; + if (!md_node_is_nil(t_coff_field(ctx, node, "alignment"))) { + t_coff_u64(ctx, node, "alignment", 0, 0, 8192, &alignment); + COFF_SectionFlags align_flags = coff_section_flag_from_align_size(alignment); + if (align_flags == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "alignment"), "unsupported COFF section alignment %llu", alignment); + } else { + section->flags |= align_flags; + } + } + t_coff_parse_named_flags(ctx, t_coff_field(ctx, node, "flags"), 0, §ion->flags); + U64 raw_flags = 0; + if (!md_node_is_nil(t_coff_field(ctx, node, "raw_flags"))) { + t_coff_u64(ctx, node, "raw_flags", 0, 0, max_U32, &raw_flags); + COFF_SectionFlags align_mask = COFF_SectionFlag_AlignMask << COFF_SectionFlag_AlignShift; + COFF_SectionFlags content_mask = COFF_SectionFlag_CntCode | COFF_SectionFlag_CntInitializedData | COFF_SectionFlag_CntUninitializedData; + if ((section->flags & raw_flags) || ((section->flags & align_mask) && (raw_flags & align_mask)) || ((section->flags & content_mask) && (raw_flags & content_mask))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "raw_flags"), "raw_flags conflicts with typed section flags"); + } + COFF_SectionFlags raw_content = (COFF_SectionFlags)raw_flags & content_mask; + if ((raw_content & (raw_content - 1)) != 0 || COFF_SectionFlags_ExtractAlign(raw_flags) == COFF_SectionFlag_AlignMask) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "raw_flags"), "raw_flags contains an invalid content kind or alignment"); + } + section->flags |= (COFF_SectionFlags)raw_flags; + } + if (section->flags & COFF_SectionFlag_LnkNRelocOvfl) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "link_nreloc_overflow is writer-owned and cannot be specified"); + } + MD_Node *data = t_coff_field(ctx, node, "data"); + if (md_node_is_nil(data)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "section requires data"); + } else { + t_bytes_from_producer(ctx->run, data, §ion->data); + } + SLLQueuePush(object->first_section, object->last_section, section); + object->section_count += 1; + t_coff_parse_relocations(ctx, object, section, t_coff_field(ctx, node, "relocations")); + } +} + +internal void +t_coff_parse_symbols(T_ParseContext *ctx, T_COFF_Object *object, MD_Node *list) +{ + if (md_node_is_nil(list)) { return; } + for + MD_EachNode(node, list->first) + { + if (node->string.size == 0 || t_coff_symbol_from_id(object, node->string) != 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "symbol IDs must be non-empty and unique"); + continue; + } + T_COFF_Symbol *symbol = push_array(ctx->arena, T_COFF_Symbol, 1); + symbol->node = node; + symbol->id = node->string; + String8 kind = {0}; + t_coff_scalar(ctx, node, "kind", 1, &kind); + symbol->kind = t_coff_symbol_kind_from_string(kind); + if (symbol->kind == T_COFF_SymbolKind_Null) { t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "kind"), "unknown COFF symbol kind '%S'", kind); } + char *known_fields[] = {"kind", "name", "section", "value", "size", "selection", "associate", "fallback", "search", "storage", 0}; + char *regular_fields[] = {"kind", "name", "section", "value", 0}; + char *section_definition_fields[] = {"kind", "section", "selection", "associate", 0}; + char *weak_fields[] = {"kind", "name", "fallback", "search", 0}; + char *absolute_fields[] = {"kind", "name", "value", "storage", 0}; + char *undefined_fields[] = {"kind", "name", 0}; + char *undefined_section_fields[] = {"kind", "name", "value", 0}; + char *section_fields[] = {"kind", "name", "section", 0}; + char *common_fields[] = {"kind", "name", "size", 0}; + char **allowed = known_fields; + switch (symbol->kind) { + case T_COFF_SymbolKind_External: + case T_COFF_SymbolKind_ExternalFunction: + case T_COFF_SymbolKind_Static: + allowed = regular_fields; + break; + case T_COFF_SymbolKind_SectionDefinition: + allowed = section_definition_fields; + break; + case T_COFF_SymbolKind_Weak: + allowed = weak_fields; + break; + case T_COFF_SymbolKind_Absolute: + allowed = absolute_fields; + break; + case T_COFF_SymbolKind_Undefined: + case T_COFF_SymbolKind_UndefinedFunction: + allowed = undefined_fields; + break; + case T_COFF_SymbolKind_UndefinedSection: + allowed = undefined_section_fields; + break; + case T_COFF_SymbolKind_Section: + allowed = section_fields; + break; + case T_COFF_SymbolKind_Common: + allowed = common_fields; + break; + default: + break; + } + t_coff_reject_unknown(ctx, node, allowed); + t_coff_scalar(ctx, node, "name", 0, &symbol->name); + t_coff_scalar(ctx, node, "section", 0, &symbol->section_id); + t_coff_scalar(ctx, node, "associate", 0, &symbol->associate_id); + t_coff_scalar(ctx, node, "fallback", 0, &symbol->fallback_id); + U64 value = 0, size = 0; + t_coff_u64(ctx, node, "value", 0, 0, max_U32, &value); + t_coff_u64(ctx, node, "size", 0, 0, max_U32, &size); + symbol->value = (U32)value; + symbol->size = (U32)size; + String8 selection = {0}; + if (!md_node_is_nil(t_coff_field(ctx, node, "selection")) && t_coff_scalar(ctx, node, "selection", 0, &selection)) { + symbol->selection = t_coff_selection_from_string(selection); + if (symbol->selection == (COFF_ComdatSelectType)max_U32) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "selection"), "unknown COMDAT selection '%S'", selection); + } + } + String8 search = {0}; + if (!md_node_is_nil(t_coff_field(ctx, node, "search")) && t_coff_scalar(ctx, node, "search", 0, &search)) { + symbol->weak_search = t_coff_weak_search_from_string(search); + if (symbol->weak_search == COFF_WeakExt_Null) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "search"), "unknown weak search policy '%S'", search); + } + } + String8 storage = {0}; + if (!md_node_is_nil(t_coff_field(ctx, node, "storage")) && t_coff_scalar(ctx, node, "storage", 0, &storage)) { + if (str8_match(storage, str8_lit("external"), StringMatchFlag_CaseInsensitive)) { + symbol->storage_class = COFF_SymStorageClass_External; + } else if (str8_match(storage, str8_lit("static"), StringMatchFlag_CaseInsensitive)) { + symbol->storage_class = COFF_SymStorageClass_Static; + } else { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "storage"), "absolute symbol storage must be external or static"); + } + } + SLLQueuePush(object->first_symbol, object->last_symbol, symbol); + object->symbol_count += 1; + } + + for (T_COFF_Symbol *symbol = object->first_symbol; symbol != 0; symbol = symbol->next) { + B32 needs_name = symbol->kind != T_COFF_SymbolKind_SectionDefinition; + B32 needs_section = symbol->kind == T_COFF_SymbolKind_External || symbol->kind == T_COFF_SymbolKind_ExternalFunction || symbol->kind == T_COFF_SymbolKind_Static || + symbol->kind == T_COFF_SymbolKind_SectionDefinition || symbol->kind == T_COFF_SymbolKind_Section; + if (needs_name && symbol->name.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "symbol '%S' requires name", symbol->id); } + if (needs_section && t_coff_section_from_id(object, symbol->section_id) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "symbol '%S' references unknown section '%S'", symbol->id, symbol->section_id); + } + if (symbol->kind == T_COFF_SymbolKind_SectionDefinition) { + if (md_node_is_nil(t_coff_field(ctx, symbol->node, "selection"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "section definition requires selection"); + } + if (symbol->selection == (COFF_ComdatSelectType)max_U32) { continue; } + if (symbol->selection == COFF_ComdatSelect_Associative && t_coff_section_from_id(object, symbol->associate_id) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "associative section definition requires a valid associate section"); + } + } + if (symbol->kind == T_COFF_SymbolKind_Weak) { + T_COFF_Symbol *fallback = t_coff_symbol_from_id(object, symbol->fallback_id); + if (fallback == 0 || fallback == symbol || fallback->encoded != (COFF_ObjSymbol *)1) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "weak symbol fallback '%S' must refer to an earlier symbol", symbol->fallback_id); + } + if (symbol->weak_search == COFF_WeakExt_Null) { t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "weak symbol requires search policy"); } + } + if (symbol->kind == T_COFF_SymbolKind_Absolute && md_node_is_nil(t_coff_field(ctx, symbol->node, "storage"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "absolute symbol requires storage"); + } + if (symbol->kind == T_COFF_SymbolKind_UndefinedSection && md_node_is_nil(t_coff_field(ctx, symbol->node, "value"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "undefined section symbol requires value"); + } + if (symbol->kind == T_COFF_SymbolKind_Common && md_node_is_nil(t_coff_field(ctx, symbol->node, "size"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "common symbol requires size"); + } + symbol->encoded = (COFF_ObjSymbol *)1; + } + for (T_COFF_Symbol *symbol = object->first_symbol; symbol != 0; symbol = symbol->next) { symbol->encoded = 0; } + + for + EachNode(section, T_COFF_Section, object->first_section) + { + for + EachNode(relocation, T_COFF_Relocation, section->first_relocation) + { + if (t_coff_symbol_from_id(object, relocation->symbol_id) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, relocation->node, "relocation references unknown symbol '%S'", relocation->symbol_id); + } + } + } +} + +internal T_COFF_Object * +t_coff_parse_object(T_ParseContext *ctx, MD_Node *node) +{ + char *allowed[] = {"machine", "timestamp", "sections", "symbols", "directives", 0}; + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_Object *object = push_array(ctx->arena, T_COFF_Object, 1); + object->node = node; + t_coff_machine(ctx, node, &object->machine); + U64 timestamp = 0; + t_coff_u64(ctx, node, "timestamp", 0, 0, max_U32, ×tamp); + object->timestamp = (COFF_TimeStamp)timestamp; + t_coff_parse_sections(ctx, object, t_coff_field(ctx, node, "sections")); + t_coff_parse_symbols(ctx, object, t_coff_field(ctx, node, "symbols")); + MD_Node *directives = t_coff_field(ctx, node, "directives"); + for + MD_EachNode(directive_node, directives->first) + { + if (!str8_match(directive_node->string, str8_lit("directive"), StringMatchFlag_CaseInsensitive) || md_node_is_nil(directive_node->first) || + !md_node_is_nil(directive_node->first->next)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, directive_node, "directives must contain 'directive: value' entries"); + continue; + } + T_COFF_Directive *directive = push_array(ctx->arena, T_COFF_Directive, 1); + directive->node = directive_node; + directive->string = directive_node->first->string; + SLLQueuePush(object->first_directive, object->last_directive, directive); + object->directive_count += 1; + } + return object; +} + +internal COFF_ImportByType +t_coff_import_by_from_string(String8 string) +{ + if (str8_match(string, str8_lit("ordinal"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_Ordinal; } + if (str8_match(string, str8_lit("name"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_Name; } + if (str8_match(string, str8_lit("name_no_prefix"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_NameNoPrefix; } + if (str8_match(string, str8_lit("undecorate"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_Undecorate; } + return (COFF_ImportByType)max_U32; +} + +internal void +t_coff_parse_library_member(T_ParseContext *ctx, T_COFF_Library *library, MD_Node *node) +{ + char *allowed[] = {"path", "object", "import", "dll_import", 0}; + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_LibraryMember *member = push_array(ctx->arena, T_COFF_LibraryMember, 1); + member->node = node; + member->id = node->string; + MD_Node *object_node = t_coff_field(ctx, node, "object"); + MD_Node *import_node = t_coff_field(ctx, node, "import"); + MD_Node *dll_node = t_coff_field(ctx, node, "dll_import"); + U64 kind_count = !md_node_is_nil(object_node) + !md_node_is_nil(import_node) + !md_node_is_nil(dll_node); + if (kind_count != 1) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "library member requires exactly one object, import, or dll_import definition"); + } else if (!md_node_is_nil(object_node)) { + char *object_allowed[] = {"path", "object", 0}; + t_coff_reject_unknown(ctx, node, object_allowed); + member->kind = T_COFF_LibraryMemberKind_Object; + t_coff_scalar(ctx, node, "path", 0, &member->path); + if (member->path.size == 0) { member->path = str8f(ctx->arena, "%S.obj", member->id); } + member->object = t_coff_parse_object(ctx, object_node); + } else if (!md_node_is_nil(import_node)) { + char *member_allowed[] = {"import", 0}; + t_coff_reject_unknown(ctx, node, member_allowed); + member->kind = T_COFF_LibraryMemberKind_Import; + char *import_allowed[] = {"dll", "name", "machine", "timestamp", "type", "lookup", "hint", "ordinal", 0}; + t_coff_reject_unknown(ctx, import_node, import_allowed); + t_coff_scalar(ctx, import_node, "dll", 1, &member->dll); + t_coff_scalar(ctx, import_node, "name", 1, &member->name); + t_coff_machine(ctx, import_node, &member->machine); + U64 timestamp = 0; + t_coff_u64(ctx, import_node, "timestamp", 0, 0, max_U32, ×tamp); + member->timestamp = (COFF_TimeStamp)timestamp; + String8 type = {0}; + t_coff_scalar(ctx, import_node, "type", 1, &type); + member->import_type = coff_import_header_type_from_string(type); + if (member->import_type == COFF_ImportType_Invalid) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, import_node, "type"), "unsupported import type '%S'", type); + } + String8 lookup = {0}; + t_coff_scalar(ctx, import_node, "lookup", 1, &lookup); + member->import_by = t_coff_import_by_from_string(lookup); + if (member->import_by == (COFF_ImportByType)max_U32) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, import_node, "lookup"), "unknown import lookup '%S'", lookup); + } + U64 hint_or_ordinal = 0; + char *number_field = member->import_by == COFF_ImportBy_Ordinal ? "ordinal" : "hint"; + char *name_import_allowed[] = {"dll", "name", "machine", "timestamp", "type", "lookup", "hint", 0}; + char *ordinal_import_allowed[] = {"dll", "name", "machine", "timestamp", "type", "lookup", "ordinal", 0}; + t_coff_reject_unknown(ctx, import_node, member->import_by == COFF_ImportBy_Ordinal ? ordinal_import_allowed : name_import_allowed); + t_coff_u64(ctx, import_node, number_field, 0, 0, max_U16, &hint_or_ordinal); + member->hint_or_ordinal = (U16)hint_or_ordinal; + } else if (!md_node_is_nil(dll_node)) { + char *member_allowed[] = {"dll_import", 0}; + t_coff_reject_unknown(ctx, node, member_allowed); + member->kind = T_COFF_LibraryMemberKind_DllImport; + char *dll_allowed[] = {"name", "machine", "timestamp", 0}; + t_coff_reject_unknown(ctx, dll_node, dll_allowed); + t_coff_scalar(ctx, dll_node, "name", 1, &member->name); + t_coff_machine(ctx, dll_node, &member->machine); + if (member->machine != COFF_MachineType_X64) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, dll_node, "machine"), "dll_import currently supports only x64"); + } + U64 timestamp = 0; + t_coff_u64(ctx, dll_node, "timestamp", 0, 0, max_U32, ×tamp); + member->timestamp = (COFF_TimeStamp)timestamp; + } + SLLQueuePush(library->first_member, library->last_member, member); + library->member_count += 1; +} + +internal T_COFF_Library * +t_coff_parse_library(T_ParseContext *ctx, MD_Node *node) +{ + char *allowed[] = {"timestamp", "mode", "second_linker_member", "members", 0}; + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_Library *library = push_array(ctx->arena, T_COFF_Library, 1); + library->node = node; + U64 timestamp = 0, mode = 0; + t_coff_u64(ctx, node, "timestamp", 0, 0, max_U32, ×tamp); + t_coff_u64(ctx, node, "mode", 0, 0, max_U16, &mode); + t_coff_bool(ctx, node, "second_linker_member", 0, &library->second_linker_member); + library->timestamp = (COFF_TimeStamp)timestamp; + library->mode = (U16)mode; + MD_Node *members = t_coff_field(ctx, node, "members"); + for + MD_EachNode(member, members->first) { t_coff_parse_library_member(ctx, library, member); } + if (library->second_linker_member) { + U64 emitted_member_count = 0; + for (T_COFF_LibraryMember *member = library->first_member; member != 0; member = member->next) { + emitted_member_count += member->kind == T_COFF_LibraryMemberKind_DllImport ? 3 : 1; + } + if (emitted_member_count > max_U16) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "library exceeds the second linker member limit"); } + } + return library; +} + +internal T_Result +t_coff_validate(T_ParseContext *ctx, T_Artifact *artifact) +{ + ctx->operation = str8_lit("coff"); + MD_Node *definition = artifact->definition; + char *allowed[] = {"object", "library", 0}; + t_coff_reject_unknown(ctx, definition, allowed); + MD_Node *object = t_coff_field(ctx, definition, "object"); + MD_Node *library = t_coff_field(ctx, definition, "library"); + if (md_node_is_nil(object) == md_node_is_nil(library)) { + return t_parse_errorf(ctx, T_ResultCode_ValidationError, definition, "coff requires exactly one object or library definition"); + } + T_COFF_Model *model = push_array(ctx->arena, T_COFF_Model, 1); + if (!md_node_is_nil(object)) { + model->kind = T_COFF_ModelKind_Object; + model->object = t_coff_parse_object(ctx, object); + } else { + model->kind = T_COFF_ModelKind_Library; + model->library = t_coff_parse_library(ctx, library); + } + artifact->codec_data = model; + return ctx->run->result; +} + +internal String8 +t_coff_encode_object(T_Context *ctx, T_COFF_Object *object) +{ + COFF_ObjWriter *writer = coff_obj_writer_alloc(object->timestamp, object->machine); + for (T_COFF_Section *section = object->first_section; section != 0; section = section->next) { + section->encoded = coff_obj_writer_push_section(writer, section->name, section->flags, section->data); + } + for (T_COFF_Directive *directive = object->first_directive; directive != 0; directive = directive->next) { coff_obj_writer_push_directive(writer, directive->string); } + for (T_COFF_Symbol *symbol = object->first_symbol; symbol != 0; symbol = symbol->next) { + T_COFF_Section *section = t_coff_section_from_id(object, symbol->section_id); + switch (symbol->kind) { + case T_COFF_SymbolKind_External: + symbol->encoded = coff_obj_writer_push_symbol_extern(writer, symbol->name, symbol->value, section->encoded); + break; + case T_COFF_SymbolKind_ExternalFunction: + symbol->encoded = coff_obj_writer_push_symbol_extern_func(writer, symbol->name, symbol->value, section->encoded); + break; + case T_COFF_SymbolKind_Static: + symbol->encoded = coff_obj_writer_push_symbol_static(writer, symbol->name, symbol->value, section->encoded); + break; + case T_COFF_SymbolKind_SectionDefinition: { + if (symbol->selection == COFF_ComdatSelect_Associative) { + T_COFF_Section *associate = t_coff_section_from_id(object, symbol->associate_id); + symbol->encoded = coff_obj_writer_push_symbol_associative(writer, section->encoded, associate->encoded); + } else { + symbol->encoded = coff_obj_writer_push_symbol_secdef(writer, section->encoded, symbol->selection); + } + } break; + case T_COFF_SymbolKind_Weak: { + T_COFF_Symbol *fallback = t_coff_symbol_from_id(object, symbol->fallback_id); + symbol->encoded = coff_obj_writer_push_symbol_weak(writer, symbol->name, symbol->weak_search, fallback->encoded); + } break; + case T_COFF_SymbolKind_Absolute: + symbol->encoded = coff_obj_writer_push_symbol_abs(writer, symbol->name, symbol->value, symbol->storage_class); + break; + case T_COFF_SymbolKind_Undefined: + symbol->encoded = coff_obj_writer_push_symbol_undef(writer, symbol->name); + break; + case T_COFF_SymbolKind_UndefinedFunction: + symbol->encoded = coff_obj_writer_push_symbol_undef_func(writer, symbol->name); + break; + case T_COFF_SymbolKind_UndefinedSection: + symbol->encoded = coff_obj_writer_push_symbol_undef_sect(writer, symbol->name, symbol->value); + break; + case T_COFF_SymbolKind_Section: + symbol->encoded = coff_obj_writer_push_symbol_sect(writer, symbol->name, section->encoded); + break; + case T_COFF_SymbolKind_Common: + symbol->encoded = coff_obj_writer_push_symbol_common(writer, symbol->name, symbol->size); + break; + default: + break; + } + } + for (T_COFF_Section *section = object->first_section; section != 0; section = section->next) { + for (T_COFF_Relocation *relocation = section->first_relocation; relocation != 0; relocation = relocation->next) { + T_COFF_Symbol *symbol = t_coff_symbol_from_id(object, relocation->symbol_id); + coff_obj_writer_section_push_reloc(writer, section->encoded, relocation->offset, symbol->encoded, relocation->type); + } + } + String8 result = coff_obj_writer_serialize(ctx->arena, writer); + coff_obj_writer_release(&writer); + return result; +} + +internal String8 +t_coff_encode_library(T_Context *ctx, T_COFF_Library *library) +{ + COFF_LibWriter *writer = coff_lib_writer_alloc(); + for (T_COFF_LibraryMember *member = library->first_member; member != 0; member = member->next) { + switch (member->kind) { + case T_COFF_LibraryMemberKind_Object: + coff_lib_writer_push_obj(writer, member->path, t_coff_encode_object(ctx, member->object)); + break; + case T_COFF_LibraryMemberKind_Import: + coff_lib_writer_push_import(writer, member->machine, member->timestamp, member->dll, member->import_by, member->name, member->hint_or_ordinal, member->import_type); + break; + case T_COFF_LibraryMemberKind_DllImport: { + String8 dll_name = str8_chop_last_dot(member->name); + String8 debug = lnk_make_linker_debug_symbols(writer->arena, member->machine); + coff_lib_writer_push_obj(writer, dll_name, pe_make_import_entry_obj(writer->arena, dll_name, member->timestamp, member->machine, debug)); + coff_lib_writer_push_obj(writer, dll_name, pe_make_null_import_descriptor_obj(writer->arena, member->timestamp, member->machine, debug)); + coff_lib_writer_push_obj(writer, dll_name, pe_make_null_thunk_data_obj(writer->arena, dll_name, member->timestamp, member->machine, debug)); + } break; + default: + break; + } + } + String8 result = coff_lib_writer_serialize(ctx->arena, writer, library->timestamp, library->mode, library->second_linker_member); + coff_lib_writer_release(&writer); + return result; +} + +internal T_Result +t_coff_encode(T_Context *ctx, T_Artifact *artifact) +{ + T_COFF_Model *model = artifact->codec_data; + if (model->kind == T_COFF_ModelKind_Object) { + artifact->data = t_coff_encode_object(ctx, model->object); + } else if (model->kind == T_COFF_ModelKind_Library) { + artifact->data = t_coff_encode_library(ctx, model->library); + } + return ctx->result; +} + +internal T_Result +t_coff_decode_object(T_Context *ctx, T_Artifact *artifact, String8 data, MD_Node *object) +{ + COFF_FileHeaderInfo header = coff_file_header_info_from_data(data); + if (header.section_table_range.max > data.size || header.symbol_table_range.max > data.size || header.string_table_range.max > data.size || + dim_1u64(header.section_table_range) != header.section_count_no_null * sizeof(COFF_SectionHeader) || + dim_1u64(header.symbol_table_range) != header.symbol_count * header.symbol_size) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF object has invalid table ranges"); + } + + String8 string_table = str8_substr(data, header.string_table_range); + String8 symbol_table = str8_substr(data, header.symbol_table_range); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(data, header.section_table_range).str; + String8 machine = coff_string_from_machine_type(header.machine); + if (machine.size == 0 && header.machine == COFF_MachineType_Unknown) { machine = str8_lit("Unknown"); } + t_codec_push_field(ctx, object, "machine", machine); + t_codec_push_field(ctx, object, "big_object", header.is_big_obj ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, object, "section_count", header.section_count_no_null); + t_codec_push_u64(ctx, object, "symbol_record_count", header.symbol_count); + + MD_Node *sections = t_codec_push_node(ctx->arena, object, str8_lit("sections")); + for + EachIndex(section_idx, header.section_count_no_null) + { + COFF_SectionHeader *section_header = §ion_table[section_idx]; + MD_Node *section = t_codec_push_node(ctx->arena, sections, str8f(ctx->arena, "section_%llu", section_idx + 1)); + String8 section_name = {0}; + if (!t_codec_coff_section_name(string_table, section_header, §ion_name)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF section %llu has an invalid name", section_idx + 1); + } + t_codec_push_field(ctx, section, "name", section_name); + t_codec_push_u64(ctx, section, "virtual_size", section_header->vsize); + t_codec_push_u64(ctx, section, "virtual_offset", section_header->voff); + t_codec_push_u64(ctx, section, "file_size", section_header->fsize); + t_codec_push_u64(ctx, section, "file_offset", section_header->foff); + t_codec_push_u64(ctx, section, "alignment", coff_align_size_from_section_flags(section_header->flags)); + t_codec_push_u64(ctx, section, "raw_flags", section_header->flags); + + String8 section_data = str8_zero(); + if (!(section_header->flags & COFF_SectionFlag_CntUninitializedData) && section_header->fsize != 0) { + if (section_header->foff > data.size || section_header->fsize > data.size - section_header->foff) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF section %llu has an invalid data range", section_idx + 1); + } + section_data = str8_substr(data, rng_1u64(section_header->foff, section_header->foff + section_header->fsize)); + } + t_codec_push_field(ctx, section, "data", t_hex_from_data(ctx->arena, section_data)); + + COFF_RelocInfo reloc_info = coff_reloc_info_from_section_header(data, section_header); + if (reloc_info.array_off > data.size || reloc_info.count > (data.size - reloc_info.array_off) / sizeof(COFF_Reloc)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF section %llu has an invalid relocation range", section_idx + 1); + } + MD_Node *relocations = t_codec_push_node(ctx->arena, section, str8_lit("relocations")); + COFF_Reloc *reloc_array = (COFF_Reloc *)(data.str + reloc_info.array_off); + for + EachIndex(reloc_idx, reloc_info.count) + { + COFF_Reloc *reloc = &reloc_array[reloc_idx]; + MD_Node *relocation = t_codec_push_node(ctx->arena, relocations, str8f(ctx->arena, "relocation_%llu", reloc_idx)); + t_codec_push_u64(ctx, relocation, "offset", reloc->apply_off); + t_codec_push_u64(ctx, relocation, "symbol_index", reloc->isymbol); + String8 type = coff_string_from_reloc(header.machine, reloc->type); + if (type.size != 0) { t_codec_push_field(ctx, relocation, "type", type); } + t_codec_push_u64(ctx, relocation, "raw_type", reloc->type); + } + } + + MD_Node *symbols = t_codec_push_node(ctx->arena, object, str8_lit("symbols")); + U64 symbol_idx = 0; + U64 primary_symbol_count = 0; + while (symbol_idx < header.symbol_count) { + COFF_ParsedSymbol symbol = coff_parse_symbol(header, string_table, symbol_table, (U32)symbol_idx); + if (symbol.aux_symbol_count > header.symbol_count - symbol_idx - 1) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF symbol %llu has an invalid auxiliary record count", symbol_idx); + } + MD_Node *symbol_node = t_codec_push_node(ctx->arena, symbols, str8f(ctx->arena, "symbol_%llu", primary_symbol_count)); + t_codec_push_u64(ctx, symbol_node, "record_index", symbol_idx); + t_codec_push_field(ctx, symbol_node, "name", symbol.name); + t_codec_push_u64(ctx, symbol_node, "value", symbol.value); + t_codec_push_u64(ctx, symbol_node, "section_number", symbol.section_number); + t_codec_push_u64(ctx, symbol_node, "raw_type", symbol.type.v); + t_codec_push_field(ctx, symbol_node, "storage_class", coff_string_from_sym_storage_class(symbol.storage_class)); + t_codec_push_u64(ctx, symbol_node, "raw_storage_class", symbol.storage_class); + t_codec_push_u64(ctx, symbol_node, "auxiliary_record_count", symbol.aux_symbol_count); + if (symbol.aux_symbol_count != 0) { + U64 aux_off = (symbol_idx + 1) * header.symbol_size; + String8 aux_data = str8_substr(symbol_table, rng_1u64(aux_off, aux_off + symbol.aux_symbol_count * header.symbol_size)); + t_codec_push_field(ctx, symbol_node, "auxiliary_data", t_hex_from_data(ctx->arena, aux_data)); + } + symbol_idx += 1 + symbol.aux_symbol_count; + primary_symbol_count += 1; + } + t_codec_push_u64(ctx, object, "symbol_count", primary_symbol_count); + return ctx->result; +} + +internal T_Result +t_coff_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + MD_Node *root = t_codec_push_node(ctx->arena, 0, str8_lit("coff")); + if (coff_is_obj(data) || coff_is_big_obj(data)) { + MD_Node *object = t_codec_push_node(ctx->arena, root, str8_lit("object")); + T_Result result = t_coff_decode_object(ctx, artifact, data, object); + if (!t_result_is_ok(result)) { return result; } + } else if (coff_is_regular_archive(data)) { + COFF_ArchiveParse archive_parse = coff_regular_archive_parse_from_data(data); + if (archive_parse.error.size != 0) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "invalid COFF archive: %S", archive_parse.error); + } + MD_Node *library = t_codec_push_node(ctx->arena, root, str8_lit("library")); + t_codec_push_field(ctx, library, "second_linker_member", archive_parse.has_second_header ? str8_lit("true") : str8_lit("false")); + t_codec_push_field(ctx, library, "long_names", archive_parse.long_names.size != 0 ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, library, "first_symbol_count", archive_parse.first_member.symbol_count); + if (archive_parse.has_second_header) { t_codec_push_u64(ctx, library, "second_symbol_count", archive_parse.second_member.symbol_count); } + MD_Node *members = t_codec_push_node(ctx->arena, library, str8_lit("members")); + U64 member_idx = 0; + U64 offset = coff_regular_archive_member_iter_init(data); + COFF_ArchiveMember member = {0}; + while (coff_regular_archive_member_iter_next(data, &offset, &member)) { + if (str8_match_lit("/", member.header.name, 0) || str8_match_lit("//", member.header.name, 0)) { continue; } + COFF_DataType member_type = coff_data_type_from_data(member.data); + MD_Node *member_node = t_codec_push_node(ctx->arena, members, str8f(ctx->arena, "member_%llu", member_idx)); + t_codec_push_field(ctx, member_node, "name", coff_decode_member_name(archive_parse.long_names, member.header.name)); + t_codec_push_u64(ctx, member_node, "timestamp", member.header.time_stamp); + t_codec_push_u64(ctx, member_node, "user_id", member.header.user_id); + t_codec_push_u64(ctx, member_node, "group_id", member.header.group_id); + t_codec_push_field(ctx, member_node, "mode", member.header.mode); + if (member_type == COFF_DataType_Import) { + COFF_ParsedArchiveImportHeader import = {0}; + COFF_ImportHeader *raw_import = str8_deserial_get_raw_ptr(member.data, 0, sizeof(*raw_import)); + B32 import_is_valid = raw_import != 0 && raw_import->data_size <= member.data.size - sizeof(*raw_import); + if (import_is_valid) { + String8 import_data = str8_substr(member.data, rng_1u64(sizeof(*raw_import), sizeof(*raw_import) + raw_import->data_size)); + String8 ignored = {0}; + U64 cursor = str8_deserial_read_cstr(import_data, 0, &ignored); + cursor += str8_deserial_read_cstr(import_data, cursor, &ignored); + import_is_valid = cursor == import_data.size; + } + if (!import_is_valid || coff_parse_import(member.data, 0, &import) == 0) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "archive member %llu has an invalid import header", member_idx); + } + MD_Node *import_node = t_codec_push_node(ctx->arena, member_node, str8_lit("import")); + t_codec_push_field(ctx, import_node, "machine", coff_string_from_machine_type(import.machine)); + t_codec_push_field(ctx, import_node, "dll", import.dll_name); + t_codec_push_field(ctx, import_node, "name", import.func_name); + t_codec_push_u64(ctx, import_node, "timestamp", import.time_stamp); + t_codec_push_u64(ctx, import_node, "hint_or_ordinal", import.hint_or_ordinal); + t_codec_push_field(ctx, import_node, "type", coff_string_from_import_header_type(import.type)); + t_codec_push_u64(ctx, import_node, "lookup", import.import_by); + } else { + MD_Node *object = t_codec_push_node(ctx->arena, member_node, str8_lit("object")); + T_Result result = t_coff_decode_object(ctx, artifact, member.data, object); + if (!t_result_is_ok(result)) { return result; } + } + member_idx += 1; + } + t_codec_push_u64(ctx, library, "member_count", member_idx); + } else { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "artifact is not a supported COFF object or archive"); + } + *semantic_tree_out = root; + return ctx->result; +} diff --git a/src/torture/torture_coff.h b/src/torture/torture_coff.h new file mode 100644 index 000000000..264c2a2b7 --- /dev/null +++ b/src/torture/torture_coff.h @@ -0,0 +1,150 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +typedef enum T_COFF_SymbolKind +{ + T_COFF_SymbolKind_Null, + T_COFF_SymbolKind_External, + T_COFF_SymbolKind_ExternalFunction, + T_COFF_SymbolKind_Static, + T_COFF_SymbolKind_SectionDefinition, + T_COFF_SymbolKind_Weak, + T_COFF_SymbolKind_Absolute, + T_COFF_SymbolKind_Undefined, + T_COFF_SymbolKind_UndefinedFunction, + T_COFF_SymbolKind_UndefinedSection, + T_COFF_SymbolKind_Section, + T_COFF_SymbolKind_Common, +} T_COFF_SymbolKind; + +typedef struct T_COFF_Relocation T_COFF_Relocation; +struct T_COFF_Relocation +{ + T_COFF_Relocation *next; + MD_Node *node; + String8 id; + String8 symbol_id; + U32 offset; + COFF_RelocType type; +}; + +typedef struct T_COFF_Section T_COFF_Section; +struct T_COFF_Section +{ + T_COFF_Section *next; + MD_Node *node; + String8 id; + String8 name; + String8 data; + COFF_SectionFlags flags; + T_COFF_Relocation *first_relocation; + T_COFF_Relocation *last_relocation; + U64 relocation_count; + COFF_ObjSection *encoded; +}; + +typedef struct T_COFF_Symbol T_COFF_Symbol; +struct T_COFF_Symbol +{ + T_COFF_Symbol *next; + MD_Node *node; + String8 id; + String8 name; + String8 section_id; + String8 associate_id; + String8 fallback_id; + U32 value; + U32 size; + T_COFF_SymbolKind kind; + COFF_ComdatSelectType selection; + COFF_WeakExtType weak_search; + COFF_SymStorageClass storage_class; + COFF_ObjSymbol *encoded; +}; + +typedef struct T_COFF_Directive T_COFF_Directive; +struct T_COFF_Directive +{ + T_COFF_Directive *next; + MD_Node *node; + String8 string; +}; + +typedef struct T_COFF_Object T_COFF_Object; +struct T_COFF_Object +{ + MD_Node *node; + COFF_MachineType machine; + COFF_TimeStamp timestamp; + T_COFF_Section *first_section; + T_COFF_Section *last_section; + U64 section_count; + T_COFF_Symbol *first_symbol; + T_COFF_Symbol *last_symbol; + U64 symbol_count; + T_COFF_Directive *first_directive; + T_COFF_Directive *last_directive; + U64 directive_count; +}; + +typedef enum T_COFF_LibraryMemberKind +{ + T_COFF_LibraryMemberKind_Null, + T_COFF_LibraryMemberKind_Object, + T_COFF_LibraryMemberKind_Import, + T_COFF_LibraryMemberKind_DllImport, +} T_COFF_LibraryMemberKind; + +typedef struct T_COFF_LibraryMember T_COFF_LibraryMember; +struct T_COFF_LibraryMember +{ + T_COFF_LibraryMember *next; + MD_Node *node; + String8 id; + String8 path; + T_COFF_LibraryMemberKind kind; + T_COFF_Object *object; + String8 dll; + String8 name; + COFF_MachineType machine; + COFF_TimeStamp timestamp; + COFF_ImportType import_type; + COFF_ImportByType import_by; + U16 hint_or_ordinal; +}; + +typedef struct T_COFF_Library T_COFF_Library; +struct T_COFF_Library +{ + MD_Node *node; + COFF_TimeStamp timestamp; + U16 mode; + B32 second_linker_member; + T_COFF_LibraryMember *first_member; + T_COFF_LibraryMember *last_member; + U64 member_count; +}; + +typedef enum T_COFF_ModelKind +{ + T_COFF_ModelKind_Null, + T_COFF_ModelKind_Object, + T_COFF_ModelKind_Library, +} T_COFF_ModelKind; + +typedef struct T_COFF_Model T_COFF_Model; +struct T_COFF_Model +{ + T_COFF_ModelKind kind; + union + { + T_COFF_Object *object; + T_COFF_Library *library; + }; +}; + +internal T_Result t_coff_validate(T_ParseContext *ctx, T_Artifact *artifact); +internal T_Result t_coff_encode(T_Context *ctx, T_Artifact *artifact); +internal T_Result t_coff_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); diff --git a/src/torture/torture_controller.c b/src/torture/torture_controller.c new file mode 100644 index 000000000..01ce4c5b5 --- /dev/null +++ b/src/torture/torture_controller.c @@ -0,0 +1,317 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal T_ControllerResult +t_controller_result(T_ControllerResultCode code, U64 exit_code) +{ + T_ControllerResult result = {code, exit_code}; + return result; +} + +internal void +t_controller_file_close(File *file) +{ + file_close(*file); + MemoryZeroStruct(file); +} + +internal U64 +t_controller_endt_from_timeout(U64 timeout_us) +{ + if (timeout_us == max_U64) { return max_U64; } + U64 now_us = now_time_us(); + return timeout_us > max_U64 - now_us ? max_U64 : now_us + timeout_us; +} + +internal B32 +t_controller_endt_is_expired(U64 endt_us) +{ + return endt_us != max_U64 && now_time_us() >= endt_us; +} + +internal String8List * +t_controller_capture_from_stream(T_Controller *controller, T_ControllerStream stream) +{ + if (stream == T_ControllerStream_Stdout) { return &controller->stdout_capture; } + if (stream == T_ControllerStream_Stderr) { return controller->merge_outputs ? &controller->stdout_capture : &controller->stderr_capture; } + return 0; +} + +internal U64 * +t_controller_cursor_from_stream(T_Controller *controller, T_ControllerStream stream) +{ + if (stream == T_ControllerStream_Stdout) { return &controller->stdout_cursor; } + if (stream == T_ControllerStream_Stderr) { return &controller->stderr_cursor; } + return 0; +} + +internal B32 +t_controller_capture_has(Arena *arena, String8List *capture, U64 *cursor, String8 needle) +{ + String8 output = str8_list_join(arena, capture, 0); + U64 match_off = str8_find_needle(output, *cursor, needle, 0); + B32 result = match_off < output.size; + if (result) { *cursor = match_off + needle.size; } + return result; +} + +internal T_ControllerResult +t_controller_pump_pipe(T_Controller *controller, File pipe, String8List *capture) +{ + U64 available = file_pipe_bytes_available(pipe); + if (available != 0) { + U64 size = Min(available, KB(64)); + U8 *data = push_array(controller->arena, U8, size); + U64 read_size = file_pipe_read(pipe, data, size); + if (read_size == 0) { return t_controller_result(T_ControllerResultCode_IoFailed, controller->exit_code); } + str8_list_push(controller->arena, capture, str8(data, read_size)); + } + return t_controller_result(T_ControllerResultCode_Ok, controller->exit_code); +} + +internal void +t_controller_drain_pipe(T_Controller *controller, File pipe, String8List *capture) +{ + U64 endt_us = t_controller_endt_from_timeout(10 * 1000000ull); + for (;;) { + t_controller_pump_pipe(controller, pipe, capture); + if (file_pipe_is_end(pipe)) { break; } + if (t_controller_endt_is_expired(endt_us)) { break; } + sleep_ms(1); + } +} + +internal T_ControllerResult +t_controller_poll_exit(T_Controller *controller) +{ + if (controller->launched && !controller->exited) { + U64 exit_code = max_U64; + if (process_poll(controller->process, &exit_code)) { + controller->exited = 1; + controller->exit_code = exit_code; + } + } + return t_controller_result(T_ControllerResultCode_Ok, controller->exit_code); +} + +internal T_ControllerResult +t_controller_launch(Arena *arena, T_Controller *controller, T_ControllerLaunchParams *params) +{ + MemoryZeroStruct(controller); + controller->arena = arena; + controller->exit_code = max_U64; + controller->merge_outputs = params->merge_outputs; + controller->new_console = params->new_console; + + FilePair stdin_pipe = file_pipe_make(1, 0); + FilePair stdout_pipe = file_pipe_make(0, 1); + FilePair stderr_pipe = {0}; + if (!params->merge_outputs) { stderr_pipe = file_pipe_make(0, 1); } + + B32 pipes_are_ok = file_pair_ok(stdin_pipe) && file_pair_ok(stdout_pipe) && + (params->merge_outputs || file_pair_ok(stderr_pipe)); + if (!pipes_are_ok) { goto launch_failed; } + + controller->process_group = process_group_make(1); + if (controller->process_group.u64[0] == 0) { goto launch_failed; } + + ProcessLaunchParams process_params = { + .cmd_line = params->cmd_line, + .path = params->path, + .env = params->env, + .inherit_env = params->inherit_env, + .consoleless = params->consoleless, + .new_console = params->new_console, + .process_group = controller->process_group, + .stdin_file = stdin_pipe.read, + .stdout_file = stdout_pipe.write, + .stderr_file = params->merge_outputs ? stdout_pipe.write : stderr_pipe.write, + }; + controller->process = process_launch(&process_params); + + if (process_match(controller->process, process_zero())) { goto launch_failed; } + + controller->stdin_write = stdin_pipe.write; + controller->stdout_read = stdout_pipe.read; + controller->stderr_read = stderr_pipe.read; + controller->launched = 1; + + MemoryZeroStruct(&stdin_pipe.write); + MemoryZeroStruct(&stdout_pipe.read); + MemoryZeroStruct(&stderr_pipe.read); + + file_pair_close(&stdin_pipe); + file_pair_close(&stdout_pipe); + file_pair_close(&stderr_pipe); + + return t_controller_result(T_ControllerResultCode_Ok, max_U64); + +launch_failed:; + + file_pair_close(&stdin_pipe); + file_pair_close(&stdout_pipe); + file_pair_close(&stderr_pipe); + + process_group_close(controller->process_group); + MemoryZeroStruct(&controller->process_group); + controller->closed = 1; + + return t_controller_result(T_ControllerResultCode_LaunchFailed, max_U64); +} + +internal T_ControllerResult +t_controller_pump(T_Controller *controller) +{ + T_ControllerResult result = t_controller_pump_pipe(controller, controller->stdout_read, &controller->stdout_capture); + if (result.code == T_ControllerResultCode_Ok && !controller->merge_outputs) { result = t_controller_pump_pipe(controller, controller->stderr_read, &controller->stderr_capture); } + if (result.code == T_ControllerResultCode_Ok) { result = t_controller_poll_exit(controller); } + if (result.code == T_ControllerResultCode_Ok && controller->exited) { + result = t_controller_pump_pipe(controller, controller->stdout_read, &controller->stdout_capture); + if (result.code == T_ControllerResultCode_Ok && !controller->merge_outputs) { + result = t_controller_pump_pipe(controller, controller->stderr_read, &controller->stderr_capture); + } + } + return result; +} + +internal T_ControllerResult +t_controller_send(T_Controller *controller, String8 data) +{ + if (controller->exited) { return t_controller_result(T_ControllerResultCode_ProcessExited, controller->exit_code); } + + for (U64 off = 0; off < data.size;) { + U64 write_size = file_pipe_write(controller->stdin_write, data.str + off, data.size - off); + if (write_size == 0) { return t_controller_result(T_ControllerResultCode_IoFailed, controller->exit_code); } + off += write_size; + } + return t_controller_result(T_ControllerResultCode_Ok, controller->exit_code); +} + +internal T_ControllerResult +t_controller_send_line(T_Controller *controller, String8 line) +{ + T_ControllerResult result = t_controller_send(controller, line); + if (result.code == T_ControllerResultCode_Ok) { result = t_controller_send(controller, str8_lit("\n")); } + return result; +} + +internal T_ControllerResult +t_controller_wait(T_Controller *controller, U64 timeout_us) +{ + U64 endt_us = t_controller_endt_from_timeout(timeout_us); + for (;;) { + T_ControllerResult result = t_controller_pump(controller); + if (result.code != T_ControllerResultCode_Ok) { return result; } + if (controller->exited) { return result; } + if (t_controller_endt_is_expired(endt_us)) { return t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); } + sleep_ms(1); + } +} + +internal T_ControllerResult +t_controller_expect(T_Controller *controller, T_ControllerStream stream, String8 needle, U64 timeout_us) +{ + String8List *capture = t_controller_capture_from_stream(controller, stream); + U64 *cursor = t_controller_cursor_from_stream(controller, stream); + + Temp scratch = scratch_begin(&controller->arena, 1); + U64 endt_us = t_controller_endt_from_timeout(timeout_us); + T_ControllerResult result = t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); + for (;;) { + result = t_controller_pump(controller); + if (result.code != T_ControllerResultCode_Ok) { break; } + Temp iteration = temp_begin(scratch.arena); + B32 has_match = t_controller_capture_has(scratch.arena, capture, cursor, needle); + temp_end(iteration); + if (has_match) { break; } + if (controller->exited) { + result = t_controller_result(T_ControllerResultCode_ProcessExited, controller->exit_code); + break; + } + if (t_controller_endt_is_expired(endt_us)) { + result = t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); + break; + } + sleep_ms(1); + } + scratch_end(scratch); + return result; +} + +internal T_ControllerResult +t_controller_wait_until_quiet(T_Controller *controller, U64 quiet_us, U64 timeout_us) +{ + U64 endt_us = t_controller_endt_from_timeout(timeout_us); + U64 quiet_begin_us = now_time_us(); + U64 capture_size = controller->stdout_capture.total_size + controller->stderr_capture.total_size; + for (;;) { + T_ControllerResult result = t_controller_pump(controller); + if (result.code != T_ControllerResultCode_Ok) { return result; } + U64 new_capture_size = controller->stdout_capture.total_size + controller->stderr_capture.total_size; + if (new_capture_size != capture_size) { + capture_size = new_capture_size; + quiet_begin_us = now_time_us(); + } + if (controller->exited) { return t_controller_result(T_ControllerResultCode_ProcessExited, controller->exit_code); } + U64 now_us = now_time_us(); + if (now_us - quiet_begin_us >= quiet_us) { return result; } + if (endt_us != max_U64 && now_us >= endt_us) { return t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); } + sleep_ms(1); + } +} + +internal T_ControllerResult +t_controller_interrupt(T_Controller *controller) +{ + T_ControllerResultCode code = process_send_ctrl_c(controller->process) ? T_ControllerResultCode_Ok : T_ControllerResultCode_IoFailed; + return t_controller_result(code, controller->exit_code); +} + +internal void +t_controller_close(T_Controller *controller) +{ + if (controller->closed) { return; } + + t_controller_file_close(&controller->stdin_write); + + if (controller->launched) { + t_controller_pump(controller); + + process_group_close(controller->process_group); + MemoryZeroStruct(&controller->process_group); + + if (!process_match(controller->process, process_zero())) { + U64 exit_code = max_U64; + U64 endt_us = t_controller_endt_from_timeout(10 * 1000000ull); + + if (!process_join(controller->process, endt_us, &exit_code)) { + process_kill(controller->process); + process_join(controller->process, max_U64, &exit_code); + } + + MemoryZeroStruct(&controller->process); + controller->exited = 1; + controller->exit_code = exit_code; + } + + t_controller_drain_pipe(controller, controller->stdout_read, &controller->stdout_capture); + if (!controller->merge_outputs) { t_controller_drain_pipe(controller, controller->stderr_read, &controller->stderr_capture); } + } + + t_controller_file_close(&controller->stdout_read); + t_controller_file_close(&controller->stderr_read); + controller->closed = 1; +} + +internal String8 +t_controller_stdout(Arena *arena, T_Controller *controller) +{ + return str8_list_join(arena, &controller->stdout_capture, 0); +} + +internal String8 +t_controller_stderr(Arena *arena, T_Controller *controller) +{ + return str8_list_join(arena, &controller->stderr_capture, 0); +} + diff --git a/src/torture/torture_controller.h b/src/torture/torture_controller.h new file mode 100644 index 000000000..2ad79ab9b --- /dev/null +++ b/src/torture/torture_controller.h @@ -0,0 +1,77 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +// Generic subprocess transport. Suites own command formatting and response semantics. + +typedef enum T_ControllerResultCode +{ + T_ControllerResultCode_Ok, + T_ControllerResultCode_LaunchFailed, + T_ControllerResultCode_IoFailed, + T_ControllerResultCode_Timeout, + T_ControllerResultCode_ProcessExited, +} T_ControllerResultCode; + +typedef struct T_ControllerResult T_ControllerResult; +struct T_ControllerResult +{ + T_ControllerResultCode code; + U64 exit_code; +}; + +typedef enum T_ControllerStream +{ + T_ControllerStream_Stdout, + T_ControllerStream_Stderr, +} T_ControllerStream; + +typedef struct T_ControllerLaunchParams T_ControllerLaunchParams; +struct T_ControllerLaunchParams +{ + String8List cmd_line; + String8 path; + String8List env; + B32 inherit_env; + B32 consoleless; + B32 new_console; + B32 merge_outputs; // Redirect stderr into stdout while preserving one ordered transcript. +}; + +typedef struct T_Controller T_Controller; +struct T_Controller +{ + Arena *arena; + Process process; + ProcessGroup process_group; + + File stdin_write; + File stdout_read; + File stderr_read; + + String8List stdout_capture; + String8List stderr_capture; + U64 stdout_cursor; + U64 stderr_cursor; + + B32 merge_outputs; + B32 new_console; + B32 launched; + B32 exited; + B32 closed; + U64 exit_code; +}; + +internal T_ControllerResult t_controller_launch(Arena *arena, T_Controller *controller, T_ControllerLaunchParams *params); +internal T_ControllerResult t_controller_pump(T_Controller *controller); +internal T_ControllerResult t_controller_send(T_Controller *controller, String8 data); +internal T_ControllerResult t_controller_send_line(T_Controller *controller, String8 line); +internal T_ControllerResult t_controller_wait(T_Controller *controller, U64 timeout_us); +internal T_ControllerResult t_controller_expect(T_Controller *controller, T_ControllerStream stream, String8 needle, U64 timeout_us); +internal T_ControllerResult t_controller_wait_until_quiet(T_Controller *controller, U64 quiet_us, U64 timeout_us); +internal T_ControllerResult t_controller_interrupt(T_Controller *controller); +internal void t_controller_close(T_Controller *controller); + +internal String8 t_controller_stdout(Arena *arena, T_Controller *controller); +internal String8 t_controller_stderr(Arena *arena, T_Controller *controller); diff --git a/src/torture/torture_linker.c b/src/torture/torture_linker.c new file mode 100644 index 000000000..127ec8a3a --- /dev/null +++ b/src/torture/torture_linker.c @@ -0,0 +1,1512 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal B32 +t_codec_coff_section_name(String8 string_table, COFF_SectionHeader *header, String8 *name_out) +{ + String8 name = str8_cstring_capped(header->name, header->name + sizeof(header->name)); + if (name.size != 0 && name.str[0] == '/') { + U64 offset = 0; + if (!try_u64_from_str8_c_rules(str8_skip(name, 1), &offset) || offset >= string_table.size) { return 0; } + name = str8_cstring_capped(string_table.str + offset, string_table.str + string_table.size); + if (name.size >= string_table.size - offset) { return 0; } + } + *name_out = name; + return 1; +} + +internal void +t_codec_validate_fields(T_ParseContext *ctx, MD_Node *node, char **allowed) +{ + for + MD_EachNode(child, node->first) + { + B32 is_allowed = 0; + for (U64 i = 0; allowed[i] != 0; i += 1) { + if (str8_match(child->string, str8_cstring(allowed[i]), StringMatchFlag_CaseInsensitive)) { + is_allowed = 1; + break; + } + } + if (!is_allowed) { t_parse_errorf(ctx, T_ResultCode_ValidationError, child, "unknown field '%S'", child->string); } + for (MD_Node *previous = node->first; previous != child; previous = previous->next) { + if (str8_match(previous->string, child->string, StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, child, "duplicate field '%S'", child->string); + break; + } + } + } +} + +internal void +t_codec_validate_scalar(T_ParseContext *ctx, MD_Node *node, char *name, B32 required) +{ + MD_Node *field = t_codec_child(node, name); + if (md_node_is_nil(field)) { + if (required) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "operation requires %s", name); } + } else if (md_node_is_nil(field->first) || !md_node_is_nil(field->first->next)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, field, "%s requires exactly one value", name); + } +} + +internal T_Result +t_codec_pe_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + if (!pe_check_magic(data)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "artifact is not a PE image"); } + PE_BinInfo pe = pe_bin_info_from_data(ctx->arena, data); + MD_Node *root = t_codec_push_node(ctx->arena, 0, str8_lit("pe")); + t_codec_push_field(ctx, root, "is_pe32", pe.is_pe32 ? str8_lit("true") : str8_lit("false")); + t_codec_push_field(ctx, root, "arch", string_from_arch(pe.arch)); + t_codec_push_field(ctx, root, "subsystem", pe_string_from_subsystem(pe.subsystem)); + t_codec_push_u64(ctx, root, "section_count", pe.section_count); + t_codec_push_u64(ctx, root, "section_alignment", pe.virt_section_align); + t_codec_push_u64(ctx, root, "file_alignment", pe.file_section_align); + t_codec_push_u64(ctx, root, "symbol_count", pe.symbol_count); + t_codec_push_u64(ctx, root, "data_directory_count", pe.data_dir_count); + t_codec_push_u64(ctx, root, "entry_point", pe.entry_point); + + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(data, pe.section_table_range).str; + String8 string_table = str8_substr(data, pe.string_table_range); + MD_Node *section_list = t_codec_push_node(ctx->arena, root, str8_lit("sections")); + MD_Node *indexed_section_list = t_codec_push_node(ctx->arena, root, str8_lit("sections_by_index")); + for + EachIndex(i, pe.section_count) + { + COFF_SectionHeader *header = §ion_table[i]; + String8 section_name = {0}; + if (!t_codec_coff_section_name(string_table, header, §ion_name)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("pe"), "PE section %llu has an invalid name", i + 1); + } + MD_Node *section = t_codec_push_node(ctx->arena, section_list, section_name); + MD_Node *indexed_section = t_codec_push_node(ctx->arena, indexed_section_list, str8f(ctx->arena, "section_%llu", i + 1)); + t_codec_push_field(ctx, indexed_section, "name", section_name); + t_codec_push_u64(ctx, section, "virtual_size", header->vsize); + t_codec_push_u64(ctx, indexed_section, "virtual_size", header->vsize); + t_codec_push_u64(ctx, section, "virtual_offset", header->voff); + t_codec_push_u64(ctx, indexed_section, "virtual_offset", header->voff); + t_codec_push_u64(ctx, section, "file_size", header->fsize); + t_codec_push_u64(ctx, indexed_section, "file_size", header->fsize); + t_codec_push_u64(ctx, section, "file_offset", header->foff); + t_codec_push_u64(ctx, indexed_section, "file_offset", header->foff); + t_codec_push_u64(ctx, section, "alignment", coff_align_size_from_section_flags(header->flags)); + t_codec_push_u64(ctx, indexed_section, "alignment", coff_align_size_from_section_flags(header->flags)); + t_codec_push_u64(ctx, section, "raw_flags", header->flags); + t_codec_push_u64(ctx, indexed_section, "raw_flags", header->flags); + if (header->fsize != 0 && (header->foff > data.size || header->fsize > data.size - header->foff)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("pe"), "PE section %llu has an invalid raw data range", i + 1); + } + U64 section_data_size = Min(header->vsize, header->fsize); + if (section_data_size != 0 && (header->foff > data.size || section_data_size > data.size - header->foff)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("pe"), "PE section %llu has an invalid data range", i + 1); + } + if (section_data_size != 0) { + String8 section_data = str8_substr(data, rng_1u64(header->foff, header->foff + section_data_size)); + t_codec_push_field(ctx, section, "data", t_hex_from_data(ctx->arena, section_data)); + t_codec_push_field(ctx, indexed_section, "data", t_hex_from_data(ctx->arena, section_data)); + } + } + + local_persist char *directory_names[] = {"exports", "imports", "resources", "exceptions", "certificates", "base_relocations", + "debug", "architecture", "global_ptr", "tls", "load_config", "bound_imports", + "import_address", "delay_imports", "com_descriptor", "reserved"}; + MD_Node *directories = t_codec_push_node(ctx->arena, root, str8_lit("data_directories")); + U64 directory_count = Min(pe.data_dir_count, ArrayCount(directory_names)); + for EachIndex(i, directory_count) + { + MD_Node *directory = t_codec_push_node(ctx->arena, directories, str8_cstring(directory_names[i])); + t_codec_push_u64(ctx, directory, "virtual_offset", pe.data_dir_vranges[i].min); + t_codec_push_u64(ctx, directory, "virtual_size", dim_1u64(pe.data_dir_vranges[i])); + t_codec_push_u64(ctx, directory, "file_offset", pe.data_dir_franges[i].min); + t_codec_push_u64(ctx, directory, "file_size", dim_1u64(pe.data_dir_franges[i])); + } + + U16 *optional_magic = str8_deserial_get_raw_ptr(data, pe.optional_header_off, sizeof(*optional_magic)); + U64 optional_header_size = pe.section_table_range.min >= pe.optional_header_off ? pe.section_table_range.min - pe.optional_header_off : 0; + if (optional_magic == 0 || (*optional_magic != PE_PE32_MAGIC && *optional_magic != PE_PE32PLUS_MAGIC)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "PE optional header has invalid magic"); + } + if (*optional_magic == PE_PE32PLUS_MAGIC) { + if (optional_header_size < sizeof(PE_OptionalHeader32Plus)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "declared PE optional header is truncated"); + } + PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(data, pe.optional_header_off, sizeof(*opt)); + if (opt == 0) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "PE optional header is truncated"); } + MD_Node *optional = t_codec_push_node(ctx->arena, root, str8_lit("optional")); + t_codec_push_u64(ctx, optional, "sizeof_code", opt->sizeof_code); + t_codec_push_u64(ctx, optional, "sizeof_initialized_data", opt->sizeof_inited_data); + t_codec_push_u64(ctx, optional, "sizeof_uninitialized_data", opt->sizeof_uninited_data); + t_codec_push_u64(ctx, optional, "code_base", opt->code_base); + t_codec_push_u64(ctx, optional, "image_base", opt->image_base); + t_codec_push_u64(ctx, optional, "major_os_version", opt->major_os_ver); + t_codec_push_u64(ctx, optional, "minor_os_version", opt->minor_os_ver); + t_codec_push_u64(ctx, optional, "major_image_version", opt->major_img_ver); + t_codec_push_u64(ctx, optional, "minor_image_version", opt->minor_img_ver); + t_codec_push_u64(ctx, optional, "major_subsystem_version", opt->major_subsystem_ver); + t_codec_push_u64(ctx, optional, "minor_subsystem_version", opt->minor_subsystem_ver); + t_codec_push_u64(ctx, optional, "win32_version", opt->win32_version_value); + t_codec_push_u64(ctx, optional, "sizeof_image", opt->sizeof_image); + t_codec_push_u64(ctx, optional, "sizeof_headers", opt->sizeof_headers); + t_codec_push_u64(ctx, optional, "dll_characteristics", opt->dll_characteristics); + t_codec_push_u64(ctx, optional, "sizeof_stack_reserve", opt->sizeof_stack_reserve); + t_codec_push_u64(ctx, optional, "sizeof_stack_commit", opt->sizeof_stack_commit); + t_codec_push_u64(ctx, optional, "sizeof_heap_reserve", opt->sizeof_heap_reserve); + t_codec_push_u64(ctx, optional, "sizeof_heap_commit", opt->sizeof_heap_commit); + t_codec_push_u64(ctx, optional, "loader_flags", opt->loader_flags); + } else { + if (optional_header_size < sizeof(PE_OptionalHeader32)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "declared PE optional header is truncated"); + } + PE_OptionalHeader32 *opt = str8_deserial_get_raw_ptr(data, pe.optional_header_off, sizeof(*opt)); + if (opt == 0) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "PE optional header is truncated"); } + MD_Node *optional = t_codec_push_node(ctx->arena, root, str8_lit("optional")); + t_codec_push_u64(ctx, optional, "sizeof_code", opt->sizeof_code); + t_codec_push_u64(ctx, optional, "sizeof_initialized_data", opt->sizeof_inited_data); + t_codec_push_u64(ctx, optional, "sizeof_uninitialized_data", opt->sizeof_uninited_data); + t_codec_push_u64(ctx, optional, "code_base", opt->code_base); + t_codec_push_u64(ctx, optional, "data_base", opt->data_base); + t_codec_push_u64(ctx, optional, "image_base", opt->image_base); + t_codec_push_u64(ctx, optional, "major_os_version", opt->major_os_ver); + t_codec_push_u64(ctx, optional, "minor_os_version", opt->minor_os_ver); + t_codec_push_u64(ctx, optional, "major_image_version", opt->major_img_ver); + t_codec_push_u64(ctx, optional, "minor_image_version", opt->minor_img_ver); + t_codec_push_u64(ctx, optional, "major_subsystem_version", opt->major_subsystem_ver); + t_codec_push_u64(ctx, optional, "minor_subsystem_version", opt->minor_subsystem_ver); + t_codec_push_u64(ctx, optional, "win32_version", opt->win32_version_value); + t_codec_push_u64(ctx, optional, "sizeof_image", opt->sizeof_image); + t_codec_push_u64(ctx, optional, "sizeof_headers", opt->sizeof_headers); + t_codec_push_u64(ctx, optional, "dll_characteristics", opt->dll_characteristics); + t_codec_push_u64(ctx, optional, "sizeof_stack_reserve", opt->sizeof_stack_reserve); + t_codec_push_u64(ctx, optional, "sizeof_stack_commit", opt->sizeof_stack_commit); + t_codec_push_u64(ctx, optional, "sizeof_heap_reserve", opt->sizeof_heap_reserve); + t_codec_push_u64(ctx, optional, "sizeof_heap_commit", opt->sizeof_heap_commit); + t_codec_push_u64(ctx, optional, "loader_flags", opt->loader_flags); + } + + if (pe.data_dir_count > PE_DataDirectoryIndex_EXPORT && dim_1u64(pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]) != 0) { + PE_ParsedExportTable table = pe_exports_from_data(ctx->arena, pe.section_count, section_table, data, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], + pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); + MD_Node *exports = t_codec_push_node(ctx->arena, root, str8_lit("exports")); + t_codec_push_u64(ctx, exports, "flags", table.flags); + t_codec_push_u64(ctx, exports, "timestamp", table.time_stamp); + t_codec_push_u64(ctx, exports, "major_version", table.major_ver); + t_codec_push_u64(ctx, exports, "minor_version", table.minor_ver); + t_codec_push_u64(ctx, exports, "ordinal_base", table.ordinal_base); + t_codec_push_u64(ctx, exports, "count", table.export_count); + MD_Node *entries = t_codec_push_node(ctx->arena, exports, str8_lit("entries")); + for EachIndex(i, table.export_count) + { + PE_ParsedExport *entry = &table.exports[i]; + MD_Node *entry_node = t_codec_push_node(ctx->arena, entries, str8f(ctx->arena, "export_%llu", i)); + t_codec_push_field(ctx, entry_node, "name", entry->name); + t_codec_push_field(ctx, entry_node, "forwarder", entry->forwarder); + t_codec_push_u64(ctx, entry_node, "virtual_offset", entry->voff); + t_codec_push_u64(ctx, entry_node, "ordinal", entry->ordinal); + } + } + + if (pe.data_dir_count > PE_DataDirectoryIndex_IMPORT && dim_1u64(pe.data_dir_vranges[PE_DataDirectoryIndex_IMPORT]) != 0) { + PE_ParsedStaticImportTable table = pe_static_imports_from_data(ctx->arena, pe.is_pe32, pe.section_count, section_table, data, pe.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); + MD_Node *imports = t_codec_push_node(ctx->arena, root, str8_lit("imports")); + t_codec_push_u64(ctx, imports, "count", table.count); + for EachIndex(dll_idx, table.count) + { + PE_ParsedStaticDLLImport *dll = &table.v[dll_idx]; + MD_Node *dll_node = t_codec_push_node(ctx->arena, imports, str8f(ctx->arena, "dll_%llu", dll_idx)); + t_codec_push_field(ctx, dll_node, "name", dll->name); + t_codec_push_u64(ctx, dll_node, "import_address_table", dll->import_address_table_voff); + t_codec_push_u64(ctx, dll_node, "import_name_table", dll->import_name_table_voff); + t_codec_push_u64(ctx, dll_node, "timestamp", dll->time_stamp); + t_codec_push_u64(ctx, dll_node, "forwarder_chain", dll->forwarder_chain); + t_codec_push_u64(ctx, dll_node, "count", dll->import_count); + MD_Node *entries = t_codec_push_node(ctx->arena, dll_node, str8_lit("entries")); + for EachIndex(import_idx, dll->import_count) + { + PE_ParsedImport *import = &dll->imports[import_idx]; + MD_Node *import_node = t_codec_push_node(ctx->arena, entries, str8f(ctx->arena, "import_%llu", import_idx)); + if (import->type == PE_ParsedImport_Name) { + t_codec_push_field(ctx, import_node, "type", str8_lit("name")); + t_codec_push_field(ctx, import_node, "name", import->u.name.string); + t_codec_push_u64(ctx, import_node, "hint", import->u.name.hint); + } else if (import->type == PE_ParsedImport_Ordinal) { + t_codec_push_field(ctx, import_node, "type", str8_lit("ordinal")); + t_codec_push_u64(ctx, import_node, "ordinal", import->u.ordinal); + } + } + } + } + + if (pe.data_dir_count > PE_DataDirectoryIndex_DELAY_IMPORT && dim_1u64(pe.data_dir_vranges[PE_DataDirectoryIndex_DELAY_IMPORT]) != 0) { + PE_ParsedDelayImportTable table = pe_delay_imports_from_data(ctx->arena, pe.is_pe32, pe.section_count, section_table, data, pe.data_dir_franges[PE_DataDirectoryIndex_DELAY_IMPORT]); + MD_Node *imports = t_codec_push_node(ctx->arena, root, str8_lit("delay_imports")); + t_codec_push_u64(ctx, imports, "count", table.count); + for EachIndex(dll_idx, table.count) + { + PE_ParsedDelayDLLImport *dll = &table.v[dll_idx]; + MD_Node *dll_node = t_codec_push_node(ctx->arena, imports, str8f(ctx->arena, "dll_%llu", dll_idx)); + t_codec_push_field(ctx, dll_node, "name", dll->name); + t_codec_push_u64(ctx, dll_node, "attributes", dll->attributes); + t_codec_push_u64(ctx, dll_node, "module_handle", dll->module_handle_voff); + t_codec_push_u64(ctx, dll_node, "import_address_table", dll->iat_voff); + t_codec_push_u64(ctx, dll_node, "import_name_table", dll->name_table_voff); + t_codec_push_u64(ctx, dll_node, "bound_table", dll->bound_table_voff); + t_codec_push_u64(ctx, dll_node, "unload_table", dll->unload_table_voff); + t_codec_push_u64(ctx, dll_node, "timestamp", dll->time_stamp); + t_codec_push_u64(ctx, dll_node, "bound_count", dll->bound_table_count); + t_codec_push_u64(ctx, dll_node, "unload_count", dll->unload_table_count); + t_codec_push_u64(ctx, dll_node, "count", dll->import_count); + MD_Node *entries = t_codec_push_node(ctx->arena, dll_node, str8_lit("entries")); + for EachIndex(import_idx, dll->import_count) + { + PE_ParsedImport *import = &dll->imports[import_idx]; + MD_Node *import_node = t_codec_push_node(ctx->arena, entries, str8f(ctx->arena, "import_%llu", import_idx)); + if (import->type == PE_ParsedImport_Name) { + t_codec_push_field(ctx, import_node, "type", str8_lit("name")); + t_codec_push_field(ctx, import_node, "name", import->u.name.string); + t_codec_push_u64(ctx, import_node, "hint", import->u.name.hint); + } else if (import->type == PE_ParsedImport_Ordinal) { + t_codec_push_field(ctx, import_node, "type", str8_lit("ordinal")); + t_codec_push_u64(ctx, import_node, "ordinal", import->u.ordinal); + } + } + } + } + + *semantic_tree_out = root; + return ctx->result; +} + +typedef struct T_PdbMsfInfo T_PdbMsfInfo; +struct T_PdbMsfInfo +{ + U64 stream_count; + B32 *stream_present; +}; + +typedef struct T_PdbKindCount T_PdbKindCount; +struct T_PdbKindCount +{ + T_PdbKindCount *next; + U16 kind; + U64 count; +}; + +internal B32 +t_codec_pdb_range_is_valid(U64 off, U64 size, U64 cap) +{ + return off <= cap && size <= cap - off; +} + +internal B32 +t_codec_pdb_cstr_from_offset(String8 data, U64 off, String8 *string_out) +{ + if (off >= data.size) { return 0; } + String8 string = str8_cstring_capped(data.str + off, data.str + data.size); + if (string.size >= data.size - off) { return 0; } + *string_out = string; + return 1; +} + +internal String8 +t_codec_pdb_sym_kind_name(Arena *arena, CV_SymKind kind) +{ + String8 name = cv_string_from_symbol_kind(arena, kind); + if (name.size == 2) { name = str8f(arena, "S_0x%04x", kind); } + return name; +} + +internal String8 +t_codec_pdb_leaf_kind_name(Arena *arena, CV_LeafKind kind) +{ + String8 name = cv_string_from_leaf_name(arena, kind); + if (name.size == 3) { name = str8f(arena, "LF_0x%04x", kind); } + return name; +} + +internal B32 +t_codec_pdb_validate_msf(Arena *arena, String8 data, T_PdbMsfInfo *info, String8 *error) +{ + if (!msf_check_magic_70(data) || data.size < sizeof(MSF_Header70)) { + *error = str8_lit("file is not an MSF 7.0 PDB"); + return 0; + } + MSF_Header70 *header = (MSF_Header70 *)data.str; + U64 page_size = header->page_size; + U64 page_count = header->page_count; + if (page_size < MSF_MIN_PAGE_SIZE || page_size > MSF_MAX_PAGE_SIZE || !IsPow2OrZero(page_size)) { + *error = str8_lit("MSF has an invalid page size"); + return 0; + } + if (page_count == 0) { + *error = str8_lit("MSF has an invalid page count"); + return 0; + } + U64 directory_size = header->stream_table_size; + U64 directory_page_count = CeilIntegerDiv(directory_size, page_size); + U64 directory_map_size = directory_page_count * sizeof(U32); + if (directory_size < sizeof(U32) || directory_size > data.size || directory_map_size > page_size || header->root_pn >= page_count) { + *error = str8_lit("MSF stream directory map is invalid"); + return 0; + } + U64 map_off = (U64)header->root_pn * page_size; + if (!t_codec_pdb_range_is_valid(map_off, directory_map_size, data.size)) { + *error = str8_lit("MSF stream directory map is truncated"); + return 0; + } + U8 *directory = push_array_no_zero(arena, U8, directory_size); + for EachIndex(i, directory_page_count) + { + U32 pn = 0; + MemoryCopy(&pn, data.str + map_off + i * sizeof(pn), sizeof(pn)); + U64 copy_size = Min(page_size, directory_size - i * page_size); + if (pn >= page_count || !t_codec_pdb_range_is_valid((U64)pn * page_size, copy_size, data.size)) { + *error = str8f(arena, "MSF stream directory page %llu is invalid", i); + return 0; + } + MemoryCopy(directory + i * page_size, data.str + (U64)pn * page_size, copy_size); + } + U32 stream_count = 0; + MemoryCopy(&stream_count, directory, sizeof(stream_count)); + if (stream_count > (directory_size - sizeof(U32)) / sizeof(U32)) { + *error = str8_lit("MSF stream size table is truncated"); + return 0; + } + U64 index_cursor = sizeof(U32) + (U64)stream_count * sizeof(U32); + B32 *present = push_array(arena, B32, stream_count); + for EachIndex(i, stream_count) + { + U32 stream_size = 0; + MemoryCopy(&stream_size, directory + sizeof(U32) + i * sizeof(U32), sizeof(stream_size)); + if (stream_size == MSF_DELETED_STREAM_STAMP) { continue; } + present[i] = 1; + U64 stream_page_count = CeilIntegerDiv((U64)stream_size, page_size); + U64 indices_size = stream_page_count * sizeof(U32); + if (!t_codec_pdb_range_is_valid(index_cursor, indices_size, directory_size)) { + *error = str8f(arena, "MSF page list for stream %llu is truncated", i); + return 0; + } + for EachIndex(page_idx, stream_page_count) + { + U32 pn = 0; + MemoryCopy(&pn, directory + index_cursor + page_idx * sizeof(pn), sizeof(pn)); + U64 page_data_size = Min(page_size, (U64)stream_size - page_idx * page_size); + if (pn >= page_count || !t_codec_pdb_range_is_valid((U64)pn * page_size, page_data_size, data.size)) { + *error = str8f(arena, "MSF stream %llu references invalid page %u", i, pn); + return 0; + } + } + index_cursor += indices_size; + } + info->stream_count = stream_count; + info->stream_present = present; + return 1; +} + +internal B32 +t_codec_pdb_symbol_name(CV_Symbol symbol, String8 *name_out) +{ + U64 off = symbol.data.size; + switch (symbol.kind) + { + case CV_SymKind_OBJNAME: off = sizeof(CV_SymObjName); break; + case CV_SymKind_CONSTANT: + { + off = sizeof(CV_SymConstant); + if (off > symbol.data.size) { return 0; } + CV_NumericParsed numeric = {0}; + U64 numeric_size = cv_read_numeric(symbol.data, off, &numeric); + if (numeric_size == 0) { return 0; } + off += numeric_size; + } break; + case CV_SymKind_UDT: off = sizeof(CV_SymUDT); break; + case CV_SymKind_LDATA32: case CV_SymKind_GDATA32: + case CV_SymKind_LTHREAD32: case CV_SymKind_GTHREAD32: off = sizeof(CV_SymData32); break; + case CV_SymKind_PUB32: off = sizeof(CV_SymPub32); break; + case CV_SymKind_LPROC32: case CV_SymKind_GPROC32: + case CV_SymKind_LPROC32_ID: case CV_SymKind_GPROC32_ID: off = sizeof(CV_SymProc32); break; + case CV_SymKind_PROCREF: case CV_SymKind_LPROCREF: case CV_SymKind_DATAREF: off = sizeof(CV_SymRef2); break; + default: return 0; + } + return t_codec_pdb_cstr_from_offset(symbol.data, off, name_out); +} + +internal B32 +t_codec_pdb_read_symbol(String8 data, U64 off, U64 align, CV_Symbol *symbol_out, U64 *read_size_out, String8 *error) +{ + if (!t_codec_pdb_range_is_valid(off, sizeof(CV_SymbolHeader), data.size)) { + *error = str8_lit("CodeView symbol header is truncated"); + return 0; + } + CV_SymbolHeader header = {0}; + MemoryCopy(&header, data.str + off, sizeof(header)); + U64 raw_size = sizeof(CV_SymSize) + header.size; + if (header.size < sizeof(CV_SymKind) || !t_codec_pdb_range_is_valid(off, raw_size, data.size)) { + *error = str8_lit("CodeView symbol record is truncated"); + return 0; + } + U64 read_size = AlignPow2(raw_size, align); + if (read_size < raw_size || !t_codec_pdb_range_is_valid(off, read_size, data.size)) { + *error = str8_lit("CodeView symbol padding is truncated"); + return 0; + } + String8 record_data = str8(data.str + off + sizeof(CV_SymbolHeader), header.size - sizeof(CV_SymKind)); + U64 fixed_size = cv_header_struct_size_from_sym_kind(header.kind); + if (CV_IsProc32(header.kind)) { fixed_size = sizeof(CV_SymProc32); } + if (fixed_size > record_data.size) { + *error = str8_lit("CodeView symbol fixed fields are truncated"); + return 0; + } + CV_Symbol symbol = {0}; + cv_read_symbol(str8_skip(data, off), 0, align, &symbol); + String8 ignored = {0}; + if ((header.kind == CV_SymKind_CONSTANT || header.kind == CV_SymKind_UDT || header.kind == CV_SymKind_PUB32 || header.kind == CV_SymKind_PROCREF || + header.kind == CV_SymKind_LPROCREF || header.kind == CV_SymKind_DATAREF || CV_IsProc32(header.kind) || header.kind == CV_SymKind_LDATA32 || + header.kind == CV_SymKind_GDATA32) && !t_codec_pdb_symbol_name(symbol, &ignored)) { + *error = str8_lit("CodeView symbol name is truncated"); + return 0; + } + *symbol_out = symbol; + *read_size_out = read_size; + return 1; +} + +internal MD_Node * +t_codec_pdb_push_symbol(T_Context *ctx, MD_Node *parent, String8 key, CV_Symbol symbol, U64 off) +{ + MD_Node *node = t_codec_push_node(ctx->arena, parent, key); + t_codec_push_field(ctx, node, "kind", t_codec_pdb_sym_kind_name(ctx->arena, symbol.kind)); + t_codec_push_u64(ctx, node, "kind_value", symbol.kind); + t_codec_push_u64(ctx, node, "offset", off); + t_codec_push_u64(ctx, node, "data_size", symbol.data.size); + String8 name = {0}; + if (t_codec_pdb_symbol_name(symbol, &name)) { t_codec_push_field(ctx, node, "name", name); } + if (symbol.kind == CV_SymKind_PROCREF || symbol.kind == CV_SymKind_LPROCREF || symbol.kind == CV_SymKind_DATAREF) { + CV_SymRef2 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "suc_name", value.suc_name); + t_codec_push_u64(ctx, node, "sym_off", value.sym_off); + t_codec_push_u64(ctx, node, "imod", value.imod); + } else if (symbol.kind == CV_SymKind_PUB32) { + CV_SymPub32 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "flags", value.flags); + t_codec_push_u64(ctx, node, "section", value.sec); + t_codec_push_u64(ctx, node, "section_offset", value.off); + } else if (symbol.kind == CV_SymKind_LDATA32 || symbol.kind == CV_SymKind_GDATA32) { + CV_SymData32 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "type_index", value.itype); + t_codec_push_u64(ctx, node, "section", value.sec); + t_codec_push_u64(ctx, node, "section_offset", value.off); + } else if (CV_IsProc32(symbol.kind)) { + CV_SymProc32 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "type_index", value.itype); + t_codec_push_u64(ctx, node, "section", value.sec); + t_codec_push_u64(ctx, node, "section_offset", value.off); + t_codec_push_u64(ctx, node, "length", value.len); + } + return node; +} + +internal B32 +t_codec_pdb_push_symbol_stream(T_Context *ctx, MD_Node *parent, String8 data, U64 align, String8 *error) +{ + MD_Node *symbols = t_codec_push_node(ctx->arena, parent, str8_lit("symbols")); + MD_Node *by_name = t_codec_push_node(ctx->arena, parent, str8_lit("by_name")); + T_PdbKindCount *first_count = 0; + U64 count = 0; + U64 proc_stub_count = 0; + U64 public_or_proc_ref_count = 0; + for (U64 cursor = 0; cursor < data.size; count += 1) + { + CV_Symbol symbol = {0}; U64 read_size = 0; + if (!t_codec_pdb_read_symbol(data, cursor, align, &symbol, &read_size, error)) { return 0; } + MD_Node *symbol_node = t_codec_pdb_push_symbol(ctx, symbols, str8f(ctx->arena, "symbol_%llu", count), symbol, cursor); + String8 name = {0}; + if (t_codec_pdb_symbol_name(symbol, &name) && md_node_is_nil(md_child_from_string(by_name, name, 0))) { + MD_Node *named = t_codec_push_node(ctx->arena, by_name, name); + for MD_EachNode(field, symbol_node->first) { MD_Node *copy = t_codec_push_node(ctx->arena, named, field->string); t_codec_push_node(ctx->arena, copy, field->first->string); } + } + T_PdbKindCount *kind_count = first_count; + for (; kind_count != 0 && kind_count->kind != symbol.kind; kind_count = kind_count->next) {} + if (kind_count == 0) { kind_count = push_array(ctx->arena, T_PdbKindCount, 1); kind_count->kind = symbol.kind; kind_count->next = first_count; first_count = kind_count; } + kind_count->count += 1; + proc_stub_count += symbol.kind == CV_SymKind_LPROC32 || symbol.kind == CV_SymKind_END; + public_or_proc_ref_count += symbol.kind == CV_SymKind_PUB32 || symbol.kind == CV_SymKind_LPROCREF; + cursor += read_size; + } + t_codec_push_u64(ctx, parent, "size", data.size); + t_codec_push_u64(ctx, parent, "count", count); + t_codec_push_u64(ctx, parent, "proc_stub_symbol_count", proc_stub_count); + t_codec_push_u64(ctx, parent, "non_proc_stub_symbol_count", count - proc_stub_count); + t_codec_push_u64(ctx, parent, "public_or_proc_ref_symbol_count", public_or_proc_ref_count); + t_codec_push_u64(ctx, parent, "non_public_or_proc_ref_symbol_count", count - public_or_proc_ref_count); + MD_Node *kind_counts = t_codec_push_node(ctx->arena, parent, str8_lit("kind_counts")); + MD_Node *kind_set = t_codec_push_node(ctx->arena, parent, str8_lit("kind_set")); + for (T_PdbKindCount *item = first_count; item != 0; item = item->next) { + String8 name = t_codec_pdb_sym_kind_name(ctx->arena, item->kind); + t_codec_push_u64_s8(ctx, kind_counts, name, item->count); + t_codec_push_node(ctx->arena, kind_set, name); + } + return 1; +} + +internal B32 +t_codec_pdb_push_tpi(T_Context *ctx, MD_Node *root, char *name, MSF_Parsed *msf, MSF_StreamNumber sn, String8 *error) +{ + MD_Node *node = t_codec_push_node(ctx->arena, root, str8_cstring(name)); + String8 data = msf_data_from_stream(msf, sn); + t_codec_push_u64(ctx, node, "stream_index", sn); + t_codec_push_u64(ctx, node, "size", data.size); + if (data.size < sizeof(PDB_TpiHeader)) { *error = str8f(ctx->arena, "%s stream header is truncated", name); return 0; } + PDB_TpiHeader *header = (PDB_TpiHeader *)data.str; + if (header->version != PDB_TpiVersion_IMPV80 || header->header_size < sizeof(*header) || header->header_size > data.size || + header->leaf_data_size > data.size - header->header_size || header->ti_hi < header->ti_lo) { + *error = str8f(ctx->arena, "%s stream header is invalid", name); return 0; + } + U64 expected_count = (U64)header->ti_hi - header->ti_lo; + String8 leaf_data = str8(data.str + header->header_size, header->leaf_data_size); + U64 parsed_count = 0; + for (U64 cursor = 0; cursor < leaf_data.size; parsed_count += 1) { + CV_Leaf leaf = {0}; U64 read_size = cv_read_leaf(str8_skip(leaf_data, cursor), 0, PDB_LEAF_ALIGN, &leaf); + if (read_size == 0 || cv_header_struct_size_from_leaf_kind(leaf.kind) > leaf.data.size) { + *error = str8f(ctx->arena, "%s leaf %llu is malformed", name, parsed_count); return 0; + } + cursor += read_size; + } + if (parsed_count != expected_count) { *error = str8f(ctx->arena, "%s type index range does not match its leaf count", name); return 0; } + PDB_TpiParsed *tpi = pdb_tpi_from_data(ctx->arena, data); + CV_DebugT debug_t = cv_debug_t_from_data(ctx->arena, pdb_leaf_data_from_tpi(tpi), PDB_LEAF_ALIGN); + if (debug_t.count != parsed_count) { *error = str8f(ctx->arena, "%s CodeView leaf parse is inconsistent", name); return 0; } + t_codec_push_u64(ctx, node, "header_size", header->header_size); + t_codec_push_u64(ctx, node, "index_first", tpi->itype_first); + t_codec_push_u64(ctx, node, "index_opl", tpi->itype_opl); + t_codec_push_u64(ctx, node, "leaf_count", parsed_count); + t_codec_push_field(ctx, node, "header_only", parsed_count == 0 && data.size == header->header_size ? str8_lit("true") : str8_lit("false")); + MD_Node *leaves = t_codec_push_node(ctx->arena, node, str8_lit("leaves")); + for EachIndex(i, debug_t.count) + { + CV_Leaf leaf = cv_debug_t_get_leaf(&debug_t, i); + MD_Node *leaf_node = t_codec_push_node(ctx->arena, leaves, str8f(ctx->arena, "leaf_%llu", i)); + t_codec_push_field(ctx, leaf_node, "kind", t_codec_pdb_leaf_kind_name(ctx->arena, leaf.kind)); + t_codec_push_u64(ctx, leaf_node, "kind_value", leaf.kind); + t_codec_push_u64(ctx, leaf_node, "type_index", tpi->itype_first + i); + t_codec_push_u64(ctx, leaf_node, "data_size", leaf.data.size); + if (leaf.kind == CV_LeafKind_POINTER) { + CV_LeafPointer value = {0}; MemoryCopy(&value, leaf.data.str, sizeof(value)); + t_codec_push_u64(ctx, leaf_node, "type", value.itype); t_codec_push_u64(ctx, leaf_node, "attributes", value.attribs); + } else if (leaf.kind == CV_LeafKind_PROCEDURE) { + CV_LeafProcedure value = {0}; MemoryCopy(&value, leaf.data.str, sizeof(value)); + t_codec_push_u64(ctx, leaf_node, "return_type", value.ret_itype); t_codec_push_u64(ctx, leaf_node, "call_kind", value.call_kind); + t_codec_push_u64(ctx, leaf_node, "attributes", value.attribs); t_codec_push_u64(ctx, leaf_node, "argument_count", value.arg_count); + t_codec_push_u64(ctx, leaf_node, "argument_list_type", value.arg_itype); + } + } + return 1; +} + +internal B32 +t_codec_pdb_validate_gsi(T_Context *ctx, String8 data, String8 symbols, String8 *error) +{ + if (data.size < sizeof(PDB_GsiHeader)) { *error = str8_lit("GSI header is truncated"); return 0; } + PDB_GsiHeader *header = (PDB_GsiHeader *)data.str; + U64 bitmap_size = CeilIntegerDiv(4097, 32) * sizeof(U32); + U64 hash_off = sizeof(*header); + U64 bitmap_off = hash_off + header->hash_record_arr_size; + U64 total_size = bitmap_off + header->bucket_data_size; + if (header->signature != PDB_GsiSignature_Basic || header->version != PDB_GsiVersion_V70 || header->hash_record_arr_size % sizeof(PDB_GsiHashRecord) != 0 || + header->bucket_data_size < bitmap_size || total_size != data.size) { + *error = str8_lit("GSI header or ranges are invalid"); return 0; + } + U64 hash_count = header->hash_record_arr_size / sizeof(PDB_GsiHashRecord); + U64 packed_count = (header->bucket_data_size - bitmap_size) / sizeof(U32); + U64 occupied_count = 0; + for EachIndex(i, CeilIntegerDiv(4097, 32)) { U32 bits = 0; MemoryCopy(&bits, data.str + bitmap_off + i * sizeof(bits), sizeof(bits)); occupied_count += count_bits_set32(bits); } + if ((header->bucket_data_size - bitmap_size) % sizeof(U32) != 0 || occupied_count != packed_count) { *error = str8_lit("GSI bucket bitmap and offset array disagree"); return 0; } + U64 offsets_off = bitmap_off + bitmap_size; + U32 previous = 0; + for EachIndex(i, packed_count) { + U32 value = 0; MemoryCopy(&value, data.str + offsets_off + i * sizeof(value), sizeof(value)); + if ((i == 0 && hash_count != 0 && value != 0) || value % sizeof(PDB_GsiHashRecordOffsetCalc) != 0 || + value / sizeof(PDB_GsiHashRecordOffsetCalc) > hash_count || (i != 0 && value < previous)) { + *error = str8_lit("GSI bucket offset is invalid"); return 0; + } + previous = value; + } + if (hash_count != 0 && packed_count == 0) { *error = str8_lit("GSI has hash records but no occupied bucket"); return 0; } + for EachIndex(i, hash_count) { + PDB_GsiHashRecord record = {0}; MemoryCopy(&record, data.str + hash_off + i * sizeof(record), sizeof(record)); + if (record.symbol_off == 0) { *error = str8_lit("GSI references the null symbol offset"); return 0; } + CV_Symbol symbol = {0}; U64 read_size = 0; + if (!t_codec_pdb_read_symbol(symbols, record.symbol_off - 1, 1, &symbol, &read_size, error)) { return 0; } + String8 name = {0}; + if (!t_codec_pdb_symbol_name(symbol, &name)) { *error = str8_lit("GSI references a symbol without a valid name"); return 0; } + } + return 1; +} + +internal B32 +t_codec_pdb_push_gsi(T_Context *ctx, MD_Node *root, char *name, MSF_Parsed *msf, MSF_StreamNumber sn, String8 symbols, B32 is_psi, String8 *error) +{ + MD_Node *node = t_codec_push_node(ctx->arena, root, str8_cstring(name)); + t_codec_push_u64(ctx, node, "stream_index", sn); + if (sn >= msf->stream_count) { t_codec_push_field(ctx, node, "present", str8_lit("false")); return 1; } + String8 stream = msf_data_from_stream(msf, sn); + t_codec_push_field(ctx, node, "present", str8_lit("true")); + t_codec_push_u64(ctx, node, "size", stream.size); + String8 gsi_data = stream; + if (is_psi) { + if (stream.size < sizeof(PDB_PsiHeader)) { *error = str8_lit("PSI header is truncated"); return 0; } + PDB_PsiHeader *header = (PDB_PsiHeader *)stream.str; + U64 ranges_size = sizeof(*header); + U64 range_sizes[] = {header->sym_hash_size, header->addr_map_size}; + for EachElement(i, range_sizes) { if (range_sizes[i] > stream.size - Min(ranges_size, stream.size)) { *error = str8_lit("PSI ranges are truncated"); return 0; } ranges_size += range_sizes[i]; } + if (header->thunk_count != 0 && header->thunk_size > (stream.size - Min(ranges_size, stream.size)) / header->thunk_count) { *error = str8_lit("PSI thunk table is truncated"); return 0; } + ranges_size += (U64)header->thunk_count * header->thunk_size; + if ((U64)header->sec_count > (stream.size - Min(ranges_size, stream.size)) / sizeof(U32)) { *error = str8_lit("PSI section table is truncated"); return 0; } + t_codec_push_u64(ctx, node, "address_map_size", header->addr_map_size); + t_codec_push_u64(ctx, node, "thunk_count", header->thunk_count); + t_codec_push_u64(ctx, node, "section_count", header->sec_count); + gsi_data = str8(stream.str + sizeof(*header), header->sym_hash_size); + } + if (!t_codec_pdb_validate_gsi(ctx, gsi_data, symbols, error)) { return 0; } + PDB_GsiParsed *gsi = pdb_gsi_from_data(ctx->arena, gsi_data); + MD_Node *indexed = t_codec_push_node(ctx->arena, node, str8_lit("symbols")); + MD_Node *kind_counts = t_codec_push_node(ctx->arena, node, str8_lit("kind_counts")); + T_PdbKindCount *first_count = 0; + U64 count = 0; + for EachElement(bucket_idx, gsi->buckets) { + PDB_GsiBucket bucket = gsi->buckets[bucket_idx]; + for EachIndex(i, bucket.count) { + U64 off = bucket.offs[i]; CV_Symbol symbol = {0}; U64 read_size = 0; + if (!t_codec_pdb_read_symbol(symbols, off, 1, &symbol, &read_size, error)) { return 0; } + String8 symbol_name = {0}; t_codec_pdb_symbol_name(symbol, &symbol_name); + if (pdb_gsi_symbol_from_string(gsi, symbols, symbol_name) >= symbols.size) { *error = str8_lit("GSI symbol is not queryable by name"); return 0; } + if (md_node_is_nil(md_child_from_string(indexed, symbol_name, 0))) { t_codec_pdb_push_symbol(ctx, indexed, symbol_name, symbol, off); } + T_PdbKindCount *item = first_count; for (; item != 0 && item->kind != symbol.kind; item = item->next) {} + if (item == 0) { item = push_array(ctx->arena, T_PdbKindCount, 1); item->kind = symbol.kind; item->next = first_count; first_count = item; } + item->count += 1; count += 1; + } + } + for (T_PdbKindCount *item = first_count; item != 0; item = item->next) { String8 kind = t_codec_pdb_sym_kind_name(ctx->arena, item->kind); t_codec_push_u64_s8(ctx, kind_counts, kind, item->count); } + t_codec_push_u64(ctx, node, "indexed_symbol_count", count); + return 1; +} + +internal B32 +t_codec_pdb_push_dbi(T_Context *ctx, MD_Node *root, MSF_Parsed *msf, String8 *error) +{ + String8 data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); + MD_Node *node = t_codec_push_node(ctx->arena, root, str8_lit("dbi")); + t_codec_push_u64(ctx, node, "stream_index", PDB_FixedStream_Dbi); + t_codec_push_u64(ctx, node, "size", data.size); + if (data.size < sizeof(PDB_DbiHeader)) { *error = str8_lit("DBI header is truncated"); return 0; } + PDB_DbiHeader *header = (PDB_DbiHeader *)data.str; + U64 ranges_size = header->module_info_size; + U32 sizes[] = {header->sec_con_size, header->sec_map_size, header->file_info_size, header->tsm_size, header->ec_info_size, header->dbg_header_size}; + for EachElement(i, sizes) { if (sizes[i] > data.size - Min(ranges_size, data.size)) { *error = str8_lit("DBI substream ranges are truncated"); return 0; } ranges_size += sizes[i]; } + if (header->sig != PDB_DbiHeaderSignature_V1 || ranges_size > data.size - sizeof(*header)) { *error = str8_lit("DBI header or substream ranges are invalid"); return 0; } + PDB_DbiParsed *dbi = pdb_dbi_from_data(ctx->arena, data); + t_codec_push_field(ctx, node, "machine", coff_string_from_machine_type(dbi->machine_type)); + t_codec_push_u64(ctx, node, "machine_value", dbi->machine_type); + t_codec_push_u64(ctx, node, "gsi_stream", dbi->gsi_sn); t_codec_push_u64(ctx, node, "psi_stream", dbi->psi_sn); t_codec_push_u64(ctx, node, "symbol_stream", dbi->sym_sn); + String8 module_info = pdb_data_from_dbi_range(dbi, PDB_DbiRange_ModuleInfo); + for (U64 cursor = 0; cursor < module_info.size;) { + if (!t_codec_pdb_range_is_valid(cursor, sizeof(PDB_DbiCompUnitHeader), module_info.size)) { *error = str8_lit("DBI module header is truncated"); return 0; } + PDB_DbiCompUnitHeader *mod = (PDB_DbiCompUnitHeader *)(module_info.str + cursor); + U64 name_off = cursor + sizeof(*mod); String8 first = {0}, second = {0}; + if (!t_codec_pdb_cstr_from_offset(module_info, name_off, &first) || !t_codec_pdb_cstr_from_offset(module_info, name_off + first.size + 1, &second)) { *error = str8_lit("DBI module name is truncated"); return 0; } + cursor = AlignPow2(name_off + first.size + 1 + second.size + 1, 4); + U64 ranges = (U64)mod->symbols_size + mod->c11_lines_size + mod->c13_lines_size; + if ((ranges != 0 && mod->sn >= msf->stream_count) || (mod->sn < msf->stream_count && ranges > msf_data_from_stream(msf, mod->sn).size)) { + *error = str8_lit("DBI module stream ranges are invalid"); return 0; + } + } + PDB_CompUnitArray *modules = pdb_comp_unit_array_from_data(ctx->arena, module_info); + t_codec_push_u64(ctx, node, "module_count", modules->count); + MD_Node *module_nodes = t_codec_push_node(ctx->arena, node, str8_lit("modules")); + for EachIndex(i, modules->count) { + PDB_CompUnit *mod = modules->units[i]; + MD_Node *mod_node = t_codec_push_node(ctx->arena, module_nodes, str8f(ctx->arena, "module_%llu", i)); + t_codec_push_field(ctx, mod_node, "object_name", mod->obj_name); + t_codec_push_field(ctx, mod_node, "object_file_name", str8_skip_last_slash(mod->obj_name)); + t_codec_push_field(ctx, mod_node, "group_name", mod->group_name); + t_codec_push_u64(ctx, mod_node, "stream_index", mod->sn); + String8 sym_data = pdb_data_from_unit_range(msf, mod, PDB_DbiCompUnitRange_Symbols); + String8 c11_data = pdb_data_from_unit_range(msf, mod, PDB_DbiCompUnitRange_C11); + String8 c13_data = pdb_data_from_unit_range(msf, mod, PDB_DbiCompUnitRange_C13); + t_codec_push_u64(ctx, mod_node, "symbol_size", sym_data.size); t_codec_push_u64(ctx, mod_node, "c11_size", c11_data.size); t_codec_push_u64(ctx, mod_node, "c13_size", c13_data.size); + MD_Node *module_symbols = t_codec_push_node(ctx->arena, mod_node, str8_lit("module_symbols")); + if (!t_codec_pdb_push_symbol_stream(ctx, module_symbols, sym_data, PDB_SYMBOL_ALIGN, error)) { return 0; } + } + MD_Node *global_symbols = t_codec_push_node(ctx->arena, root, str8_lit("global_symbols")); + t_codec_push_u64(ctx, global_symbols, "stream_index", dbi->sym_sn); + String8 symbols = {0}; + if (dbi->sym_sn < msf->stream_count) { + t_codec_push_field(ctx, global_symbols, "present", str8_lit("true")); + symbols = msf_data_from_stream(msf, dbi->sym_sn); + if (!t_codec_pdb_push_symbol_stream(ctx, global_symbols, symbols, PDB_SYMBOL_ALIGN, error)) { return 0; } + } else { + t_codec_push_field(ctx, global_symbols, "present", str8_lit("false")); + } + if (!t_codec_pdb_push_gsi(ctx, root, "gsi", msf, dbi->gsi_sn, symbols, 0, error)) { return 0; } + if (!t_codec_pdb_push_gsi(ctx, root, "psi", msf, dbi->psi_sn, symbols, 1, error)) { return 0; } + return 1; +} + +internal T_Result +t_codec_pdb_decode(T_Context *ctx, String8 data, MD_Node **semantic_tree_out) +{ + T_PdbMsfInfo msf_info = {0}; String8 error = {0}; + if (!t_codec_pdb_validate_msf(ctx->arena, data, &msf_info, &error)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "%S", error); } + MSF_Parsed *msf = msf_parsed_from_data(ctx->arena, data); + if (msf == 0 || msf->stream_count != msf_info.stream_count || msf->stream_count <= PDB_FixedStream_Ipi) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "PDB is missing fixed streams"); + } + MD_Node *root = t_codec_push_node(ctx->arena, 0, str8_lit("pdb")); + t_codec_push_u64(ctx, root, "file_size", data.size); t_codec_push_u64(ctx, root, "page_size", msf->page_size); t_codec_push_u64(ctx, root, "page_count", msf->page_count); + t_codec_push_u64(ctx, root, "stream_count", msf->stream_count); + MD_Node *streams = t_codec_push_node(ctx->arena, root, str8_lit("streams")); + for EachIndex(i, msf->stream_count) { + MD_Node *stream = t_codec_push_node(ctx->arena, streams, str8f(ctx->arena, "stream_%llu", i)); + t_codec_push_field(ctx, stream, "present", msf_info.stream_present[i] ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, stream, "size", msf->streams[i].size); + } + MD_Node *fixed = t_codec_push_node(ctx->arena, root, str8_lit("fixed_streams")); + struct { char *name; MSF_StreamNumber sn; } fixed_defs[] = {{"info", PDB_FixedStream_Info}, {"tpi", PDB_FixedStream_Tpi}, {"dbi", PDB_FixedStream_Dbi}, {"ipi", PDB_FixedStream_Ipi}}; + for EachElement(i, fixed_defs) { + MD_Node *stream = t_codec_push_node(ctx->arena, fixed, str8_cstring(fixed_defs[i].name)); + t_codec_push_u64(ctx, stream, "index", fixed_defs[i].sn); t_codec_push_field(ctx, stream, "present", msf_info.stream_present[fixed_defs[i].sn] ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, stream, "size", msf->streams[fixed_defs[i].sn].size); + if (!msf_info.stream_present[fixed_defs[i].sn]) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "PDB fixed stream %s is absent", fixed_defs[i].name); } + } + if (!t_codec_pdb_push_tpi(ctx, root, "tpi", msf, PDB_FixedStream_Tpi, &error) || !t_codec_pdb_push_tpi(ctx, root, "ipi", msf, PDB_FixedStream_Ipi, &error) || + !t_codec_pdb_push_dbi(ctx, root, msf, &error)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "%S", error); + } + *semantic_tree_out = root; + return ctx->result; +} + +internal T_Result +t_codec_run_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"path", "args", "expect_exit", "timeout_ms", "stdout_matches", "stderr_matches", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "path", 1); + for (U64 i = 1; allowed[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 0); } + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + if (path.size == 0 || str8_find_needle(path, 0, str8_lit("/"), 0) < path.size || str8_find_needle(path, 0, str8_lit("\\"), 0) < path.size || str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "path"), "run path must be a relative file name"); + } + String8 expected = t_codec_scalar(t_codec_child(arguments, "expect_exit")); + U64 value = 0; + if (expected.size != 0 && !str8_matchi(expected, str8_lit("nonzero")) && !str8_matchi(expected, str8_lit("any")) && !try_u64_from_str8_c_rules(expected, &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "expect_exit"), "invalid expected process exit code"); + } + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout) && !try_u64_from_str8_c_rules(t_codec_scalar(timeout), &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, timeout, "timeout_ms must be an integer"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_run_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 path = t_make_file_path(ctx->arena, t_codec_scalar(t_codec_child(arguments, "path"))); + U64 timeout_ms = max_U64; + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout)) { try_u64_from_str8_c_rules(t_codec_scalar(timeout), &timeout_ms); } + U64 timeout_us = timeout_ms == max_U64 || timeout_ms > max_U64 / 1000 ? max_U64 : timeout_ms * 1000; + if (!t_invoke(path, t_codec_scalar(t_codec_child(arguments, "args")), timeout_us)) { + return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("run"), "unable to launch '%S'", path); + } + if (g_last_exit_code == max_U64) { return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process did not exit before timeout"); } + String8 expected = t_codec_scalar(t_codec_child(arguments, "expect_exit")); + B32 any = str8_matchi(expected, str8_lit("any")); + B32 nonzero = str8_matchi(expected, str8_lit("nonzero")); + U64 expected_code = 0; + if (expected.size != 0 && !any && !nonzero) { try_u64_from_str8_c_rules(expected, &expected_code); } + if (!any && (nonzero ? g_last_exit_code == 0 : g_last_exit_code != expected_code)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process exited with %llu, expected %S\n%S", g_last_exit_code, + nonzero ? str8_lit("nonzero") : str8f(ctx->arena, "%llu", expected_code), g_errors); + } + StringMatchFlags flags = StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive; + String8 stdout_pattern = t_codec_scalar(t_codec_child(arguments, "stdout_matches")); + String8 stderr_pattern = t_codec_scalar(t_codec_child(arguments, "stderr_matches")); + if (stdout_pattern.size != 0 && !str8_match_wildcard(g_output, stdout_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process stdout does not match '%S'\n%S", stdout_pattern, g_output); + } + if (stderr_pattern.size != 0 && !str8_match_wildcard(g_errors, stderr_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process stderr does not match '%S'\n%S", stderr_pattern, g_errors); + } + return ctx->result; +} + +internal B32 +t_codec_is_safe_file_name(String8 name) +{ + return name.size != 0 && str8_find_needle(name, 0, str8_lit("/"), 0) >= name.size && str8_find_needle(name, 0, str8_lit("\\"), 0) >= name.size && + str8_find_needle(name, 0, str8_lit(":"), 0) >= name.size; +} + +internal T_Result +t_codec_clang_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"input", "output", "args", "expect_exit", "timeout_ms", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + for (U64 i = 0; i < 3; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 1); } + t_codec_validate_scalar(ctx, arguments, "expect_exit", 0); + t_codec_validate_scalar(ctx, arguments, "timeout_ms", 0); + for (U64 i = 0; i < 2; i += 1) { + if (!t_codec_is_safe_file_name(t_codec_scalar(t_codec_child(arguments, allowed[i])))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, allowed[i]), "clang %s must be a relative file name", allowed[i]); + } + } + U64 value = 0; + MD_Node *expect_exit = t_codec_child(arguments, "expect_exit"); + if (!md_node_is_nil(expect_exit) && !try_u64_from_str8_c_rules(t_codec_scalar(expect_exit), &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expect_exit, "clang expect_exit must be an integer"); + } + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout) && !try_u64_from_str8_c_rules(t_codec_scalar(timeout), &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, timeout, "clang timeout_ms must be an integer"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_clang_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 input = t_make_file_path(ctx->arena, t_codec_scalar(t_codec_child(arguments, "input"))); + String8 output = t_make_file_path(ctx->arena, t_codec_scalar(t_codec_child(arguments, "output"))); + String8 args = str8f(ctx->arena, "%S -o %S %S", input, output, t_codec_scalar(t_codec_child(arguments, "args"))); + U64 timeout_ms = max_U64; + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout)) { try_u64_from_str8_c_rules(t_codec_scalar(timeout), &timeout_ms); } + U64 timeout_us = timeout_ms == max_U64 || timeout_ms > max_U64 / 1000 ? max_U64 : timeout_ms * 1000; + if (!t_invoke(t_clang_path(), args, timeout_us)) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("clang"), "unable to launch Clang"); } + U64 expected = 0; + MD_Node *expect_exit = t_codec_child(arguments, "expect_exit"); + if (!md_node_is_nil(expect_exit)) { try_u64_from_str8_c_rules(t_codec_scalar(expect_exit), &expected); } + if (g_last_exit_code != expected) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("clang"), "Clang exited with %llu, expected %llu\n%S", g_last_exit_code, expected, g_errors); + } + String8 data = t_read_file(ctx->arena, t_codec_scalar(t_codec_child(arguments, "output"))); + if (data.size == 0) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("clang"), "Clang did not produce '%S'", output); } + return ctx->result; +} + +internal T_Result +t_codec_expect_pe_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "expected", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "artifact", 1); + MD_Node *artifact_node = t_codec_child(arguments, "artifact"); + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(artifact_node)); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("pe"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, artifact_node, "expect_pe requires a PE artifact"); + } + MD_Node *expected = t_codec_child(arguments, "expected"); + if (md_node_is_nil(expected) || md_node_is_nil(expected->first) || !md_node_is_nil(expected->first->next) || !str8_match(expected->first->string, str8_lit("pe"), 0)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expected, "expect_pe requires one PE expectation tree"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_pe_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + MD_Node *actual = 0; + T_Result result = artifact->codec->decode(ctx, artifact, &actual); + if (!t_result_is_ok(result)) { return result; } + MD_Node *expected = t_codec_child(arguments, "expected")->first; + return t_semantic_match(ctx, expected, actual); +} + +internal T_Result +t_codec_expect_pdb_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"path", "expected", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "path", 1); + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + if (path.size == 0 || str8_match(path, str8_lit("."), 0) || str8_match(path, str8_lit(".."), 0) || + str8_find_needle(path, 0, str8_lit("/"), 0) < path.size || str8_find_needle(path, 0, str8_lit("\\"), 0) < path.size || + str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "path"), "expect_pdb path must be a safe relative file name"); + } + MD_Node *expected = t_codec_child(arguments, "expected"); + if (md_node_is_nil(expected) || md_node_is_nil(expected->first) || !md_node_is_nil(expected->first->next) || !str8_match(expected->first->string, str8_lit("pdb"), 0)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expected, "expect_pdb requires one PDB expectation tree"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_pdb_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + String8 data = t_read_file(ctx->arena, path); + if (data.size == 0 && !file_path_exists(t_make_file_path(ctx->arena, path))) { + return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("expect_pdb"), "file '%S' does not exist", path); + } + MD_Node *actual = 0; + T_Result result = t_codec_pdb_decode(ctx, data, &actual); + if (!t_result_is_ok(result)) { return result; } + return t_semantic_match(ctx, t_codec_child(arguments, "expected")->first, actual); +} + +internal T_Result +t_codec_expect_coff_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "expected", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "artifact", 1); + MD_Node *artifact_node = t_codec_child(arguments, "artifact"); + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(artifact_node)); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("coff"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, artifact_node, "expect_coff requires a COFF artifact"); + } + MD_Node *expected = t_codec_child(arguments, "expected"); + if (md_node_is_nil(expected) || md_node_is_nil(expected->first) || !md_node_is_nil(expected->first->next) || !str8_match(expected->first->string, str8_lit("coff"), 0)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expected, "expect_coff requires one COFF expectation tree"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_coff_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + MD_Node *actual = 0; + T_Result result = artifact->codec->decode(ctx, artifact, &actual); + if (!t_result_is_ok(result)) { return result; } + MD_Node *expected = t_codec_child(arguments, "expected")->first; + return t_semantic_match(ctx, expected, actual); +} + +internal T_Result +t_codec_expect_file_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"path", "equals_artifact", "contains", "nonempty", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "path", 1); + t_codec_validate_scalar(ctx, arguments, "equals_artifact", 0); + t_codec_validate_scalar(ctx, arguments, "contains", 0); + t_codec_validate_scalar(ctx, arguments, "nonempty", 0); + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + if (str8_find_needle(path, 0, str8_lit("/"), 0) < path.size || str8_find_needle(path, 0, str8_lit("\\"), 0) < path.size || str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "path"), "expect_file path must be a relative file name"); + } + String8 artifact_name = t_codec_scalar(t_codec_child(arguments, "equals_artifact")); + if (artifact_name.size != 0 && t_artifact_from_name(ctx->run, artifact_name) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "equals_artifact"), "unknown artifact '%S'", artifact_name); + } + B32 nonempty = 0; + MD_Node *nonempty_node = t_codec_child(arguments, "nonempty"); + if (!md_node_is_nil(nonempty_node) && !t_bool_from_scalar(nonempty_node, &nonempty)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, nonempty_node, "nonempty must be true or false"); + } + if (artifact_name.size == 0 && md_node_is_nil(t_codec_child(arguments, "contains")) && md_node_is_nil(nonempty_node)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "expect_file requires equals_artifact, contains, or nonempty"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_file_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + String8 data = t_read_file(ctx->arena, path); + if (data.size == 0 && !file_path_exists(t_make_file_path(ctx->arena, path))) { + return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("expect_file"), "file '%S' does not exist", path); + } + String8 artifact_name = t_codec_scalar(t_codec_child(arguments, "equals_artifact")); + if (artifact_name.size != 0) { + T_Artifact *artifact = t_artifact_from_name(ctx, artifact_name); + if (!str8_match(data, artifact->data, 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_file"), "file '%S' differs from artifact '%S'", path, artifact_name); + } + } + String8 needle = t_codec_scalar(t_codec_child(arguments, "contains")); + if (needle.size != 0 && str8_find_needle(data, 0, needle, 0) >= data.size) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_file"), "file '%S' does not contain '%S'", path, needle); + } + B32 nonempty = 0; + MD_Node *nonempty_node = t_codec_child(arguments, "nonempty"); + if (!md_node_is_nil(nonempty_node)) { + t_bool_from_scalar(nonempty_node, &nonempty); + if (nonempty && data.size == 0) { return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_file"), "file '%S' is empty", path); } + } + return ctx->result; +} + +internal COFF_SectionHeader * +t_codec_pe_section_from_name(T_Context *ctx, T_Artifact *artifact, PE_BinInfo *pe_out, String8 name) +{ + PE_BinInfo pe = pe_bin_info_from_data(ctx->arena, artifact->data); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(artifact->data, pe.section_table_range).str; + String8 string_table = str8_substr(artifact->data, pe.string_table_range); + if (pe_out != 0) { *pe_out = pe; } + return coff_section_header_from_name(string_table, section_table, pe.section_count, name); +} + +internal T_Result +t_codec_expect_pe_word_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "section", "offset", "type", "nonzero", "equals", "target_section", + "target_offset", "target_address", "other_section", "other_offset", "other_type", "relation", "modulo", + "remainder", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + char *required[] = {"artifact", "section", "offset", "type", 0}; + for (U64 i = 0; required[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, required[i], 1); } + for (U64 i = 4; allowed[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 0); } + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(t_codec_child(arguments, "artifact"))); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("pe"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "artifact"), "expect_pe_word requires a PE artifact"); + } + String8 type = t_codec_scalar(t_codec_child(arguments, "type")); + if (!str8_match(type, str8_lit("u32"), StringMatchFlag_CaseInsensitive) && !str8_match(type, str8_lit("u64"), StringMatchFlag_CaseInsensitive) && + !str8_match(type, str8_lit("rel32"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "type"), "word type must be u32, u64, or rel32"); + } + String8 relation = t_codec_scalar(t_codec_child(arguments, "relation")); + if (relation.size != 0 && !str8_match(relation, str8_lit("equal"), StringMatchFlag_CaseInsensitive) && + !str8_match(relation, str8_lit("not_equal"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "relation"), "relation must be equal or not_equal"); + } + return ctx->run->result; +} + +internal B32 +t_codec_pe_read_word(T_Context *ctx, T_Artifact *artifact, String8 section_name, U64 offset, String8 type, U64 *value_out) +{ + PE_BinInfo pe = {0}; + COFF_SectionHeader *section = t_codec_pe_section_from_name(ctx, artifact, &pe, section_name); + if (section == 0) { return 0; } + U64 size = str8_match(type, str8_lit("u64"), StringMatchFlag_CaseInsensitive) ? 8 : 4; + if (offset > section->fsize || size > section->fsize - offset || section->foff > artifact->data.size || section->fsize > artifact->data.size - section->foff) { return 0; } + U8 *ptr = artifact->data.str + section->foff + offset; + if (str8_match(type, str8_lit("u64"), StringMatchFlag_CaseInsensitive)) { + MemoryCopy(value_out, ptr, sizeof(U64)); + } else { + U32 raw = 0; + MemoryCopy(&raw, ptr, sizeof(raw)); + if (str8_match(type, str8_lit("rel32"), StringMatchFlag_CaseInsensitive)) { + *value_out = pe.image_base + section->voff + offset + sizeof(S32) + (S32)raw; + } else { + *value_out = raw; + } + } + return 1; +} + +internal T_Result +t_codec_expect_pe_word_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + String8 section_name = t_codec_scalar(t_codec_child(arguments, "section")); + String8 type = t_codec_scalar(t_codec_child(arguments, "type")); + U64 offset = 0, value = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "offset")), &offset); + if (!t_codec_pe_read_word(ctx, artifact, section_name, offset, type, &value)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "unable to read %S at %S+%llu", type, section_name, offset); + } + MD_Node *nonzero_node = t_codec_child(arguments, "nonzero"); + B32 nonzero = 0; + if (!md_node_is_nil(nonzero_node) && t_bool_from_scalar(nonzero_node, &nonzero) && nonzero && value == 0) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu is zero", section_name, offset); + } + MD_Node *equals_node = t_codec_child(arguments, "equals"); + U64 expected = 0; + if (!md_node_is_nil(equals_node) && (!try_u64_from_str8_c_rules(t_codec_scalar(equals_node), &expected) || value != expected)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu is %llu, expected %llu", section_name, offset, value, expected); + } + String8 target_section_name = t_codec_scalar(t_codec_child(arguments, "target_section")); + if (target_section_name.size != 0) { + PE_BinInfo pe = {0}; + COFF_SectionHeader *target = t_codec_pe_section_from_name(ctx, artifact, &pe, target_section_name); + U64 target_offset = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "target_offset")), &target_offset); + String8 address = t_codec_scalar(t_codec_child(arguments, "target_address")); + if (target != 0) { expected = (str8_match(address, str8_lit("rva"), StringMatchFlag_CaseInsensitive) ? 0 : pe.image_base) + target->voff + target_offset; } + if (target == 0 || value != expected) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu does not target %S+%llu", section_name, offset, target_section_name, + target_offset); + } + } + String8 other_section = t_codec_scalar(t_codec_child(arguments, "other_section")); + if (other_section.size != 0) { + U64 other_offset = 0, other_value = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "other_offset")), &other_offset); + String8 other_type = t_codec_scalar(t_codec_child(arguments, "other_type")); + if (other_type.size == 0) { other_type = type; } + if (!t_codec_pe_read_word(ctx, artifact, other_section, other_offset, other_type, &other_value)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "unable to read comparison word at %S+%llu", other_section, other_offset); + } + B32 equal = str8_match(t_codec_scalar(t_codec_child(arguments, "relation")), str8_lit("equal"), StringMatchFlag_CaseInsensitive); + if ((equal && value != other_value) || (!equal && value == other_value)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "word relation at %S+%llu and %S+%llu does not hold", section_name, offset, other_section, + other_offset); + } + } + MD_Node *modulo_node = t_codec_child(arguments, "modulo"); + if (!md_node_is_nil(modulo_node)) { + U64 modulo = 0, remainder = 0; + try_u64_from_str8_c_rules(t_codec_scalar(modulo_node), &modulo); + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "remainder")), &remainder); + if (modulo == 0 || value % modulo != remainder) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu has unexpected remainder", section_name, offset); + } + } + return ctx->result; +} + +internal T_Result +t_codec_expect_pe_bytes_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "section", "offset", "hex", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + for (U64 i = 0; allowed[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 1); } + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(t_codec_child(arguments, "artifact"))); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("pe"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "artifact"), "expect_pe_bytes requires a PE artifact"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_pe_bytes_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + String8 section_name = t_codec_scalar(t_codec_child(arguments, "section")); + COFF_SectionHeader *section = t_codec_pe_section_from_name(ctx, artifact, 0, section_name); + U64 offset = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "offset")), &offset); + MD_Node hex_node = {0}; + MD_Node value_node = {0}; + hex_node.kind = MD_NodeKind_Main; + hex_node.string = str8_lit("hex"); + hex_node.first = hex_node.last = &value_node; + value_node.kind = MD_NodeKind_Main; + value_node.string = t_codec_scalar(t_codec_child(arguments, "hex")); + String8 expected = {0}; + T_Result result = t_bytes_from_producer(ctx, &hex_node, &expected); + if (!t_result_is_ok(result)) { return result; } + if (section == 0 || offset > section->fsize || expected.size > section->fsize - offset || section->foff > artifact->data.size || section->fsize > artifact->data.size - section->foff || + !str8_match(str8(artifact->data.str + section->foff + offset, expected.size), expected, 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_bytes"), "bytes at %S+%llu do not match", section_name, offset); + } + return ctx->result; +} + +internal T_Result +t_codec_no_arguments_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + if (!md_node_is_nil(arguments->first)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments->first, "operation takes no arguments"); + } + return ctx->run->result; +} + +#define T_CODEC_INTERNAL_API_CHECK(expr) do { if (!(expr)) { t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, operation, "assertion failed: %s", #expr); } } while (0) + +internal T_Result +t_codec_get_msf_stream_pages_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("get_msf_stream_pages"); + MSF_Context *msf = msf_alloc(MSF_DEFAULT_PAGE_SIZE, MSF_DEFAULT_FPM); + + { + U64 stream_size = MB(150) + 1; + + MSF_StreamNumber sn = msf_stream_alloc_ex(msf, stream_size); + + U8 *test = push_array(arena, U8, stream_size); + MemorySet(test, 0xca, stream_size/2); + MemorySet(test + stream_size/2, 0xbe, stream_size/2); + + String8List stream_data = msf_data_from_sn(arena, msf, sn); + T_CODEC_INTERNAL_API_CHECK(stream_data.total_size == stream_size); + T_CODEC_INTERNAL_API_CHECK(stream_data.node_count == 12); + + String8Array a = str8_array_from_list(arena, &stream_data); + T_CODEC_INTERNAL_API_CHECK(a.v[0].size == 0xffd000); + T_CODEC_INTERNAL_API_CHECK(a.v[1].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[2].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[3].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[4].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[5].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[6].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[7].size == 0xffd000); + T_CODEC_INTERNAL_API_CHECK(a.v[8].size == 0x1000); + T_CODEC_INTERNAL_API_CHECK(a.v[9].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[10].size == 0x613000); + T_CODEC_INTERNAL_API_CHECK(a.v[11].size == 1); + + String8Node buf = *stream_data.first; + U64 buf_pos = 0; + str8_buffer_write(&buf, &buf_pos, str8(test, stream_size)); + + String8 cmp = msf_stream_read_block(arena, msf, sn, stream_size); + T_CODEC_INTERNAL_API_CHECK(cmp.size == stream_size); + T_CODEC_INTERNAL_API_CHECK(MemoryCompare(cmp.str, test, stream_size) == 0); + } + + { + MSF_StreamNumber sn = msf_stream_alloc_ex(msf, 1); + String8List stream_data = msf_data_from_sn(arena, msf, sn); + T_CODEC_INTERNAL_API_CHECK(stream_data.node_count == 1); + T_CODEC_INTERNAL_API_CHECK(stream_data.total_size == 1); + T_CODEC_INTERNAL_API_CHECK(stream_data.first->string.size == 1); + } + + msf_release(msf); + return ctx->result; +} + +internal T_Result +t_codec_data_from_pdb(T_Context *ctx, MD_Node *arguments, PDB_Context *pdb, String8 *data_out) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("validate_info_stream"); + TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("foo")); + TP_Arena *tp_arena = tp_arena_alloc(tp); + pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0); + + MSF_Error msf_error = msf_build(pdb->msf); + T_CODEC_INTERNAL_API_CHECK(msf_error == MSF_Error_OK); + String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf); + T_CODEC_INTERNAL_API_CHECK(t_write_file_list(str8_lit("test.pdb"), raw_msf_list)); + + *data_out = str8_list_join(arena, &raw_msf_list, 0); + + tp_arena_release(&tp_arena); + tp_release(tp); + + return ctx->result; +} + +internal T_Result +t_codec_validate_info_stream_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("validate_info_stream"); + COFF_TimeStamp time_stamp = 123; + U32 age = 1; + Guid guid = { .data1 = max_U32, .data2 = max_U16 - 1, .data3 = max_U16 - 2, .data4 = { 1, 2, 3, 4, 5, 6, 7, 8 } }; + PDB_Context *pdb = pdb_alloc(MSF_DEFAULT_PAGE_SIZE, COFF_MachineType_X64, time_stamp, age, guid); + + char *stream_names[] = { "one", "two", "three", "four", "five" }; + MSF_StreamNumber stream_numbers[ArrayCount(stream_names)] = {0}; + + for EachElement(i, stream_names) { + stream_numbers[i] = pdb_push_named_stream(&pdb->info->named_stream_ht, pdb->msf, str8_cstring(stream_names[i])); + T_CODEC_INTERNAL_API_CHECK(stream_numbers[i] != MSF_INVALID_STREAM_NUMBER); + } + + String8 raw_msf = {0}; + t_codec_data_from_pdb(ctx, arguments, pdb, &raw_msf); + MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_msf); + String8 info_data = msf_data_from_stream(msf_parsed, PDB_FixedStream_Info); + +#if 0 + fprintf(stderr, "\n"); + for EachIndex(i, info_data.size) { + fprintf(stderr, "0x%02x, ", info_data.str[i]); + if (i % 19 == 18 && i > 0) { fprintf(stderr, "\n"); } + } +#endif + U8 expected_info_data[] = { + 0x94, 0x2e, 0x31, 0x01, 0x7b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xfd, + 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x22, 0x00, 0x00, 0x00, 0x6f, 0x6e, 0x65, 0x00, 0x74, 0x77, + 0x6f, 0x00, 0x74, 0x68, 0x72, 0x65, 0x65, 0x00, 0x66, 0x6f, 0x75, 0x72, 0x00, 0x66, 0x69, 0x76, 0x65, 0x00, 0x2f, + 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x13, + 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x51, 0x33, 0x01, + }; + T_CODEC_INTERNAL_API_CHECK(str8_match(info_data, str8_array_fixed(expected_info_data), 0)); + + pdb_release(pdb); + return ctx->result; +} + +internal T_Result +t_codec_psi_addr_map_radix_sort_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("psi_addr_map_radix_sort"); + String8 names[] = { + str8_lit("alpha"), + str8_lit("bravo"), + str8_lit("charlie"), + str8_lit("delta"), + }; + U64 address_count = (1 << 15) + 1; + U64 record_count = address_count * ArrayCount(names); + + PDB_GsiSortRecord *records = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); + PDB_GsiSortRecord *expected = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); + for EachIndex(i, record_count) { + U64 address_idx = address_count - 1 - i / ArrayCount(names); + records[i].isect_off.isect = 1 + address_idx % 257; + records[i].isect_off.off = address_idx / 257; + records[i].name = names[ArrayCount(names) - 1 - i % ArrayCount(names)]; + records[i].offset = i * sizeof(U32); + } + MemoryCopyTyped(expected, records, record_count); + radsort(expected, record_count, psi_addr_map_compar_is_before); + + TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("psi addr map sort test")); + U32 *addr_map = psi_addr_map_from_gsi_records(tp, arena, records, record_count); + + for EachIndex(i, record_count) { + T_CODEC_INTERNAL_API_CHECK(addr_map[i] == expected[i].offset); + } + tp_release(tp); + return ctx->result; +} + +internal T_Result +t_codec_u64_array_radix_sort_parallel_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("u64_array_radix_sort_parallel"); + TP_Context *tp = tp_alloc(arena, 1, 1, str8_zero()); + + U64 small_values[] = { max_U64, 0, 7, 1, 7, 0x100000000ull, 2 }; + u64_array_sort_radix_parallel(tp, ArrayCount(small_values), small_values); + for (U64 i = 1; i < ArrayCount(small_values); i += 1) { + T_CODEC_INTERNAL_API_CHECK(small_values[i-1] <= small_values[i]); + } + + U64 count = 200003; + U64 *values = push_array_no_zero(arena, U64, count); + U64 state = 0x9e3779b97f4a7c15ull; + U64 sum_before = 0; + U64 xor_before = 0; + for EachIndex(i, count) { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + values[i] = state * 0x2545f4914f6cdd1dull; + if ((i % 97) == 0) { values[i] = 0; } + if ((i % 193) == 0) { values[i] = max_U64; } + if ((i % 389) == 0) { values[i] = 0x100000001ull; } + sum_before += values[i]; + xor_before ^= values[i]; + } + + u64_array_sort_radix_parallel(tp, count, values); + U64 sum_after = 0; + U64 xor_after = 0; + B32 is_sorted = 1; + for EachIndex(i, count) { + if (i > 0 && values[i-1] > values[i]) { is_sorted = 0; } + sum_after += values[i]; + xor_after ^= values[i]; + } + T_CODEC_INTERNAL_API_CHECK(is_sorted); + T_CODEC_INTERNAL_API_CHECK(sum_before == sum_after); + T_CODEC_INTERNAL_API_CHECK(xor_before == xor_after); + tp_release(tp); + return ctx->result; +} + +internal T_Result +t_codec_patch_cv_symbol_tree_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("patch_cv_symbol_tree"); + String8List raw_symbols = {0}; + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("foo.obj"), 123))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_GPROC32, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("Proc")))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE, cv_make_inline_site(arena, (CV_SymInlineSite){0}, str8_zero()))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE_END, cv_make_inline_site_end(arena))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_END, cv_make_end(arena))); + + U64 tree_size = cv_patch_symbol_tree_offsets(raw_symbols, sizeof(CV_Signature), 4); + T_CODEC_INTERNAL_API_CHECK(tree_size == 84); + + { + String8Node buf = *raw_symbols.first; + U64 buf_pos = 0; + + CV_SymbolHeader obj_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(obj_header), &obj_header) == sizeof(obj_header)); + T_CODEC_INTERNAL_API_CHECK(obj_header.kind == CV_SymKind_OBJNAME); + T_CODEC_INTERNAL_API_CHECK(str8_buffer_skip(&buf, &buf_pos, obj_header.size - sizeof(CV_SymKind))); + + CV_SymbolHeader proc_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(proc_header), &proc_header) == sizeof(proc_header)); + T_CODEC_INTERNAL_API_CHECK(proc_header.kind == CV_SymKind_GPROC32); + + CV_SymProc32 proc; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(proc), &proc) == sizeof(proc)); + T_CODEC_INTERNAL_API_CHECK(proc.end == 0x54); + T_CODEC_INTERNAL_API_CHECK(str8_buffer_skip(&buf, &buf_pos, proc_header.size - sizeof(CV_SymKind) - sizeof(proc))); + + CV_SymbolHeader inline_site_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site_header), &inline_site_header) == sizeof(inline_site_header)); + T_CODEC_INTERNAL_API_CHECK(inline_site_header.kind == CV_SymKind_INLINESITE); + + CV_SymInlineSite inline_site; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site), &inline_site)); + T_CODEC_INTERNAL_API_CHECK(inline_site.parent == 0x14); + T_CODEC_INTERNAL_API_CHECK(inline_site.end == 0x50); + T_CODEC_INTERNAL_API_CHECK(str8_buffer_skip(&buf, &buf_pos, inline_site_header.size - sizeof(CV_SymKind) - sizeof(inline_site))); + + CV_SymbolHeader inline_end_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(inline_end_header), &inline_end_header) == sizeof(inline_end_header)); + T_CODEC_INTERNAL_API_CHECK(inline_end_header.kind == CV_SymKind_INLINESITE_END); + + CV_SymbolHeader proc_end_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(proc_end_header), &proc_end_header) == sizeof(proc_end_header)); + T_CODEC_INTERNAL_API_CHECK(proc_end_header.kind == CV_SymKind_END); + + T_CODEC_INTERNAL_API_CHECK(buf.string.size == 0); + T_CODEC_INTERNAL_API_CHECK(buf.string.str == 0); + T_CODEC_INTERNAL_API_CHECK(buf_pos == 0); + } + return ctx->result; +} + +#undef T_CODEC_INTERNAL_API_CHECK + +global T_Codec t_codec_script_codecs[] = { + {str8_lit_comp("bytes"), 0, t_codec_bytes_encode, t_codec_bytes_decode}, + {str8_lit_comp("text"), 0, t_codec_text_encode, t_codec_text_decode}, + {str8_lit_comp("coff"), t_coff_validate, t_coff_encode, t_coff_decode}, + {str8_lit_comp("pe"), 0, 0, t_codec_pe_decode}, +}; + +global T_OpSpec t_codec_script_ops[] = { + {str8_lit_comp("compare"), t_op_compare_validate, t_op_compare_execute}, + {str8_lit_comp("compare_file"), t_op_compare_file_validate, t_op_compare_file_execute}, + {str8_lit_comp("repeat"), t_op_repeat_validate, t_op_repeat_execute}, + {str8_lit_comp("run"), t_codec_run_validate, t_codec_run_execute}, + {str8_lit_comp("clang"), t_codec_clang_validate, t_codec_clang_execute}, + {str8_lit_comp("expect_coff"), t_codec_expect_coff_validate, t_codec_expect_coff_execute}, + {str8_lit_comp("expect_pe"), t_codec_expect_pe_validate, t_codec_expect_pe_execute}, + {str8_lit_comp("expect_pdb"), t_codec_expect_pdb_validate, t_codec_expect_pdb_execute}, + {str8_lit_comp("expect_file"), t_codec_expect_file_validate, t_codec_expect_file_execute}, + {str8_lit_comp("expect_pe_word"), t_codec_expect_pe_word_validate, t_codec_expect_pe_word_execute}, + {str8_lit_comp("expect_pe_bytes"), t_codec_expect_pe_bytes_validate, t_codec_expect_pe_bytes_execute}, + {str8_lit_comp("get_msf_stream_pages"), t_codec_no_arguments_validate, t_codec_get_msf_stream_pages_execute}, + {str8_lit_comp("validate_info_stream"), t_codec_no_arguments_validate, t_codec_validate_info_stream_execute}, + {str8_lit_comp("psi_addr_map_radix_sort"), t_codec_no_arguments_validate, t_codec_psi_addr_map_radix_sort_execute}, + {str8_lit_comp("u64_array_radix_sort_parallel"), t_codec_no_arguments_validate, t_codec_u64_array_radix_sort_parallel_execute}, + {str8_lit_comp("patch_cv_symbol_tree"), t_codec_no_arguments_validate, t_codec_patch_cv_symbol_tree_execute}, +}; + +global T_SuiteSpec t_codec_script_suite = { + .name = str8_lit_comp("linker"), + .codecs = t_codec_script_codecs, + .codec_count = ArrayCount(t_codec_script_codecs), + .ops = t_codec_script_ops, + .op_count = ArrayCount(t_codec_script_ops), +}; diff --git a/src/torture/torture_linker.h b/src/torture/torture_linker.h new file mode 100644 index 000000000..895e182d7 --- /dev/null +++ b/src/torture/torture_linker.h @@ -0,0 +1,26 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +internal T_Result t_codec_pe_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +internal B32 t_codec_coff_section_name(String8 string_table, COFF_SectionHeader *header, String8 *name_out); +internal T_Result t_codec_run_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_run_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_clang_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_clang_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_pdb_decode(T_Context *ctx, String8 data, MD_Node **semantic_tree_out); +internal T_Result t_codec_expect_pdb_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pdb_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_coff_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_coff_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_file_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_file_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_word_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_word_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_bytes_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_bytes_execute(T_Context *ctx, MD_Node *arguments); + +global T_SuiteSpec t_codec_script_suite; diff --git a/src/torture/torture_main.c b/src/torture/torture_main.c index 1b3a6915b..5bebac057 100644 --- a/src/torture/torture_main.c +++ b/src/torture/torture_main.c @@ -102,6 +102,11 @@ #include "linker/lnk_log.h" #include "linker/lnk_debug_helper.h" #include "torture.h" +#include "torture_controller.h" +#include "torture_script.h" +#include "torture_build.h" +#include "torture_coff.h" +#include "torture_linker.h" #include "base/base_inc.c" #include "x64/x64.c" @@ -182,6 +187,12 @@ #include "linker/pdb_ext/pdb_builder.c" #include "linker/lnk_debug_helper.c" #include "torture.c" +#include "torture_base.c" +#include "torture_controller.c" +#include "torture_script.c" +#include "torture_build.c" +#include "torture_linker.c" +#include "torture_coff.c" #include "base/tests/base_tests.c" #include "mdesk/tests/mdesk_tests.c" @@ -191,6 +202,10 @@ #include "rdi_from_pdb/tests/rdi_from_pdb_tests.c" #include "raddbg/tests/raddbg_tests.c" #include "eval2/tests/eval2_tests.c" +#include "torture/tests/torture_subprocess_tests.c" +#include "torture/tests/torture_script_tests.c" +#include "torture/tests/torture_coff_tests.c" +#include "torture/tests/torture_linker_tests.c" internal B32 frame(void) { return 0; } @@ -199,6 +214,29 @@ internal B32 frame(void) { return 0; } internal void entry_point(CmdLine *cmdline) { - t_entry_point(cmdline); + if(cmd_line_has_flag(cmdline, str8_lit("controller_fixture"))) + { + t_controller_fixture(); + } + else if(cmd_line_has_flag(cmdline, str8_lit("controller_interrupt_fixture"))) + { + t_controller_interrupt_fixture(); + } + else if(cmd_line_has_flag(cmdline, str8_lit("run_fixture_ok"))) + { + t_run_operation_fixture(0, 0); + } + else if(cmd_line_has_flag(cmdline, str8_lit("run_fixture_nonzero"))) + { + t_run_operation_fixture(0, 7); + } + else if(cmd_line_has_flag(cmdline, str8_lit("run_fixture_sleep"))) + { + t_run_operation_fixture(1, 0); + } + else + { + t_entry_point(cmdline); + } } diff --git a/src/torture/torture_script.c b/src/torture/torture_script.c new file mode 100644 index 000000000..563fbf10c --- /dev/null +++ b/src/torture/torture_script.c @@ -0,0 +1,747 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +global T_Codec t_codec_bytes = {str8_lit_comp("bytes"), 0, t_codec_bytes_encode, t_codec_bytes_decode}; +global T_Codec t_codec_text = {str8_lit_comp("text"), 0, t_codec_text_encode, t_codec_text_decode}; +global T_OpSpec t_op_compare = {str8_lit_comp("compare"), t_op_compare_validate, t_op_compare_execute}; + +inline internal B32 +t_result_is_ok(T_Result result) +{ + return result.code == T_ResultCode_Ok; +} + +internal TxtPt +t_txt_pt_from_offset(String8 source, U64 offset) +{ + TxtPt result = {1, 1}; + + U64 offset_cap = Min(offset, source.size); + + for (U64 i = 0; i < offset_cap; i += 1) { + if (source.str[i] == '\n') { + result.line += 1; + result.column = 1; + } else { + result.column += 1; + } + } + + return result; +} + +internal void +t_diagnostic_from_md_msg(T_Context *ctx, MD_Msg *msg) +{ + T_Diagnostic *diagnostic = push_array(ctx->arena, T_Diagnostic, 1); + diagnostic->kind = msg->kind; + diagnostic->file_path = ctx->file_path; + diagnostic->location = t_txt_pt_from_offset(ctx->source, msg->node->src_offset); + diagnostic->message = str8_copy(ctx->arena, msg->string); + + SLLQueuePush(ctx->result.diagnostics.first, ctx->result.diagnostics.last, diagnostic); + ctx->result.diagnostics.count += 1; + + if (ctx->result.code == T_ResultCode_Ok && msg->kind >= MD_MsgKind_Error) { + ctx->result.code = T_ResultCode_ParseError; + } +} + +internal T_Result +t_context_errorfv(T_Context *ctx, T_ResultCode code, MD_Node *node, String8 operation, char *fmt, va_list args) +{ + T_Diagnostic *diagnostic = push_array(ctx->arena, T_Diagnostic, 1); + diagnostic->kind = MD_MsgKind_Error; + diagnostic->file_path = ctx->file_path; + diagnostic->location = t_txt_pt_from_offset(ctx->source, md_node_is_nil(node) ? 0 : node->src_offset); + diagnostic->operation = operation; + diagnostic->message = push_str8fv(ctx->arena, fmt, args); + + SLLQueuePush(ctx->result.diagnostics.first, ctx->result.diagnostics.last, diagnostic); + ctx->result.diagnostics.count += 1; + + if (ctx->result.code == T_ResultCode_Ok) { ctx->result.code = code; } + return ctx->result; +} + +inline internal T_Result +t_context_errorf(T_Context *ctx, T_ResultCode code, MD_Node *node, String8 operation, char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + T_Result result = t_context_errorfv(ctx, code, node, operation, fmt, args); + va_end(args); + return result; +} + +inline internal T_Result +t_parse_errorf(T_ParseContext *ctx, T_ResultCode code, MD_Node *node, char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + T_Result result = t_context_errorfv(ctx->run, code, node, ctx->operation, fmt, args); + va_end(args); + return result; +} + +inline internal T_Codec * +t_codec_from_kind(T_SuiteSpec *suite, String8 kind) +{ + for EachIndex(i, suite->codec_count) { + if (str8_match(suite->codecs[i].kind, kind, StringMatchFlag_CaseInsensitive)) { return &suite->codecs[i]; } + } + return 0; +} + +inline internal T_OpSpec * +t_op_spec_from_name(T_SuiteSpec *suite, String8 name) +{ + for EachIndex(i, suite->op_count) { + if (str8_match(suite->ops[i].name, name, StringMatchFlag_CaseInsensitive)) { return &suite->ops[i]; } + } + return 0; +} + +inline internal T_Artifact * +t_artifact_from_name(T_Context *ctx, String8 name) +{ + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (str8_match(artifact->name, name, 0)) { return artifact; } + } + return 0; +} + +inline internal MD_Node * +t_child_from_string(MD_Node *node, char *string) +{ + return md_child_from_string(node, str8_cstring(string), StringMatchFlag_CaseInsensitive); +} + +internal MD_Node * +t_scalar_from_node(MD_Node *node) +{ + return md_node_is_nil(node) ? &md_nil_node : node->first; +} + +internal String8 +t_scalar_string_from_node(MD_Node *node) +{ + MD_Node *scalar = t_scalar_from_node(node); + return md_node_is_nil(scalar) ? str8_zero() : scalar->string; +} + +internal void +t_context_absorb_result(T_Context *ctx, T_Result result) +{ + if (result.diagnostics.first != 0 && + result.diagnostics.first != ctx->result.diagnostics.first) { + if (ctx->result.diagnostics.last == 0) { + ctx->result.diagnostics = result.diagnostics; + } else { + ctx->result.diagnostics.last->next = result.diagnostics.first; + ctx->result.diagnostics.last = result.diagnostics.last; + ctx->result.diagnostics.count += result.diagnostics.count; + } + } + if (ctx->result.code == T_ResultCode_Ok && result.code != T_ResultCode_Ok) { ctx->result.code = result.code; } +} + +internal T_Result +t_script_parse(Arena *arena, TestCtx *test_ctx, T_SuiteSpec *suite, String8 file_path, String8 source, T_Context *ctx_out) +{ + MemoryZeroStruct(ctx_out); + + T_Context *ctx = ctx_out; + ctx->arena = arena; + ctx->test_ctx = test_ctx; + ctx->suite = suite; + ctx->file_path = str8_copy(arena, file_path); + ctx->source = str8_copy(arena, source); + + // tokenize & parse mdesk + MD_TokenizeResult tokenize = md_tokenize_from_text(arena, ctx->source); + MD_ParseResult parse = md_parse_from_text_tokens(arena, ctx->file_path, ctx->source, tokenize.tokens); + + // error check tokenize & parse + for EachNode(msg, MD_Msg, tokenize.msgs.first) { t_diagnostic_from_md_msg(ctx, msg); } + for EachNode(msg, MD_Msg, parse.msgs.first) { t_diagnostic_from_md_msg(ctx, msg); } + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + ctx->root = parse.root; + + // does script have expected root node? + { + MD_Node *test_node = ctx->root->first; + if (md_node_is_nil(test_node) || !str8_matchi(test_node->string, str8_lit("test"))) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, test_node, str8_zero(), "expected one top-level 'test' node"); + } + if (t_md_node_is_valid(test_node->next)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, test_node->next, str8_zero(), "unexpected top-level node '%S'", test_node->next->string); + } + ctx->test = test_node; + } + + // set up general parse context + T_ParseContext parse_ctx = { arena, ctx, suite, ctx->file_path, ctx->source }; + + // + // @build + // + { + MD_Node *build = t_child_from_string(ctx->test, "build"); + if (t_md_node_is_valid(build)) { + parse_ctx.operation = str8_lit("build"); + T_Result result = t_build_parse(&parse_ctx, build, &ctx->build); + t_context_absorb_result(ctx, result); + } + } + + // + // @artifacts + // + { + MD_Node *artifacts = t_child_from_string(ctx->test, "artifacts"); + + for MD_EachNode(node, artifacts->first) { + + if (t_artifact_from_name(ctx, node->string) != 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, str8_zero(), "duplicate artifact '%S'", node->string); + continue; + } + + MD_Node *definition = &md_nil_node; + String8 file_name = {0}; + B32 has_file_name = 0; + for + MD_EachNode(child, node->first) + { + if (str8_match(child->string, str8_lit("file_name"), StringMatchFlag_CaseInsensitive)) { + if (has_file_name) { t_context_errorf(ctx, T_ResultCode_ValidationError, child, str8_zero(), "artifact '%S' has duplicate file_name", node->string); } + has_file_name = 1; + if (md_node_is_nil(child->first) || !md_node_is_nil(child->first->next) || child->first->string.size == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, child, str8_zero(), "artifact '%S' file_name requires one value", node->string); + } else { + file_name = child->first->string; + if (str8_find_needle(file_name, 0, str8_lit("/"), 0) < file_name.size || str8_find_needle(file_name, 0, str8_lit("\\"), 0) < file_name.size || + str8_find_needle(file_name, 0, str8_lit(":"), 0) < file_name.size) { + t_context_errorf(ctx, T_ResultCode_ValidationError, child, str8_zero(), "artifact '%S' file_name must be a relative file name", node->string); + } + } + } else if (md_node_is_nil(definition)) { + definition = child; + } else { + definition = &md_nil_node; + break; + } + } + if (md_node_is_nil(definition)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, str8_zero(), "artifact '%S' must contain exactly one codec definition", node->string); + continue; + } + T_Codec *codec = t_codec_from_kind(suite, definition->string); + if (codec == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, definition, str8_zero(), "unknown artifact codec '%S'", definition->string); + continue; + } + T_Artifact *artifact = push_array(arena, T_Artifact, 1); + artifact->name = node->string; + artifact->file_name = file_name; + artifact->codec = codec; + artifact->definition = definition; + if (file_name.size != 0) { + for (T_Artifact *other = ctx->first_artifact; other != 0; other = other->next) { + if (other->file_name.size != 0 && str8_match(other->file_name, file_name, StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, str8_zero(), "artifact file_name '%S' is already used by '%S'", file_name, other->name); + break; + } + } + } + SLLQueuePush(ctx->first_artifact, ctx->last_artifact, artifact); + ctx->artifact_count += 1; + } + + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (artifact->codec->validate != 0) { + T_Result result = artifact->codec->validate(&parse_ctx, artifact); + t_context_absorb_result(ctx, result); + } + artifact->state = t_result_is_ok(ctx->result) ? T_ArtifactState_Validated : T_ArtifactState_Failed; + } + } + + // + // @steps + // + { + MD_Node *steps = t_child_from_string(ctx->test, "steps"); + ctx->command_count = md_node_is_nil(steps) ? 0 : md_child_count_from_node(steps); + ctx->commands = push_array(arena, T_Command, ctx->command_count); + U64 command_index = 0; + for MD_EachNode(node, steps->first) { + T_OpSpec *spec = t_op_spec_from_name(suite, node->string); + if (spec == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, node->string, "unknown operation '%S'", node->string); + continue; + } + T_Command *command = &ctx->commands[command_index++]; + command->spec = spec; + command->arguments = node; + command->location = t_txt_pt_from_offset(ctx->source, node->src_offset); + command->order = command_index; + if (spec->validate != 0) { + parse_ctx.operation = spec->name; + T_Result result = spec->validate(&parse_ctx, node); + t_context_absorb_result(ctx, result); + } + } + ctx->command_count = command_index; + } + return ctx->result; +} + +internal T_Result +t_script_execute(T_Context *ctx) +{ + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (artifact->codec->encode == 0) { continue; } + T_Result result = artifact->codec->encode(ctx, artifact); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { + artifact->state = T_ArtifactState_Failed; + return ctx->result; + } + artifact->state = T_ArtifactState_Materialized; + if (artifact->file_name.size != 0 && !t_write_file(artifact->file_name, artifact->data)) { + artifact->state = T_ArtifactState_Failed; + return t_context_errorf(ctx, T_ResultCode_IoError, artifact->definition, str8_zero(), "unable to materialize artifact '%S'", artifact->file_name); + } + } + + B32 began = 0; + if (ctx->suite->begin != 0) { + began = 1; + T_Result result = ctx->suite->begin(ctx, ctx->test); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { goto exit; } + } + { + T_Result result = t_build_execute(ctx); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { goto exit; } + } + for EachIndex(i, ctx->command_count) { + T_Command *command = &ctx->commands[i]; + T_Result result = command->spec->execute(ctx, command->arguments); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { break; } + } + +exit:; + if (began && ctx->suite->end != 0) { ctx->suite->end(ctx); } + return ctx->result; +} + +internal T_Result +t_byte_producer_error(T_Context *ctx, MD_Node *node, char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + T_Result result = t_context_errorfv(ctx, T_ResultCode_ValidationError, node, str8_zero(), fmt, args); + va_end(args); + return result; +} + +internal B32 +t_hex_digit_value(U8 c, U8 *value_out) +{ + if (c >= '0' && c <= '9') { + *value_out = c - '0'; + return 1; + } + if (c >= 'a' && c <= 'f') { + *value_out = c - 'a' + 10; + return 1; + } + if (c >= 'A' && c <= 'F') { + *value_out = c - 'A' + 10; + return 1; + } + return 0; +} + +internal T_Result +t_bytes_from_producer_list(T_Context *ctx, MD_Node *first, String8 *data_out) +{ + String8List parts = {0}; + for + MD_EachNode(node, first) + { + String8 part = {0}; + T_Result result = t_bytes_from_producer(ctx, node, &part); + if (!t_result_is_ok(result)) { return result; } + str8_list_push(ctx->arena, &parts, part); + } + *data_out = str8_list_join(ctx->arena, &parts, 0); + return ctx->result; +} + +internal T_Result +t_bytes_from_producer(T_Context *ctx, MD_Node *producer, String8 *data_out) +{ + if (str8_match(producer->string, str8_lit("data"), StringMatchFlag_CaseInsensitive)) { + if (md_node_is_nil(producer->first) || !md_node_is_nil(producer->first->next)) { return t_byte_producer_error(ctx, producer, "data must contain exactly one byte producer"); } + return t_bytes_from_producer(ctx, producer->first, data_out); + } + + if (str8_match(producer->string, str8_lit("concat"), StringMatchFlag_CaseInsensitive)) { return t_bytes_from_producer_list(ctx, producer->first, data_out); } + + U64 size_prefix = 0; + if (str8_match(producer->string, str8_lit("size16le"), StringMatchFlag_CaseInsensitive)) { size_prefix = 2; } + if (str8_match(producer->string, str8_lit("size32le"), StringMatchFlag_CaseInsensitive)) { size_prefix = 4; } + if (size_prefix != 0) { + if (md_node_is_nil(producer->first) || !md_node_is_nil(producer->first->next)) { return t_byte_producer_error(ctx, producer, "size prefix requires one byte producer"); } + String8 payload = {0}; + T_Result result = t_bytes_from_producer(ctx, producer->first, &payload); + U64 max_size = size_prefix == 2 ? max_U16 : max_U32; + if (!t_result_is_ok(result)) { return result; } + if (payload.size > max_size) { return t_byte_producer_error(ctx, producer, "payload is too large for size prefix"); } + U8 *bytes = push_array_no_zero(ctx->arena, U8, size_prefix + payload.size); + for EachIndex(i, size_prefix) { bytes[i] = (U8)(payload.size >> (i * 8)); } + MemoryCopy(bytes + size_prefix, payload.str, payload.size); + *data_out = str8(bytes, size_prefix + payload.size); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("align4"), StringMatchFlag_CaseInsensitive)) { + if (md_node_is_nil(producer->first) || !md_node_is_nil(producer->first->next)) { return t_byte_producer_error(ctx, producer, "align4 requires one byte producer"); } + String8 payload = {0}; + T_Result result = t_bytes_from_producer(ctx, producer->first, &payload); + if (!t_result_is_ok(result)) { return result; } + U64 aligned_size = AlignPow2(payload.size, 4); + U8 *bytes = push_array(ctx->arena, U8, aligned_size); + MemoryCopy(bytes, payload.str, payload.size); + *data_out = str8(bytes, aligned_size); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("text"), StringMatchFlag_CaseInsensitive)) { + MD_Node *value = producer->first; + if (md_node_is_nil(value) || !md_node_is_nil(value->next)) { return t_byte_producer_error(ctx, producer, "text requires one value"); } + *data_out = str8_copy(ctx->arena, value->string); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("work_path"), StringMatchFlag_CaseInsensitive)) { + String8 file_name = t_scalar_string_from_node(producer); + if (file_name.size == 0 || str8_find_needle(file_name, 0, str8_lit("/"), 0) < file_name.size || str8_find_needle(file_name, 0, str8_lit("\\"), 0) < file_name.size || + str8_find_needle(file_name, 0, str8_lit(":"), 0) < file_name.size) { + return t_byte_producer_error(ctx, producer, "work_path requires a relative file name"); + } + *data_out = t_make_file_path(ctx->arena, file_name); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("hex"), StringMatchFlag_CaseInsensitive)) { + MD_Node *value = producer->first; + if (md_node_is_nil(value) || !md_node_is_nil(value->next)) { return t_byte_producer_error(ctx, producer, "hex requires one string"); } + U8 *bytes = push_array_no_zero(ctx->arena, U8, value->string.size / 2 + 1); + U64 count = 0; + U8 high = 0; + B32 has_high = 0; + for + EachIndex(i, value->string.size) + { + U8 c = value->string.str[i]; + if (char_is_space(c) || c == '_') { continue; } + U8 digit = 0; + if (!t_hex_digit_value(c, &digit)) { return t_byte_producer_error(ctx, value, "invalid hex digit '%c'", c); } + if (!has_high) { + high = digit; + has_high = 1; + } else { + bytes[count++] = (high << 4) | digit; + has_high = 0; + } + } + if (has_high) { return t_byte_producer_error(ctx, value, "hex producer contains an odd number of digits"); } + *data_out = str8(bytes, count); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("zero"), StringMatchFlag_CaseInsensitive)) { + U64 count = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(producer), &count)) { return t_byte_producer_error(ctx, producer, "zero requires a byte count"); } + *data_out = str8(push_array(ctx->arena, U8, count), count); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("repeat"), StringMatchFlag_CaseInsensitive)) { + MD_Node *value_node = producer->first; + MD_Node *count_node = md_node_is_nil(value_node) ? &md_nil_node : value_node->next; + U64 value = 0; + U64 count = 0; + if (md_node_is_nil(value_node) || md_node_is_nil(count_node) || !md_node_is_nil(count_node->next) || !try_u64_from_str8_c_rules(value_node->string, &value) || value > max_U8 || + !try_u64_from_str8_c_rules(count_node->string, &count)) { + return t_byte_producer_error(ctx, producer, "repeat requires a byte value and count"); + } + U8 *bytes = push_array_no_zero(ctx->arena, U8, count); + MemorySet(bytes, (U8)value, count); + *data_out = str8(bytes, count); + return ctx->result; + } + + U64 integer_size = 0; + if (str8_match(producer->string, str8_lit("u16le"), StringMatchFlag_CaseInsensitive)) { integer_size = 2; } + if (str8_match(producer->string, str8_lit("u32le"), StringMatchFlag_CaseInsensitive)) { integer_size = 4; } + if (str8_match(producer->string, str8_lit("u64le"), StringMatchFlag_CaseInsensitive)) { integer_size = 8; } + if (integer_size != 0) { + U64 value = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(producer), &value) || (integer_size < 8 && value >= (1ull << (integer_size * 8)))) { + return t_byte_producer_error(ctx, producer, "%S value is out of range", producer->string); + } + U8 *bytes = push_array_no_zero(ctx->arena, U8, integer_size); + for + EachIndex(i, integer_size) { bytes[i] = (U8)(value >> (i * 8)); } + *data_out = str8(bytes, integer_size); + return ctx->result; + } + + if (md_node_is_nil(producer->first)) { + *data_out = str8_copy(ctx->arena, producer->string); + return ctx->result; + } + return t_byte_producer_error(ctx, producer, "unknown byte producer '%S'", producer->string); +} + +internal MD_Node * +t_push_md_child(Arena *arena, MD_Node *parent, String8 string) +{ + MD_Node *node = md_push_node(arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, string, string, 0); + md_node_push_child(parent, node); + return node; +} + +internal T_Result +t_codec_bytes_encode(T_Context *ctx, T_Artifact *artifact) +{ + MD_Node *data = t_child_from_string(artifact->definition, "data"); + if (md_node_is_nil(data)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_zero(), "bytes artifact requires data"); } + return t_bytes_from_producer(ctx, data, &artifact->data); +} + +internal T_Result +t_codec_bytes_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + MD_Node *root = md_push_node(ctx->arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, str8_lit("bytes"), str8_lit("bytes"), 0); + MD_Node *size = t_push_md_child(ctx->arena, root, str8_lit("size")); + t_push_md_child(ctx->arena, size, str8f(ctx->arena, "%llu", data.size)); + MD_Node *hex = t_push_md_child(ctx->arena, root, str8_lit("hex")); + U8 *hex_data = push_array_no_zero(ctx->arena, U8, data.size * 2); + local_persist U8 digits[] = "0123456789abcdef"; + for + EachIndex(i, data.size) + { + hex_data[i * 2 + 0] = digits[data.str[i] >> 4]; + hex_data[i * 2 + 1] = digits[data.str[i] & 15]; + } + t_push_md_child(ctx->arena, hex, str8(hex_data, data.size * 2)); + *semantic_tree_out = root; + return ctx->result; +} + +internal T_Result +t_codec_text_encode(T_Context *ctx, T_Artifact *artifact) +{ + MD_Node *data = t_child_from_string(artifact->definition, "data"); + if (md_node_is_nil(data)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_zero(), "text artifact requires data"); } + return t_bytes_from_producer(ctx, data, &artifact->data); +} + +internal T_Result +t_codec_text_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + MD_Node *root = md_push_node(ctx->arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, str8_lit("text"), str8_lit("text"), 0); + MD_Node *value = t_push_md_child(ctx->arena, root, str8_lit("data")); + t_push_md_child(ctx->arena, value, str8_copy(ctx->arena, data)); + *semantic_tree_out = root; + return ctx->result; +} + +internal T_Result +t_op_compare_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + MD_Node *left = t_child_from_string(arguments, "left"); + MD_Node *right = t_child_from_string(arguments, "right"); + String8 left_name = t_scalar_string_from_node(left); + String8 right_name = t_scalar_string_from_node(right); + if (left_name.size == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare requires left"); + } else if (t_artifact_from_name(ctx->run, left_name) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, left, "unknown artifact '%S'", left_name); + } + if (right_name.size == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare requires right"); + } else if (t_artifact_from_name(ctx->run, right_name) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, right, "unknown artifact '%S'", right_name); + } + return ctx->run->result; +} + +internal T_Result +t_op_compare_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 left_name = t_scalar_string_from_node(t_child_from_string(arguments, "left")); + String8 right_name = t_scalar_string_from_node(t_child_from_string(arguments, "right")); + T_Artifact *left = t_artifact_from_name(ctx, left_name); + T_Artifact *right = t_artifact_from_name(ctx, right_name); + if (!str8_match(left->data, right->data, 0)) { + U64 mismatch = 0; + U64 common_size = Min(left->data.size, right->data.size); + while (mismatch < common_size && left->data.str[mismatch] == right->data.str[mismatch]) { mismatch += 1; } + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("compare"), "artifacts '%S' and '%S' differ at byte %llu (sizes %llu and %llu)", left_name, right_name, + mismatch, left->data.size, right->data.size); + } + return ctx->result; +} + +internal B32 +t_u64_from_scalar(MD_Node *node, U64 *value_out) +{ + return try_u64_from_str8_c_rules(t_scalar_string_from_node(node), value_out); +} + +internal T_Result +t_semantic_match_all(T_Context *ctx, MD_Node *expected, MD_Node *actual, MD_Node *tag, String8 path, String8 field_path) +{ + U64 dot = str8_find_needle(field_path, 0, str8_lit("."), 0); + String8 field = str8_prefix(field_path, dot); + String8 rest = dot < field_path.size ? str8_skip(field_path, dot + 1) : str8_zero(); + if (field.size == 0) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, tag, str8_zero(), "all requires a non-empty field path"); + } + + if (str8_match(field, str8_lit("*"), 0)) { + for MD_EachNode(child, actual->first) { + String8 child_path = str8f(ctx->arena, "%S.%S", path, child->string); + if (rest.size == 0) { + B32 matches = 0; + for (MD_Node *allowed = tag->first->next; !md_node_is_nil(allowed); allowed = allowed->next) { + if (str8_match(t_scalar_string_from_node(child), allowed->string, 0)) { matches = 1; break; } + } + if (!matches) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is not allowed", child_path); } + } else { + T_Result result = t_semantic_match_all(ctx, expected, child, tag, child_path, rest); + if (!t_result_is_ok(result)) { return result; } + } + } + return ctx->result; + } + + MD_Node *child = md_child_from_string(actual, field, 0); + String8 child_path = str8f(ctx->arena, "%S.%S", path, field); + if (md_node_is_nil(child)) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: expected field is missing", child_path); } + if (rest.size != 0) { return t_semantic_match_all(ctx, expected, child, tag, child_path, rest); } + + B32 matches = 0; + for (MD_Node *allowed = tag->first->next; !md_node_is_nil(allowed); allowed = allowed->next) { + if (str8_match(t_scalar_string_from_node(child), allowed->string, 0)) { matches = 1; break; } + } + if (!matches) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is not allowed", child_path); } + return ctx->result; +} + +internal T_Result +t_semantic_match_node(T_Context *ctx, MD_Node *expected, MD_Node *actual, String8 path) +{ + for MD_EachNode(tag, expected->first_tag) { + if (str8_match(tag->string, str8_lit("exists"), StringMatchFlag_CaseInsensitive)) { continue; } + if (str8_match(tag->string, str8_lit("count"), StringMatchFlag_CaseInsensitive)) { + U64 count = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(tag), &count) || md_child_count_from_node(actual) != count) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: child count does not match", path); + } + continue; + } + if (str8_match(tag->string, str8_lit("all"), StringMatchFlag_CaseInsensitive)) { + MD_Node *field_path = tag->first; + if (md_node_is_nil(field_path) || md_node_is_nil(field_path->next)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, tag, str8_zero(), "all requires a field path and at least one allowed value"); + } + T_Result result = t_semantic_match_all(ctx, expected, actual, tag, path, field_path->string); + if (!t_result_is_ok(result)) { return result; } + continue; + } + if (str8_match(tag->string, str8_lit("contains"), StringMatchFlag_CaseInsensitive)) { + String8 needle = t_scalar_string_from_node(tag); + String8 value = t_scalar_string_from_node(actual); + if (str8_find_needle(value, 0, needle, 0) >= value.size) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: '%S' does not contain '%S'", path, value, needle); + } + continue; + } + if (str8_match(tag->string, str8_lit("starts_with"), StringMatchFlag_CaseInsensitive)) { + String8 prefix = t_scalar_string_from_node(tag); + String8 value = t_scalar_string_from_node(actual); + if (!str8_starts_with(value, prefix)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: '%S' does not start with '%S'", path, value, prefix); + } + continue; + } + if (str8_match(tag->string, str8_lit("aligned"), StringMatchFlag_CaseInsensitive)) { + U64 align = 0; + U64 value = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(tag), &align) || align == 0 || !t_u64_from_scalar(actual, &value) || value % align != 0) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is not aligned", path); + } + continue; + } + if (str8_match(tag->string, str8_lit("range"), StringMatchFlag_CaseInsensitive)) { + MD_Node *min_node = tag->first; + MD_Node *max_node = md_node_is_nil(min_node) ? &md_nil_node : min_node->next; + U64 min = 0, max = 0, value = 0; + if (md_node_is_nil(max_node) || !md_node_is_nil(max_node->next) || !try_u64_from_str8_c_rules(min_node->string, &min) || + !try_u64_from_str8_c_rules(max_node->string, &max) || !t_u64_from_scalar(actual, &value) || value < min || value > max) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is outside expected range", path); + } + continue; + } + if (str8_match(tag->string, str8_lit("bits_set"), StringMatchFlag_CaseInsensitive) || str8_match(tag->string, str8_lit("bits_clear"), StringMatchFlag_CaseInsensitive)) { + U64 mask = 0, value = 0; + B32 bits_set = str8_match(tag->string, str8_lit("bits_set"), StringMatchFlag_CaseInsensitive); + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(tag), &mask) || !t_u64_from_scalar(actual, &value) || (bits_set ? (value & mask) != mask : (value & mask) != 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: required bits are not %s", path, bits_set ? "set" : "clear"); + } + continue; + } + if (str8_match(tag->string, str8_lit("absent"), StringMatchFlag_CaseInsensitive)) { continue; } + return t_context_errorf(ctx, T_ResultCode_ValidationError, tag, str8_zero(), "unknown semantic predicate '%S'", tag->string); + } + + for MD_EachNode(expected_child, expected->first) { + MD_Node *actual_child = md_child_from_string(actual, expected_child->string, 0); + String8 child_path = path.size == 0 ? expected_child->string : str8f(ctx->arena, "%S.%S", path, expected_child->string); + B32 expect_absent = !md_node_is_nil(md_tag_from_string(expected_child, str8_lit("absent"), StringMatchFlag_CaseInsensitive)); + if (expect_absent) { + if (!md_node_is_nil(actual_child)) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected_child, str8_zero(), "%S: field must be absent", child_path); } + continue; + } + if (md_node_is_nil(actual_child)) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected_child, str8_zero(), "%S: expected field is missing", child_path); } + T_Result result = t_semantic_match_node(ctx, expected_child, actual_child, child_path); + if (!t_result_is_ok(result)) { return result; } + } + + return ctx->result; +} + +internal T_Result +t_semantic_match(T_Context *ctx, MD_Node *expected, MD_Node *actual) +{ + if (!str8_match(expected->string, actual->string, 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "expected '%S', got '%S'", expected->string, actual->string); + } + return t_semantic_match_node(ctx, expected, actual, expected->string); +} diff --git a/src/torture/torture_script.h b/src/torture/torture_script.h new file mode 100644 index 000000000..281ae753e --- /dev/null +++ b/src/torture/torture_script.h @@ -0,0 +1,165 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +typedef struct T_Context T_Context; +typedef struct T_ParseContext T_ParseContext; +typedef struct T_Codec T_Codec; +typedef struct T_OpSpec T_OpSpec; +typedef struct T_SuiteSpec T_SuiteSpec; +typedef struct T_Artifact T_Artifact; +typedef struct T_BuildPlan T_BuildPlan; +typedef struct T_ScriptBinding T_ScriptBinding; + +typedef enum T_ArtifactState +{ + T_ArtifactState_Declared, + T_ArtifactState_Validated, + T_ArtifactState_Materialized, + T_ArtifactState_Failed, +} T_ArtifactState; + +typedef enum T_ResultCode +{ + T_ResultCode_Ok, + T_ResultCode_ParseError, + T_ResultCode_ValidationError, + T_ResultCode_IoError, + T_ResultCode_Mismatch, +} T_ResultCode; + +typedef enum T_DiagnosticKind +{ + T_DiagnosticKind_Null, + T_DiagnosticKind_Note, + T_DiagnosticKind_Warning, + T_DiagnosticKind_Error, +} T_DiagnosticKind; + +typedef struct T_Diagnostic T_Diagnostic; +struct T_Diagnostic +{ + T_Diagnostic *next; + MD_MsgKind kind; + String8 file_path; + TxtPt location; + String8 operation; + String8 message; +}; + +typedef struct T_DiagnosticList T_DiagnosticList; +struct T_DiagnosticList +{ + T_Diagnostic *first; + T_Diagnostic *last; + U64 count; +}; + +typedef struct T_Result T_Result; +struct T_Result +{ + T_ResultCode code; + T_DiagnosticList diagnostics; +}; + +struct T_Codec +{ + String8 kind; + T_Result (*validate)(T_ParseContext *ctx, T_Artifact *artifact); + T_Result (*encode)(T_Context *ctx, T_Artifact *artifact); + T_Result (*decode)(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +}; + +struct T_OpSpec +{ + String8 name; + T_Result (*validate)(T_ParseContext *ctx, MD_Node *arguments); + T_Result (*execute)(T_Context *ctx, MD_Node *arguments); +}; + +typedef struct T_Command T_Command; +struct T_Command +{ + T_OpSpec *spec; + MD_Node *arguments; + TxtPt location; + U64 order; +}; + +struct T_SuiteSpec +{ + String8 name; + T_Codec *codecs; + U64 codec_count; + T_OpSpec *ops; + U64 op_count; + T_Result (*begin)(T_Context *ctx, MD_Node *test); + void (*end)(T_Context *ctx); +}; + +struct T_Artifact +{ + T_Artifact *next; + String8 name; + String8 file_name; + T_Codec *codec; + MD_Node *definition; + String8 data; + void *codec_data; + T_ArtifactState state; +}; + +struct T_Context +{ + Arena *arena; + TestCtx *test_ctx; + T_SuiteSpec *suite; + String8 file_path; + String8 source; + MD_Node *root; + MD_Node *test; + T_Artifact *first_artifact; + T_Artifact *last_artifact; + U64 artifact_count; + T_BuildPlan *build; + T_ScriptBinding *bindings; + T_Command *commands; + U64 command_count; + T_Result result; + void *suite_data; +}; + +struct T_ParseContext +{ + Arena *arena; + T_Context *run; + T_SuiteSpec *suite; + String8 file_path; + String8 source; + String8 operation; +}; + +internal B32 t_result_is_ok(T_Result result); +internal T_Result t_context_errorf(T_Context *ctx, T_ResultCode code, MD_Node *node, String8 operation, char *fmt, ...); +internal T_Result t_parse_errorf(T_ParseContext *ctx, T_ResultCode code, MD_Node *node, char *fmt, ...); + +internal T_Codec *t_codec_from_kind(T_SuiteSpec *suite, String8 kind); +internal T_OpSpec *t_op_spec_from_name(T_SuiteSpec *suite, String8 name); +internal T_Artifact *t_artifact_from_name(T_Context *ctx, String8 name); + +internal T_Result t_script_parse(Arena *arena, TestCtx *test_ctx, T_SuiteSpec *suite, String8 file_path, String8 source, T_Context *ctx_out); +internal T_Result t_script_execute(T_Context *ctx); +internal T_Result t_bytes_from_producer(T_Context *ctx, MD_Node *producer, String8 *data_out); + +internal T_Result t_codec_bytes_encode(T_Context *ctx, T_Artifact *artifact); +internal T_Result t_codec_bytes_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +internal T_Result t_codec_text_encode(T_Context *ctx, T_Artifact *artifact); +internal T_Result t_codec_text_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +internal T_Result t_op_compare_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_op_compare_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_semantic_match(T_Context *ctx, MD_Node *expected, MD_Node *actual); + +global T_Codec t_codec_bytes; +global T_Codec t_codec_text; +global T_OpSpec t_op_compare; diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index a67c204c6..09d59e010 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -695,7 +695,63 @@ semaphore_take(Semaphore semaphore, U64 endt_us) internal void semaphore_drop_count(Semaphore semaphore, U64 drop_count) { - ReleaseSemaphore((HANDLE)*semaphore.u64, drop_count, 0); + BOOL ok = ReleaseSemaphore((HANDLE)*semaphore.u64, drop_count, 0); + // Do not log here: stderr may itself be a detoured pipe, and blocking before the assertion + // would convert a synchronization failure into an undiagnosable process hang. + AssertAlways(ok); +} + +internal B32 +semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out) +{ + LONG prev = 0; + BOOL ok = ReleaseSemaphore((HANDLE)*semaphore.u64, 1, &prev); + *prev_count_out = ok ? (U32)prev : 0; + return !!ok; +} + +// Best-effort post: succeed if there is room, silently no-op if the count is +// already at max (ERROR_TOO_MANY_POSTS). Use ONLY for "at least one pending +// signal" wakeups (e.g. the governor ping) where redundant posts are harmless. +// Any OTHER failure is still a hard error. +internal void +semaphore_drop_if_room(Semaphore semaphore) +{ + HANDLE handle = (HANDLE)semaphore.u64[0]; + BOOL ok = ReleaseSemaphore(handle, 1, 0); + if (!ok) { + DWORD err = GetLastError(); + AssertAlways(err == ERROR_TOO_MANY_POSTS); + } +} + +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if (count > 0) { + HANDLE handle = (HANDLE)semaphore.u64[0]; + BOOL ok = ReleaseSemaphore(handle, count, 0); + if (!ok) { + // ReleaseSemaphore is all-or-nothing: ERROR_TOO_MANY_POSTS does not clamp + // the count. Ignoring any failure can therefore strand a thread-pool grant. + AssertAlways(ok); + } + } +} + +internal B32 +semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us) +{ + // Blocking acquire of `count` permits, one at a time. Off the hot path only: + // used by the shared thread-pool barrier-reserve path to gather budget slots. + for (U32 i = 0; i < count; i += 1) { + if (!semaphore_take(semaphore, endt_us)) { + // partial failure: give back what we took so we don't leak permits + semaphore_drop_n(semaphore, i); + return 0; + } + } + return 1; } //- rjf: barriers